feat(ai-help): log message metadata (#424)

For each AI Help request, we start recording the following metadata:
- search duration
- response duration
- query length
- context length
- response length
- model (e.g. gpt-4-0125-preview)
- status (e.g. success)

Co-authored-by: Claas Augner <[email protected]>

ai-test/src/ai_help.rs

@@ -96,18 +96,22 @@ pub async fn ai_help_all(
                     function_call: None,
                 })
                 .collect();
-            if let Some(req) =
-                prepare_ai_help_req(openai_client, supabase_pool, !no_subscription, messages)
-                    .await?
-            {
-                let mut res = openai_client.chat().create(req.req.clone()).await?;
-                let res = res.choices.pop().map(|res| res.message);
-                let storage = Storage { req, res };
-                println!("writing: {}", json_out.display());
-                fs::write(json_out, serde_json::to_vec_pretty(&storage)?).await?;
-                println!("writing: {}", md_out.display());
-                fs::write(md_out, storage.to_md().as_bytes()).await?;
-            }
+            let mut meta = Default::default();
+            let req = prepare_ai_help_req(
+                openai_client,
+                supabase_pool,
+                !no_subscription,
+                messages,
+                &mut meta,
+            )
+            .await?;
+            let mut res = openai_client.chat().create(req.req.clone()).await?;
+            let res = res.choices.pop().map(|res| res.message);
+            let storage = Storage { req, res };
+            println!("writing: {}", json_out.display());
+            fs::write(json_out, serde_json::to_vec_pretty(&storage)?).await?;
+            println!("writing: {}", md_out.display());
+            fs::write(md_out, storage.to_md().as_bytes()).await?;
             Ok(())
         })
         .await?;

migrations/2024-02-20-093804_ai_help_metadata/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE ai_help_message_meta;
+DROP TYPE ai_help_message_status;

migrations/2024-02-20-093804_ai_help_metadata/up.sql

@@ -0,0 +1,34 @@
+CREATE TYPE ai_help_message_status AS ENUM (
+    'success',
+    'search_error',
+    'ai_api_error',
+    'completion_error',
+    'moderation_error',
+    'no_user_prompt_error',
+    'token_limit_error',
+    'timeout',
+    'finished_too_long',
+    'finished_content_filter',
+    'finished_no_reason',
+    'user_stopped',
+    'user_timeout',
+    'unknown'
+);
+
+CREATE TABLE ai_help_message_meta (
+    id                  BIGSERIAL PRIMARY KEY,
+    user_id             BIGSERIAL REFERENCES users (id) ON DELETE CASCADE,
+    chat_id             UUID NOT NULL,
+    message_id          UUID NOT NULL,
+    parent_id           UUID DEFAULT NULL,
+    created_at          TIMESTAMP NOT NULL DEFAULT now(),
+    search_duration     BIGINT DEFAULT NULL,
+    response_duration   BIGINT DEFAULT NULL,
+    query_len           BIGINT DEFAULT NULL,
+    context_len         BIGINT DEFAULT NULL,
+    response_len        BIGINT DEFAULT NULL,
+    model               text NOT NULL,
+    status              ai_help_message_status NOT NULL DEFAULT 'unknown',
+    sources             JSONB NOT NULL DEFAULT '[]'::jsonb,
+    UNIQUE(message_id)
+);

src/ai/help.rs

@@ -1,3 +1,5 @@
+use std::time::{Duration, Instant};
+
 use async_openai::{
     config::OpenAIConfig,
     types::{
@@ -33,12 +35,22 @@ pub struct AIHelpRequest {
     pub refs: Vec<RefDoc>,
 }
 
+#[derive(Default)]
+pub struct AIHelpRequestMeta {
+    pub query_len: Option<usize>,
+    pub context_len: Option<usize>,
+    pub search_duration: Option<Duration>,
+    pub model: Option<&'static str>,
+    pub sources: Option<Vec<RefDoc>>,
+}
+
 pub async fn prepare_ai_help_req(
     client: &Client<OpenAIConfig>,
     pool: &SupaPool,
     is_subscriber: bool,
     messages: Vec<ChatCompletionRequestMessage>,
-) -> Result<Option<AIHelpRequest>, AIError> {
+    request_meta: &mut AIHelpRequestMeta,
+) -> Result<AIHelpRequest, AIError> {
     let config = if is_subscriber {
         AI_HELP_GPT4_FULL_DOC_NEW_PROMPT
     } else {
@@ -81,24 +93,29 @@ pub async fn prepare_ai_help_req(
         .last()
         .and_then(|msg| msg.content.as_ref())
         .ok_or(AIError::NoUserPrompt)?;
+    request_meta.query_len = Some(last_user_message.len());
 
+    let start = Instant::now();
     let related_docs = if config.full_doc {
         get_related_macro_docs(client, pool, last_user_message.replace('\n', " ")).await?
     } else {
         get_related_docs(client, pool, last_user_message.replace('\n', " ")).await?
     };
+    request_meta.search_duration = Some(start.elapsed());
 
     let mut context = vec![];
     let mut refs = vec![];
-    let mut token_len = 0;
+    let mut context_len = 0;
+    let mut context_token_len = 0;
     for doc in related_docs.into_iter() {
         debug!("url: {}", doc.url);
+        context_len += doc.content.len();
         let bpe = tiktoken_rs::r50k_base().unwrap();
         let tokens = bpe.encode_with_special_tokens(&doc.content).len();
-        token_len += tokens;
-        debug!("tokens: {}, token_len: {}", tokens, token_len);
-        if token_len >= config.context_limit {
-            token_len -= tokens;
+        context_token_len += tokens;
+        debug!("tokens: {}, token_len: {}", tokens, context_token_len);
+        if context_token_len >= config.context_limit {
+            context_token_len -= tokens;
             continue;
         }
         if !refs.iter().any(|r: &RefDoc| r.url == doc.url) {
@@ -109,6 +126,9 @@ pub async fn prepare_ai_help_req(
         }
         context.push(doc);
     }
+    request_meta.sources = Some(refs.clone());
+    request_meta.context_len = Some(context_len);
+
     let system_message = ChatCompletionRequestMessageArgs::default()
         .role(Role::System)
         .content(config.system_prompt)
@@ -143,8 +163,9 @@ pub async fn prepare_ai_help_req(
         .messages(messages)
         .temperature(0.0)
         .build()?;
+    request_meta.model = Some(config.model);
 
-    Ok(Some(AIHelpRequest { req, refs }))
+    Ok(AIHelpRequest { req, refs })
 }
 
 pub fn prepare_ai_help_summary_req(