feat(ai-help): log message metadata

ai-test/src/ai_help.rs

@@ -96,18 +96,16 @@ pub async fn ai_help_all(
                     function_call: None,
                 })
                 .collect();
-            if let Some(req) =
+            let (req, _) =
                 prepare_ai_help_req(openai_client, supabase_pool, !no_subscription, messages)
-                    .await?
-            {
-                let mut res = openai_client.chat().create(req.req.clone()).await?;
-                let res = res.choices.pop().map(|res| res.message);
-                let storage = Storage { req, res };
-                println!("writing: {}", json_out.display());
-                fs::write(json_out, serde_json::to_vec_pretty(&storage)?).await?;
-                println!("writing: {}", md_out.display());
-                fs::write(md_out, storage.to_md().as_bytes()).await?;
-            }
+                    .await?;
+            let mut res = openai_client.chat().create(req.req.clone()).await?;
+            let res = res.choices.pop().map(|res| res.message);
+            let storage = Storage { req, res };
+            println!("writing: {}", json_out.display());
+            fs::write(json_out, serde_json::to_vec_pretty(&storage)?).await?;
+            println!("writing: {}", md_out.display());
+            fs::write(md_out, storage.to_md().as_bytes()).await?;
             Ok(())
         })
         .await?;

migrations/2024-02-20-093804_ai_help_metadata/down.sql

@@ -0,0 +1,2 @@
+DROP TABLE ai_help_message_meta;
+DROP TYPE ai_help_message_status;

migrations/2024-02-20-093804_ai_help_metadata/up.sql

@@ -0,0 +1,30 @@
+CREATE TYPE ai_help_message_status AS ENUM (
+    'success',
+    'search_error',
+    'ai_api_error',
+    'completion_error',
+    'moderation_error',
+    'no_user_prompt_error',
+    'token_limit_error',
+    'user_stopped',
+    'user_timeout',
+    'unknown'
+);
+
+CREATE TABLE ai_help_message_meta (
+    id                  BIGSERIAL PRIMARY KEY,
+    user_id             BIGSERIAL REFERENCES users (id) ON DELETE CASCADE,
+    chat_id             UUID NOT NULL,
+    message_id          UUID NOT NULL,
+    parent_id           UUID DEFAULT NULL REFERENCES ai_help_message_meta (message_id) ON DELETE CASCADE,
+    created_at          TIMESTAMP NOT NULL DEFAULT now(),
+    search_duration     BIGINT NOT NULL,
+    response_duration   BIGINT NOT NULL,
+    query_len           BIGINT NOT NULL,
+    context_len         BIGINT NOT NULL,
+    response_len        BIGINT NOT NULL,
+    model               text NOT NULL,
+    status              ai_help_message_status NOT NULL DEFAULT 'unknown',
+    sources             JSONB NOT NULL DEFAULT '[]'::jsonb,
+    UNIQUE(message_id)
+);

src/ai/help.rs

@@ -1,3 +1,5 @@
+use std::time::{Duration, Instant};
+
 use async_openai::{
     config::OpenAIConfig,
     types::{
@@ -32,12 +34,18 @@ pub struct AIHelpRequest {
     pub refs: Vec<RefDoc>,
 }
 
+pub struct AIHelpRequestMeta {
+    pub query_len: usize,
+    pub context_len: usize,
+    pub search_duration: Duration,
+}
+
 pub async fn prepare_ai_help_req(
     client: &Client<OpenAIConfig>,
     pool: &SupaPool,
     is_subscriber: bool,
     messages: Vec<ChatCompletionRequestMessage>,
-) -> Result<Option<AIHelpRequest>, AIError> {
+) -> Result<(AIHelpRequest, AIHelpRequestMeta), AIError> {
     let config = if is_subscriber {
         AI_HELP_GPT4_FULL_DOC_NEW_PROMPT
     } else {
@@ -76,24 +84,29 @@ pub async fn prepare_ai_help_req(
         .last()
         .and_then(|msg| msg.content.as_ref())
         .ok_or(AIError::NoUserPrompt)?;
+    let query_len = last_user_message.len();
 
+    let start = Instant::now();
     let related_docs = if config.full_doc {
         get_related_macro_docs(client, pool, last_user_message.replace('\n', " ")).await?
     } else {
         get_related_docs(client, pool, last_user_message.replace('\n', " ")).await?
     };
+    let search_duration = start.elapsed();
 
     let mut context = vec![];
     let mut refs = vec![];
-    let mut token_len = 0;
+    let mut context_len = 0;
+    let mut context_token_len = 0;
     for doc in related_docs.into_iter() {
         debug!("url: {}", doc.url);
+        context_len += doc.content.len();
         let bpe = tiktoken_rs::r50k_base().unwrap();
         let tokens = bpe.encode_with_special_tokens(&doc.content).len();
-        token_len += tokens;
-        debug!("tokens: {}, token_len: {}", tokens, token_len);
-        if token_len >= config.context_limit {
-            token_len -= tokens;
+        context_token_len += tokens;
+        debug!("tokens: {}, token_len: {}", tokens, context_token_len);
+        if context_token_len >= config.context_limit {
+            context_token_len -= tokens;
             continue;
         }
         if !refs.iter().any(|r: &RefDoc| r.url == doc.url) {
@@ -139,7 +152,14 @@ pub async fn prepare_ai_help_req(
         .temperature(0.0)
         .build()?;
 
-    Ok(Some(AIHelpRequest { req, refs }))
+    Ok((
+        AIHelpRequest { req, refs },
+        AIHelpRequestMeta {
+            query_len,
+            context_len,
+            search_duration,
+        },
+    ))
 }
 
 pub fn prepare_ai_help_summary_req(