fix(ai-help): configurable artificial error triggers

.settings.dev.toml

@@ -51,3 +51,5 @@ flag_repo = "flags"
 [ai]
 api_key = ""
 limit_reset_duration_in_sec = 3600
+trigger_error_for_search_term = "Please give me an error in the search phase of AI conversation"
+trigger_error_for_chat_term = "Please give me an error in the chat phase of the AI conversation"

.settings.test.toml

@@ -13,10 +13,10 @@ scopes = "openid profile email profile:subscriptions"
 auth_cookie_name = "auth-cookie"
 login_cookie_name = "login-cookie"
 auth_cookie_secure = false
-client_id="TEST_CLIENT_ID"
-client_secret="TEST_CLIENT_SECRET"
+client_id = "TEST_CLIENT_ID"
+client_secret = "TEST_CLIENT_SECRET"
 cookie_key = "DUwIFZuUYzRhHPlhOm6DwTHSDUSyR5SyvZHIeHdx4DIanxm5/GD/4dqXROLvn5vMofOYUq37HhhivjCyMCWP4w=="
-admin_update_bearer_token="TEST_TOKEN"
+admin_update_bearer_token = "TEST_TOKEN"
 
 [application]
 document_base_url = "http://localhost:4321"
@@ -52,3 +52,5 @@ flag_repo = "flags"
 limit_reset_duration_in_sec = 5
 api_key = ""
 explain_sign_key = "kmMAMku9PB/fTtaoLg82KjTvShg8CSZCBUNuJhUz5Pg="
+trigger_error_for_search_term = "Please give me an error in the search phase of the AI conversation"
+trigger_error_for_chat_term = "Please give me an error in the chat phase of the AI conversation"

src/ai/help.rs

@@ -18,6 +18,7 @@ use crate::{
         helpers::{cap_messages, into_user_messages, sanitize_messages},
     },
     db::SupaPool,
+    settings::SETTINGS,
 };
 
 #[derive(Eq, Hash, PartialEq, Serialize, Deserialize, Debug, Clone)]
@@ -44,6 +45,10 @@ pub async fn prepare_ai_help_req(
         AI_HELP_GPT3_5_FULL_DOC_NEW_PROMPT
     };
 
+    // // check for secret error trigger in the last message
+    // // just for QA purposes
+    qa_check_for_error_trigger(&messages)?;
+
     let open_ai_messages = sanitize_messages(messages);
 
     // TODO: sign messages os we don't check again
@@ -165,3 +170,37 @@ pub fn prepare_ai_help_summary_req(
 
     Ok(req)
 }
+
+// This function is for QA purposes only, it enables triggering
+// an error based on the input message. The message can be optionally
+// set in the settings `ai.trigger_error_for_search_term`. Nothing
+// will be triggered if the setting is missing, which should be the
+// situation in production-like environments.
+fn qa_check_for_error_trigger(
+    messages: &[ChatCompletionRequestMessage],
+) -> Result<(), async_openai::error::OpenAIError> {
+    if let Some(magic_words) = SETTINGS
+        .ai
+        .as_ref()
+        .and_then(|ai| ai.trigger_error_for_chat_term.as_ref())
+    {
+        if let Some(msg_text) = messages
+            .iter()
+            .filter(|m| m.role == Role::User)
+            .last()
+            .and_then(|m| m.content.as_ref())
+        {
+            if msg_text == magic_words {
+                return Err(async_openai::error::OpenAIError::ApiError(
+                    async_openai::error::ApiError {
+                        message: "Artificial QA error in search phase".to_string(),
+                        r#type: None,
+                        param: None,
+                        code: None,
+                    },
+                ));
+            }
+        }
+    }
+    Ok(())
+}

src/api/ai_help.rs

@@ -7,7 +7,10 @@ use actix_web_lab::{__reexports::tokio::sync::mpsc, sse};
 use async_openai::{
     config::OpenAIConfig,
     error::OpenAIError,
-    types::{ChatCompletionRequestMessage, CreateChatCompletionStreamResponse, Role::Assistant},
+    types::{
+        ChatCompletionRequestMessage, CreateChatCompletionStreamResponse,
+        Role::{self, Assistant},
+    },
     Client,
 };
 use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
@@ -30,6 +33,7 @@ use crate::{
         settings::get_settings,
         SupaPool,
     },
+    settings::SETTINGS,
 };
 use crate::{
     api::error::ApiError,
@@ -414,6 +418,8 @@ pub async fn ai_help(
 
         match prepare_res? {
             Some(ai_help_req) => {
+                qa_check_for_error_trigger(&ai_help_req.req.messages)?;
+
                 let sources = ai_help_req.refs;
                 let created_at = match record_sources(
                     &diesel_pool,
@@ -596,3 +602,30 @@ pub async fn ai_help_delete_full_history(
     delete_full_help_history(&mut conn, &user)?;
     Ok(HttpResponse::Created().finish())
 }
+
+// This function is for QA purposes only, it triggering
+// an error based on the input message. The message can be optionally
+// set in the settings `ai.trigger_error_for_chat_term`. Nothing
+// will be triggered if the setting is missing, which should be the
+// situation in production-like environments.
+fn qa_check_for_error_trigger(
+    messages: &[ChatCompletionRequestMessage],
+) -> Result<(), crate::api::error::ApiError> {
+    if let Some(magic_words) = SETTINGS
+        .ai
+        .as_ref()
+        .and_then(|ai| ai.trigger_error_for_chat_term.as_ref())
+    {
+        if let Some(msg_text) = messages
+            .iter()
+            .filter(|m| m.role == Role::User)
+            .last()
+            .and_then(|m| m.content.as_ref())
+        {
+            if msg_text == magic_words {
+                return Err(crate::api::error::ApiError::Artificial);
+            }
+        }
+    }
+    Ok(())
+}

src/settings.rs

@@ -78,6 +78,8 @@ pub struct Basket {
 #[derive(Debug, Deserialize)]
 pub struct AI {
     pub api_key: String,
+    pub trigger_error_for_search_term: Option<String>,
+    pub trigger_error_for_chat_term: Option<String>,
     pub limit_reset_duration_in_sec: i64,
     #[serde_as(as = "Base64")]
     pub explain_sign_key: [u8; 32],