Generate prompts with placeholders

examples/cli-ai-tests/chatgpt-example.yml

@@ -8,8 +8,12 @@ config:
 scenarios:
   "Accept survey":
     setup:
+      placeholders:
+        NAME:
+          - John
+          - Jane
       prompt: |
-        I want you to play the role of a customer talking to a company's online chatbot. You must not
+        I want you to play the role of a customer called {NAME}, talking to a company's online chatbot. You must not
         break from this role, and all of your responses must be based on how a customer would realistically talk to a company's chatbot.
 
         To help you play the role of a customer consider the following points when writing a response:

packages/genesys-web-messaging-tester-cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ovotech/genesys-web-messaging-tester-cli",
-  "version": "3.0.0",
+  "version": "3.0.1",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
   "license": "Apache-2.0",

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/createAiTestCommand.ts

@@ -20,6 +20,7 @@ import * as googleAi from './chatCompletionClients/googleVertexAi/createChatComp
 import * as openAi from './chatCompletionClients/chatGpt/createChatCompletionClient';
 import { ChatCompletionClient, Utterance } from './chatCompletionClients/chatCompletionClient';
 import { validateOpenAiEnvVariables } from './chatCompletionClients/chatGpt/validateOpenAiEnvVariables';
+import { promptGenerator } from './prompt/generation/promptGenerator';
 
 export interface AiTestCommandDependencies {
   command?: Command;
@@ -155,11 +156,15 @@ export function createAiTestCommand({
         const convo = conversationFactory(session);
         const messages: Utterance[] = [];
 
+        const generatedPrompt = promptGenerator(scenario.setup);
+        outputConfig.writeOut(ui.displayPrompt(generatedPrompt));
+        outputConfig.writeOut(ui.conversationStartHeader());
+
         let endConversation: ShouldEndConversationResult = {
           hasEnded: false,
         };
         do {
-          const utterance = await chatCompletionClient.predict(scenario.setup.prompt, messages);
+          const utterance = await chatCompletionClient.predict(generatedPrompt.prompt, messages);
 
           if (utterance) {
             messages.push(utterance);
@@ -198,47 +203,6 @@ export function createAiTestCommand({
         if (scenario.followUp) {
           outputConfig.writeOut(ui.followUpDetailsUnderDevelopment());
         }
-        // if (scenario.followUp) {
-        //   const content = substituteTemplatePlaceholders(scenario.followUp.prompt, transcript);
-        //   const { choices } = await openai.chat.completions.create({
-        //     model: chatGptModel,
-        //     n: 1, // Number of choices
-        //     temperature,
-        //     messages: [
-        //       {
-        //         role: 'system',
-        //         content,
-        //       },
-        //     ],
-        //   });
-        //
-        //   if (choices[0].message?.content) {
-        //     const result = containsTerminatingPhrases(choices[0].message.content, {
-        //       fail: scenario.setup.terminatingPhrases.fail,
-        //       pass: scenario.setup.terminatingPhrases.pass,
-        //     });
-        //
-        //     outputConfig.writeOut(ui.followUpDetails(choices[0].message.content));
-        //     if (result.phraseFound) {
-        //       outputConfig.writeOut(ui.followUpResult(result));
-        //       if (result.phraseIndicates === 'fail') {
-        //         throw new CommandExpectedlyFailedError();
-        //       }
-        //     }
-        //   }
-        //
-        //   // endConversation = shouldEndConversation(
-        //   //   messages,
-        //   //   scenario.setup.terminatingPhrases.fail,
-        //   //   scenario.setup.terminatingPhrases.pass,
-        //   // );
-        //   // if (choices[0].message?.content) {
-        //   //   messages.push({ role: 'assistant', content: choices[0].message.content });
-        //   //   await convo.sendText(choices[0].message.content);
-        //   // } else {
-        //   //   messages.push({ role: 'assistant', content: '' });
-        //   // }
-        // }
       },
     );
 }

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.spec.ts

@@ -0,0 +1,42 @@
+import { AiScenarioFollowUpSection } from '../../testScript/modelTypes';
+import { promptGenerator } from './promptGenerator';
+
+test('Placeholders are replaced if value present', () => {
+  const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
+    placeholders: {
+      FIRST_NAME: ['John'],
+      SECOND_NAME: ['Doe'],
+    },
+    prompt: 'Your first name is {FIRST_NAME} and your second name is {SECOND_NAME}',
+  };
+
+  expect(promptGenerator(scenario)).toStrictEqual({
+    prompt: 'Your first name is John and your second name is Doe',
+    placeholderValues: { FIRST_NAME: 'John', SECOND_NAME: 'Doe' },
+  });
+});
+
+test('Placeholders ignored if no values present', () => {
+  const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
+    placeholders: {
+      FIRST_NAME: [],
+    },
+    prompt: 'Your first name is {FIRST_NAME}',
+  };
+
+  expect(promptGenerator(scenario)).toStrictEqual({
+    prompt: 'Your first name is {FIRST_NAME}',
+    placeholderValues: {},
+  });
+});
+
+test('Original prompt returned if placeholder values not present', () => {
+  const scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'> = {
+    prompt: 'Your first name is {FIRST_NAME}',
+  };
+
+  expect(promptGenerator(scenario)).toStrictEqual({
+    prompt: 'Your first name is {FIRST_NAME}',
+    placeholderValues: {},
+  });
+});

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/promptGenerator.ts

@@ -0,0 +1,33 @@
+import { AiScenarioFollowUpSection } from '../../testScript/modelTypes';
+import { replacePlaceholders } from './replacePlaceholders';
+
+export interface PromptGeneratorResult {
+  placeholderValues: Record<string, string>;
+  prompt: string;
+}
+
+export function promptGenerator(
+  scenario: Pick<AiScenarioFollowUpSection, 'prompt' | 'placeholders'>,
+  updatePrompt: typeof replacePlaceholders = replacePlaceholders,
+  randomIndex = (max: number) => Math.floor(Math.random() * max),
+): PromptGeneratorResult {
+  if (!scenario.placeholders) {
+    return {
+      placeholderValues: {},
+      prompt: scenario.prompt,
+    };
+  }
+
+  const chosenValues: Record<string, string> = Object.fromEntries(
+    Object.entries(scenario.placeholders)
+      .filter(([, values]) => values.length > 0)
+      .map(([placeholder, values]) => {
+        return [placeholder, values[randomIndex(values.length)]];
+      }),
+  );
+
+  return {
+    placeholderValues: chosenValues,
+    prompt: updatePrompt(scenario.prompt, chosenValues),
+  };
+}

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.spec.ts

@@ -0,0 +1,13 @@
+import { replacePlaceholders } from './replacePlaceholders';
+
+test('Placeholders are replaced', () => {
+  expect(
+    replacePlaceholders('{FIRST_NAME} {LAST_NAME} {ABC}', { FIRST_NAME: 'John', LAST_NAME: 'Doe' }),
+  ).toStrictEqual('John Doe {ABC}');
+});
+
+test('Prompt with missing placeholders are ignored', () => {
+  expect(replacePlaceholders('{FIRST_NAME} {LAST_NAME}', { FIRST_NAME: 'John' })).toStrictEqual(
+    'John {LAST_NAME}',
+  );
+});

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/generation/replacePlaceholders.ts

@@ -0,0 +1,11 @@
+export function replacePlaceholders(
+  prompt: string,
+  placeholderValues: Record<string, string>,
+): string {
+  return Object.entries(placeholderValues).reduce(
+    (previousValue, [placeholderKey, placeholderValue]) => {
+      return previousValue.replace(`{${placeholderKey}}`, placeholderValue);
+    },
+    prompt,
+  );
+}

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/prompt/shouldEndConversation.ts

@@ -37,10 +37,9 @@ export function shouldEndConversation(
     };
   }
 
-  const lastAiMsg = utterances.filter((m) => m.role === 'customer').slice(-1);
-
-  if (lastAiMsg[0]?.content) {
-    const phraseResult = containsTerminatingPhrases(lastAiMsg[0].content, {
+  const lastMsg = utterances.slice(-1);
+  if (lastMsg[0]?.content) {
+    const phraseResult = containsTerminatingPhrases(lastMsg[0].content, {
       pass: passPhrases,
       fail: failPhrases,
     });
@@ -50,7 +49,7 @@ export function shouldEndConversation(
         hasEnded: true,
         reason: {
           type: phraseResult.phraseIndicates,
-          description: `Terminating phrase found in response: '${lastAiMsg[0].content}'`,
+          description: `Terminating phrase found in response: '${lastMsg[0].content}'`,
         },
       };
     }
@@ -70,20 +69,20 @@ export function shouldEndConversation(
   //   }
   // }
 
-  const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2);
-  if (lastTwoChatBotMsgs.length === 2) {
-    const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content;
-    if (areMessagesTheSame) {
-      return {
-        hasEnded: true,
-
-        reason: {
-          type: 'fail',
-          description: 'The Chatbot repeated itself',
-        },
-      };
-    }
-  }
+  // const lastTwoChatBotMsgs = utterances.filter((m) => m.role === 'bot').slice(-2);
+  // if (lastTwoChatBotMsgs.length === 2) {
+  //   const areMessagesTheSame = lastTwoChatBotMsgs[0].content === lastTwoChatBotMsgs[1].content;
+  //   if (areMessagesTheSame) {
+  //     return {
+  //       hasEnded: true,
+  //
+  //       reason: {
+  //         type: 'fail',
+  //         description: 'The Chatbot repeated itself',
+  //       },
+  //     };
+  //   }
+  // }
 
   return { hasEnded: false };
 }

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelSchema.ts

@@ -34,14 +34,17 @@ export const schema = Joi.object<TestPromptFile>({
           temperature: Joi.number(),
         }),
       }),
-    }),
+    }).required(),
   }).required(),
   scenarios: Joi.object()
     .min(1)
     .pattern(
       /./,
       Joi.object({
         setup: Joi.object({
+          placeholders: Joi.object()
+            .min(1)
+            .pattern(/./, Joi.array().items(Joi.string()).required()),
           prompt: Joi.string().required(),
           terminatingPhrases: Joi.object({
             pass: Joi.array().items(Joi.string()).min(1).required(),

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/modelTypes.ts

@@ -8,6 +8,7 @@ export interface AiScenarioSetupSection {
 
 export interface AiScenarioFollowUpSection {
   readonly prompt: string;
+  readonly placeholders?: Record<string, string[]>;
   readonly terminatingPhrases: {
     readonly pass: string[];
     readonly fail: string[];

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/testScript/validatePromptScript.spec.ts

@@ -17,6 +17,9 @@ test('Valid', () => {
     scenarios: {
       'test-name-of-test-1': {
         setup: {
+          placeholders: {
+            TEST: ['test-1', 'test-2'],
+          },
           prompt: 'test-prompt-1',
           terminatingPhrases: {
             fail: ['test-failing-phrase-1'],
@@ -65,6 +68,9 @@ test('Valid', () => {
         'test-name-of-test-1': {
           setup: {
             prompt: 'test-prompt-1',
+            placeholders: {
+              TEST: ['test-1', 'test-2'],
+            },
             terminatingPhrases: {
               fail: ['test-failing-phrase-1'],
               pass: ['test-passing-phrase-1'],

packages/genesys-web-messaging-tester-cli/src/commands/aiTest/ui.ts

@@ -2,8 +2,8 @@ import chalk from 'chalk';
 import { ValidationError } from 'joi';
 import { ShouldEndConversationEndedResult } from './prompt/shouldEndConversation';
 import { TranscribedMessage } from '@ovotech/genesys-web-messaging-tester';
-import { PhraseFound } from './prompt/containsTerminatingPhrases';
 import { PreflightError } from './chatCompletionClients/chatCompletionClient';
+import { PromptGeneratorResult } from './prompt/generation/promptGenerator';
 
 export class Ui {
   /**
@@ -51,6 +51,14 @@ export class Ui {
     );
   }
 
+  public displayPrompt({ prompt }: PromptGeneratorResult): string {
+    return Ui.trailingNewline(chalk.grey(prompt));
+  }
+
+  public conversationStartHeader(): string {
+    return Ui.trailingNewline(['Conversation', '------------'].join('\n'));
+  }
+
   public testResult(result: ShouldEndConversationEndedResult): string {
     const resultMessage =
       result.reason.type === 'pass'
@@ -74,17 +82,4 @@ export class Ui {
       chalk.bold.yellow('Follow up definitions ignored, as functionality is under development'),
     );
   }
-
-  public followUpDetails(feedback: string): string {
-    return Ui.trailingNewline(['\n---------------------', feedback].join('\n'));
-  }
-
-  public followUpResult(result: PhraseFound): string {
-    const resultMessage =
-      result.phraseIndicates === 'fail'
-        ? chalk.bold.red(`FAILED: ${result.subject}`)
-        : chalk.bold.green(`PASSED: ${result.subject}`);
-
-    return Ui.trailingNewline(['\n---------------------', resultMessage].join('\n'));
-  }
 }