feat: conform to chat protocol (#67)

packages/chat-component/src/core/http/index.ts

@@ -3,19 +3,23 @@ export async function callHttpApi(
   { method, url, stream }: ChatHttpOptions,
 ) {
   const chatBody = JSON.stringify({
-    history: [
+    messages: [
       {
         user: question,
       },
     ],
-    approach,
-    overrides,
+    context: {
+      ...overrides,
+      approach,
+    },
     stream,
   });
   const askBody = JSON.stringify({
     question,
-    approach,
-    overrides,
+    context: {
+      ...overrides,
+      approach,
+    },
     stream: false,
   });
   const body = type === 'chat' ? chatBody : askBody;

packages/search/src/lib/approaches/approach-base.ts

@@ -1,7 +1,7 @@
 import { type SearchClient } from '@azure/search-documents';
 import { type OpenAiService } from '../../plugins/openai.js';
 import { parseBoolean, removeNewlines } from '../util/index.js';
-import { type ApproachOverrides } from './approach.js';
+import { type ApproachContext } from './approach.js';
 
 export interface SearchDocumentsResult {
   query: string;
@@ -19,12 +19,12 @@ export class ApproachBase {
     protected contentField: string,
   ) {}
 
-  protected async searchDocuments(query?: string, overrides: ApproachOverrides = {}): Promise<SearchDocumentsResult> {
-    const hasText = ['text', 'hybrid', undefined].includes(overrides?.retrieval_mode);
-    const hasVectors = ['vectors', 'hybrid', undefined].includes(overrides?.retrieval_mode);
-    const useSemanticCaption = parseBoolean(overrides?.semantic_captions) && hasText;
-    const top = overrides?.top ? Number(overrides?.top) : 3;
-    const excludeCategory: string | undefined = overrides?.exclude_category;
+  protected async searchDocuments(query?: string, context: ApproachContext = {}): Promise<SearchDocumentsResult> {
+    const hasText = ['text', 'hybrid', undefined].includes(context?.retrieval_mode);
+    const hasVectors = ['vectors', 'hybrid', undefined].includes(context?.retrieval_mode);
+    const useSemanticCaption = parseBoolean(context?.semantic_captions) && hasText;
+    const top = context?.top ? Number(context?.top) : 3;
+    const excludeCategory: string | undefined = context?.exclude_category;
     const filter = excludeCategory ? `category ne '${excludeCategory.replace("'", "''")}'` : undefined;
 
     // If retrieval mode includes vectors, compute an embedding for the query
@@ -42,7 +42,7 @@ export class ApproachBase {
     const queryText = hasText ? query : '';
 
     // Use semantic L2 reranker if requested and if retrieval mode is text or hybrid (vectors + text)
-    const searchResults = await (overrides?.semantic_ranker && hasText
+    const searchResults = await (context?.semantic_ranker && hasText
       ? this.search.search(queryText, {
           filter,
           queryType: 'semantic',

packages/search/src/lib/approaches/approach.ts

@@ -1,14 +1,31 @@
-import { type HistoryMessage } from '../message.js';
+import { type Message, type HistoryMessage } from '../message.js';
 
 export interface ApproachResponse {
-  data_points: string[];
-  answer: string;
-  thoughts: string;
+  choices: Array<{
+    index: number;
+    message: ApproachResponseMessage;
+  }>;
+  object: 'chat.completion';
 }
 
-export type ApproachResponseChunk = Partial<ApproachResponse>;
+export interface ApproachResponseChunk {
+  choices: Array<{
+    index: number;
+    delta: Partial<ApproachResponseMessage>;
+    finish_reason: string | null;
+  }>;
+  object: 'chat.completion.chunk';
+}
+
+export type ApproachResponseMessage = Message & {
+  context?: Record<string, any> & {
+    data_points?: string[];
+    thoughts?: string;
+  };
+  session_state?: Record<string, any>;
+};
 
-export type ApproachOverrides = {
+export type ApproachContext = {
   retrieval_mode?: 'hybrid' | 'text' | 'vectors';
   semantic_ranker?: boolean;
   semantic_captions?: boolean;
@@ -20,19 +37,19 @@ export type ApproachOverrides = {
   exclude_category?: string;
 };
 
-export type ChatApproachOverrides = ApproachOverrides & {
+export type ChatApproachContext = ApproachContext & {
   suggest_followup_questions?: boolean;
 };
 
 export interface ChatApproach {
-  run(history: HistoryMessage[], overrides?: ChatApproachOverrides): Promise<ApproachResponse>;
+  run(history: HistoryMessage[], context?: ChatApproachContext): Promise<ApproachResponse>;
   runWithStreaming(
     history: HistoryMessage[],
-    overrides?: ChatApproachOverrides,
+    context?: ChatApproachContext,
   ): AsyncGenerator<ApproachResponseChunk, void>;
 }
 
 export interface AskApproach {
-  run(query: string, overrides?: ApproachOverrides): Promise<ApproachResponse>;
-  runWithStreaming(query: string, overrides?: ApproachOverrides): AsyncGenerator<ApproachResponseChunk, void>;
+  run(query: string, context?: ApproachContext): Promise<ApproachResponse>;
+  runWithStreaming(query: string, context?: ApproachContext): AsyncGenerator<ApproachResponseChunk, void>;
 }

packages/search/src/lib/approaches/ask-read-retrieve-read.ts

@@ -9,7 +9,7 @@ import { type LangchainService } from '../../plugins/langchain.js';
 import { CsvLookupTool, HtmlCallbackHandler } from '../langchain/index.js';
 import {
   type ApproachResponseChunk,
-  type ApproachOverrides,
+  type ApproachContext,
   type AskApproach,
   type ApproachResponse,
 } from './approach.js';
@@ -57,15 +57,15 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
     super(search, openai, chatGptModel, embeddingModel, sourcePageField, contentField);
   }
 
-  async run(userQuery: string, overrides?: ApproachOverrides): Promise<ApproachResponse> {
+  async run(userQuery: string, context?: ApproachContext): Promise<ApproachResponse> {
     let searchResults: string[] = [];
 
     const htmlTracer = new HtmlCallbackHandler();
     const callbackManager = new CallbackManager();
     callbackManager.addHandler(htmlTracer);
 
     const searchAndStore = async (query: string): Promise<string> => {
-      const { results, content } = await this.searchDocuments(query, overrides);
+      const { results, content } = await this.searchDocuments(query, context);
       searchResults = results;
       return content;
     };
@@ -82,14 +82,14 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
     ];
 
     const chatModel = await this.langchain.getChat({
-      temperature: Number(overrides?.temperature) || 0.3,
+      temperature: Number(context?.temperature) || 0.3,
     });
 
     const executor = await initializeAgentExecutorWithOptions(tools, chatModel, {
       agentType: 'chat-zero-shot-react-description',
       agentArgs: {
-        prefix: overrides?.prompt_template_prefix ?? TEMPLATE_PREFIX,
-        suffix: overrides?.prompt_template_suffix ?? TEMPLATE_SUFFIX,
+        prefix: context?.prompt_template_prefix ?? TEMPLATE_PREFIX,
+        suffix: context?.prompt_template_suffix ?? TEMPLATE_SUFFIX,
         inputVariables: ['input', 'agent_scratchpad'],
       },
       returnIntermediateSteps: true,
@@ -103,14 +103,25 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
     const answer = result.output.replace('[CognitiveSearch]', '').replace('[Employee]', '');
 
     return {
-      data_points: searchResults,
-      answer,
-      thoughts: htmlTracer.getAndResetLog(),
+      choices: [
+        {
+          index: 0,
+          message: {
+            content: answer,
+            role: 'assistant' as const,
+            context: {
+              data_points: searchResults,
+              thoughts: htmlTracer.getAndResetLog(),
+            },
+          },
+        },
+      ],
+      object: 'chat.completion',
     };
   }
 
   // eslint-disable-next-line require-yield
-  async *runWithStreaming(_query: string, _overrides?: ApproachOverrides): AsyncGenerator<ApproachResponseChunk, void> {
+  async *runWithStreaming(_query: string, _context?: ApproachContext): AsyncGenerator<ApproachResponseChunk, void> {
     throw new Error('Streaming not supported for this approach.');
   }
 }

packages/search/src/lib/approaches/ask-retrieve-then-read.ts

@@ -4,7 +4,7 @@ import { messagesToString } from '../message.js';
 import { MessageBuilder } from '../message-builder.js';
 import {
   type ApproachResponse,
-  type ApproachOverrides,
+  type ApproachContext,
   type AskApproach,
   type ApproachResponseChunk,
 } from './approach.js';
@@ -46,9 +46,9 @@ export class AskRetrieveThenRead extends ApproachBase implements AskApproach {
     super(search, openai, chatGptModel, embeddingModel, sourcePageField, contentField);
   }
 
-  async run(userQuery: string, overrides?: ApproachOverrides): Promise<ApproachResponse> {
-    const { query, results, content } = await this.searchDocuments(userQuery, overrides);
-    const messageBuilder = new MessageBuilder(overrides?.prompt_template || SYSTEM_CHAT_TEMPLATE, this.chatGptModel);
+  async run(userQuery: string, context?: ApproachContext): Promise<ApproachResponse> {
+    const { query, results, content } = await this.searchDocuments(userQuery, context);
+    const messageBuilder = new MessageBuilder(context?.prompt_template || SYSTEM_CHAT_TEMPLATE, this.chatGptModel);
 
     // Add user question
     const userContent = `${userQuery}\nSources:\n${content}`;
@@ -64,22 +64,33 @@ export class AskRetrieveThenRead extends ApproachBase implements AskApproach {
     const chatCompletion = await openAiChat.completions.create({
       model: this.chatGptModel,
       messages,
-      temperature: Number(overrides?.temperature ?? 0.3),
+      temperature: Number(context?.temperature ?? 0.3),
       max_tokens: 1024,
       n: 1,
     });
 
     const messageToDisplay = messagesToString(messages);
 
     return {
-      data_points: results,
-      answer: chatCompletion.choices[0].message.content ?? '',
-      thoughts: `Question:<br>${query}<br><br>Prompt:<br>${messageToDisplay.replace('\n', '<br>')}`,
+      choices: [
+        {
+          index: 0,
+          message: {
+            role: 'assistant' as const,
+            content: chatCompletion.choices[0].message.content ?? '',
+            context: {
+              data_points: results,
+              thoughts: `Question:<br>${query}<br><br>Prompt:<br>${messageToDisplay.replace('\n', '<br>')}`,
+            },
+          },
+        },
+      ],
+      object: 'chat.completion',
     };
   }
 
   // eslint-disable-next-line require-yield
-  async *runWithStreaming(_query: string, _overrides?: ApproachOverrides): AsyncGenerator<ApproachResponseChunk, void> {
+  async *runWithStreaming(_query: string, _context?: ApproachContext): AsyncGenerator<ApproachResponseChunk, void> {
     throw new Error('Streaming not supported for this approach.');
   }
 }

packages/search/src/lib/approaches/chat-read-retrieve-read.ts

@@ -3,7 +3,7 @@ import { type OpenAiService } from '../../plugins/openai.js';
 import {
   type ChatApproach,
   type ApproachResponse,
-  type ChatApproachOverrides,
+  type ChatApproachContext,
   type ApproachResponseChunk,
 } from './approach.js';
 import { ApproachBase } from './approach-base.js';
@@ -60,24 +60,35 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
     this.chatGptTokenLimit = getTokenLimit(chatGptModel);
   }
 
-  async run(history: HistoryMessage[], overrides?: ChatApproachOverrides): Promise<ApproachResponse> {
-    const { completionRequest, dataPoints, thoughts } = await this.baseRun(history, overrides);
+  async run(history: HistoryMessage[], context?: ChatApproachContext): Promise<ApproachResponse> {
+    const { completionRequest, dataPoints, thoughts } = await this.baseRun(history, context);
     const openAiChat = await this.openai.getChat();
     const chatCompletion = await openAiChat.completions.create(completionRequest);
     const chatContent = chatCompletion.choices[0].message.content ?? '';
 
     return {
-      data_points: dataPoints,
-      answer: chatContent,
-      thoughts: thoughts,
+      choices: [
+        {
+          index: 0,
+          message: {
+            content: chatContent,
+            role: 'assistant',
+            context: {
+              data_points: dataPoints,
+              thoughts: thoughts,
+            },
+          },
+        },
+      ],
+      object: 'chat.completion',
     };
   }
 
   async *runWithStreaming(
     history: HistoryMessage[],
-    overrides?: ChatApproachOverrides,
+    context?: ChatApproachContext,
   ): AsyncGenerator<ApproachResponseChunk, void> {
-    const { completionRequest, dataPoints, thoughts } = await this.baseRun(history, overrides);
+    const { completionRequest, dataPoints, thoughts } = await this.baseRun(history, context);
     const openAiChat = await this.openai.getChat();
     const chatCompletion = await openAiChat.completions.create({
       ...completionRequest,
@@ -86,16 +97,28 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
     let id = 0;
     for await (const chunk of chatCompletion) {
       const responseChunk = {
-        data_points: id === 0 ? dataPoints : undefined,
-        thoughts: id === 0 ? thoughts : undefined,
-        answer: chunk.choices[0].delta.content ?? '',
+        choices: [
+          {
+            index: 0,
+            delta: {
+              content: chunk.choices[0].delta.content ?? '',
+              role: 'assistant' as const,
+              context: {
+                data_points: id === 0 ? dataPoints : undefined,
+                thoughts: id === 0 ? thoughts : undefined,
+              },
+            },
+            finish_reason: chunk.choices[0].finish_reason,
+          },
+        ],
+        object: 'chat.completion.chunk' as const,
       };
       yield responseChunk;
       id++;
     }
   }
 
-  private async baseRun(history: HistoryMessage[], overrides?: ChatApproachOverrides) {
+  private async baseRun(history: HistoryMessage[], context?: ChatApproachContext) {
     const userQuery = 'Generate search query for: ' + history[history.length - 1].user;
 
     // STEP 1: Generate an optimized keyword search query based on the chat history and the last question
@@ -128,14 +151,14 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
     // STEP 2: Retrieve relevant documents from the search index with the GPT optimized query
     // -----------------------------------------------------------------------
 
-    const { query, results, content } = await this.searchDocuments(queryText, overrides);
-    const followUpQuestionsPrompt = overrides?.suggest_followup_questions ? FOLLOW_UP_QUESTIONS_PROMPT_CONTENT : '';
+    const { query, results, content } = await this.searchDocuments(queryText, context);
+    const followUpQuestionsPrompt = context?.suggest_followup_questions ? FOLLOW_UP_QUESTIONS_PROMPT_CONTENT : '';
 
     // STEP 3: Generate a contextual and content specific answer using the search results and chat history
     // -----------------------------------------------------------------------
 
     // Allow client to replace the entire prompt, or to inject into the exiting prompt using >>>
-    const promptOverride = overrides?.prompt_template;
+    const promptOverride = context?.prompt_template;
     let systemMessage: string;
     if (promptOverride?.startsWith('>>>')) {
       systemMessage = SYSTEM_MESSAGE_CHAT_CONVERSATION.replace(
@@ -171,7 +194,7 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
       completionRequest: {
         model: this.chatGptModel,
         messages: finalMessages,
-        temperature: Number(overrides?.temperature ?? 0.7),
+        temperature: Number(context?.temperature ?? 0.7),
         max_tokens: 1024,
         n: 1,
       },
@@ -200,6 +223,9 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
     messageBuilder.appendMessage('user', userContent, appendIndex);
 
     for (const historyMessage of history.slice(0, -1).reverse()) {
+      if (messageBuilder.tokens > maxTokens) {
+        break;
+      }
       if (historyMessage.bot) {
         messageBuilder.appendMessage('assistant', historyMessage.bot, appendIndex);
       }