feat: new chat protocol extension points

Azure-Samples · Dec 19, 2023 · 8ee6cb7 · 8ee6cb7
1 parent f9ff141
commit 8ee6cb7
Show file tree

Hide file tree

Showing 12 changed files with 64 additions and 13 deletions.
diff --git a/acr.txt b/acr.txt
@@ -0,0 +1,30 @@
+azd deploy
+
+Deploying services (azd deploy)
+
+  Deploying service indexer
+  Deploying service indexer (Building Docker image)
+  (x) Failed: Deploying service indexer
+
+ERROR: failed deploying service 'indexer': failing invoking action 'deploy', failed logging into container registry, token: failed getting ACR token: POST https://crx2y5k2ccncqou.azurecr.io/oauth2/exchange
+--------------------------------------------------------------------------------
+RESPONSE 401: 401 Unauthorized
+ERROR CODE UNAVAILABLE
+--------------------------------------------------------------------------------
+Response contained no body
+--------------------------------------------------------------------------------
+, admin: getting container registry credentials: POST https://management.azure.com/subscriptions/7944cce1-9fc2-4583-a3a2-8211a625ea06/resourceGroups/rg-search-openai-js/providers/Microsoft.ContainerRegistry/registries/crx2y5k2ccncqou/listCredentials
+--------------------------------------------------------------------------------
+RESPONSE 400: 400 Bad Request
+ERROR CODE: UnAuthorizedForCredentialOperations
+--------------------------------------------------------------------------------
+{
+  "error": {
+    "code": "UnAuthorizedForCredentialOperations",
+    "message": "Cannot perform credential operations for /subscriptions/7944cce1-9fc2-4583-a3a2-8211a625ea06/resourceGroups/rg-search-openai-js/providers/Microsoft.ContainerRegistry/registries/crx2y5k2ccncqou as admin user is disabled. Kindly enable admin user as per docs: https://docs.microsoft.com/en-us/azure/container-registry/container-registry-authentication#admin-account"
+  },
+  "status": "Failed"
+}
+--------------------------------------------------------------------------------
+
+TraceID: 0c60e512e83de6e1b8af4daa7dcbe49c
diff --git a/packages/chat-component/src/components/chat-controller.ts b/packages/chat-component/src/components/chat-controller.ts
@@ -138,7 +138,7 @@ export class ChatController implements ReactiveController {
       followingSteps.push(...(processedText.arrays[1] as string[]));
       followupQuestions.push(...(processedText.arrays[2] as string[]));
       thoughts = generatedResponse.context?.thoughts ?? '';
-      dataPoints = generatedResponse.context?.data_points ?? [];
+      dataPoints = generatedResponse.context?.data_points?.text ?? [];
 
       await updateChatWithMessageOrChunk(messageToUpdate, false);
     }

diff --git a/packages/chat-component/src/core/parser/index.ts b/packages/chat-component/src/core/parser/index.ts
@@ -49,7 +49,7 @@ export async function parseStreamedMessages({
 
     const { content, context } = chunk.choices[0].delta;
     if (context?.data_points) {
-      updatedEntry.dataPoints = context.data_points ?? [];
+      updatedEntry.dataPoints = context.data_points?.text ?? [];
       updatedEntry.thoughts = context.thoughts ?? '';
 
       continue;

diff --git a/packages/chat-component/src/types.d.ts b/packages/chat-component/src/types.d.ts
@@ -82,7 +82,10 @@ declare interface BotResponseChunk {
 
 declare type BotResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

diff --git a/packages/search/src/lib/approaches/approach.ts b/packages/search/src/lib/approaches/approach.ts
@@ -19,7 +19,10 @@ export interface ApproachResponseChunk {
 
 export type ApproachResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

diff --git a/packages/search/src/lib/approaches/ask-read-retrieve-read.ts b/packages/search/src/lib/approaches/ask-read-retrieve-read.ts
@@ -110,7 +110,9 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
             content: answer,
             role: 'assistant' as const,
             context: {
-              data_points: searchResults,
+              data_points: {
+                text: searchResults,
+              },
               thoughts: htmlTracer.getAndResetLog(),
             },
           },

diff --git a/packages/search/src/lib/approaches/ask-retrieve-then-read.ts b/packages/search/src/lib/approaches/ask-retrieve-then-read.ts
@@ -79,7 +79,9 @@ export class AskRetrieveThenRead extends ApproachBase implements AskApproach {
             role: 'assistant' as const,
             content: chatCompletion.choices[0].message.content ?? '',
             context: {
-              data_points: results,
+              data_points: {
+                text: results,
+              },
               thoughts: `Question:<br>${query}<br><br>Prompt:<br>${messageToDisplay.replace('\n', '<br>')}`,
             },
           },

diff --git a/packages/search/src/lib/approaches/chat-read-retrieve-read.ts b/packages/search/src/lib/approaches/chat-read-retrieve-read.ts
@@ -78,7 +78,9 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
             content: chatContent,
             role: 'assistant',
             context: {
-              data_points: dataPoints,
+              data_points: {
+                text: dataPoints,
+              },
               thoughts: thoughts,
             },
           },
@@ -108,7 +110,7 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
               content: chunk.choices[0].delta.content ?? '',
               role: 'assistant' as const,
               context: {
-                data_points: id === 0 ? dataPoints : undefined,
+                data_points: id === 0 ? { text: dataPoints } : undefined,
                 thoughts: id === 0 ? thoughts : undefined,
               },
             },

diff --git a/packages/search/src/plugins/schemas.ts b/packages/search/src/plugins/schemas.ts
@@ -59,8 +59,17 @@ export const messageSchema = {
       type: 'object',
       properties: {
         data_points: {
-          type: 'array',
-          items: { type: 'string' },
+          type: 'object',
+          properties: {
+            text: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+            images: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+          },
         },
         thoughts: { type: 'string' },
       },

diff --git a/tests/e2e/hars/default-ask-response.har b/tests/e2e/hars/default-ask-response.har
diff --git a/tests/e2e/hars/default-chat-response-nostream.har b/tests/e2e/hars/default-chat-response-nostream.har
diff --git a/tests/e2e/hars/error-chat-response-stream.har b/tests/e2e/hars/error-chat-response-stream.har
@@ -61,7 +61,7 @@
           "content": {
             "size": -1,
             "mimeType": "application/x-ndjson",
-            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":[],\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
+            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":{}},\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
           },
           "headersSize": -1,
           "bodySize": -1,