feat: new chat protocol extension points

README.md

@@ -435,3 +435,5 @@ Here are the most common failure scenarios and solutions:
 1. You're getting "same resource name not allowed" conflicts. That's likely because you've run the sample multiple times and deleted the resources you've been creating each time, but are forgetting to purge them. Azure keeps resources for 48 hours unless you purge from soft delete. See [this article on purging resources](https://learn.microsoft.com/azure/cognitive-services/manage-resources?tabs=azure-portal#purge-a-deleted-resource).
 
 1. After running `azd up` and visiting the website, you see a '404 Not Found' in the browser. Wait 10 minutes and try again, as it might be still starting up. Then try running `azd deploy` and wait again. If you still encounter errors with the deployed app, consult these [tips for debugging App Service app deployments](http://blog.pamelafox.org/2023/06/tips-for-debugging-flask-deployments-to.html) and file an issue if the error logs don't help you resolve the issue.
+
+1. You're getting an error `401 Principal does not have access to API/Operation` while running the project locally or trying to deploy. That's likely because your environment variables include `AZURE_TENANT_ID`, `AZURE_CLIENT_ID` and `AZURE_CLIENT_SECRET`. You should either grant permissions to the related *Service Principal* or remove these variables from your environment to ensure normal access. For more details, please refer to [Azure identity SDK](https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/identity/identity/README.md#defaultazurecredential). 

packages/chat-component/src/components/chat-controller.ts

@@ -138,7 +138,7 @@ export class ChatController implements ReactiveController {
       followingSteps.push(...(processedText.arrays[1] as string[]));
       followupQuestions.push(...(processedText.arrays[2] as string[]));
       thoughts = generatedResponse.context?.thoughts ?? '';
-      dataPoints = generatedResponse.context?.data_points ?? [];
+      dataPoints = generatedResponse.context?.data_points?.text ?? [];
 
       await updateChatWithMessageOrChunk(messageToUpdate, false);
     }

packages/chat-component/src/core/parser/index.ts

@@ -49,7 +49,7 @@ export async function parseStreamedMessages({
 
     const { content, context } = chunk.choices[0].delta;
     if (context?.data_points) {
-      updatedEntry.dataPoints = context.data_points ?? [];
+      updatedEntry.dataPoints = context.data_points?.text ?? [];
       updatedEntry.thoughts = context.thoughts ?? '';
 
       continue;

packages/chat-component/src/types.d.ts

@@ -82,7 +82,10 @@ declare interface BotResponseChunk {
 
 declare type BotResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

packages/search/src/lib/approaches/approach.ts

@@ -19,7 +19,10 @@ export interface ApproachResponseChunk {
 
 export type ApproachResponseMessage = Message & {
   context?: Record<string, any> & {
-    data_points?: string[];
+    data_points?: {
+      text?: string[];
+      images?: string[];
+    };
     thoughts?: string;
   };
   session_state?: Record<string, any>;

packages/search/src/lib/approaches/ask-read-retrieve-read.ts

@@ -110,7 +110,9 @@ export class AskReadRetrieveRead extends ApproachBase implements AskApproach {
             content: answer,
             role: 'assistant' as const,
             context: {
-              data_points: searchResults,
+              data_points: {
+                text: searchResults,
+              },
               thoughts: htmlTracer.getAndResetLog(),
             },
           },

packages/search/src/lib/approaches/ask-retrieve-then-read.ts

@@ -79,7 +79,9 @@ export class AskRetrieveThenRead extends ApproachBase implements AskApproach {
             role: 'assistant' as const,
             content: chatCompletion.choices[0].message.content ?? '',
             context: {
-              data_points: results,
+              data_points: {
+                text: results,
+              },
               thoughts: `Question:<br>${query}<br><br>Prompt:<br>${messageToDisplay.replace('\n', '<br>')}`,
             },
           },

packages/search/src/lib/approaches/chat-read-retrieve-read.ts

@@ -78,7 +78,9 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
             content: chatContent,
             role: 'assistant',
             context: {
-              data_points: dataPoints,
+              data_points: {
+                text: dataPoints,
+              },
               thoughts: thoughts,
             },
           },
@@ -108,7 +110,7 @@ export class ChatReadRetrieveRead extends ApproachBase implements ChatApproach {
               content: chunk.choices[0].delta.content ?? '',
               role: 'assistant' as const,
               context: {
-                data_points: id === 0 ? dataPoints : undefined,
+                data_points: id === 0 ? { text: dataPoints } : undefined,
                 thoughts: id === 0 ? thoughts : undefined,
               },
             },

packages/search/src/plugins/schemas.ts

@@ -59,8 +59,17 @@ export const messageSchema = {
       type: 'object',
       properties: {
         data_points: {
-          type: 'array',
-          items: { type: 'string' },
+          type: 'object',
+          properties: {
+            text: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+            images: {
+              type: 'array',
+              items: { type: 'string' },
+            },
+          },
         },
         thoughts: { type: 'string' },
       },

tests/e2e/hars/error-chat-response-stream.har

@@ -61,7 +61,7 @@
           "content": {
             "size": -1,
             "mimeType": "application/x-ndjson",
-            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":[],\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
+            "text": "{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{\"data_points\":{},\"thoughts\":\"Here's how to do it: 1.\"}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"To\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" search\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" and\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" book\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" rentals\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"content_filter\"}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\" on\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":null}],\"object\":\"chat.completion.chunk\"}\n{\"choices\":[{\"index\":0,\"delta\":{\"content\":\"\",\"role\":\"assistant\",\"context\":{}},\"finish_reason\":\"stop\"}],\"object\":\"chat.completion.chunk\"}\n"
           },
           "headersSize": -1,
           "bodySize": -1,