Feat: assisstant messages can have only the ID of the audio payload.

eng/pipelines/templates/variables/image.yml

@@ -0,0 +1,29 @@
+# Default pool image selection. Set as variable so we can override at pipeline level
+
+variables:
+  - name: LINUXPOOL
+    value: azsdk-pool-mms-ubuntu-2004-general
+  - name: LINUXNEXTPOOL
+    value: azsdk-pool-mms-ubuntu-2204-general
+  - name: WINDOWSPOOL
+    value: azsdk-pool-mms-win-2022-general
+  - name: MACPOOL
+    value: Azure Pipelines
+
+  - name: LINUXVMIMAGE
+    value: azsdk-pool-mms-ubuntu-2004-1espt
+  - name: LINUXNEXTVMIMAGE
+    value: azsdk-pool-mms-ubuntu-2204-1espt
+  - name: WINDOWSVMIMAGE
+    value: azsdk-pool-mms-win-2022-1espt
+  - name: MACVMIMAGE
+    value: macos-latest
+
+  # Values required for pool.os field in 1es pipeline templates
+  - name: LINUXOS
+    value: linux
+  - name: WINDOWSOS
+    value: windows
+  - name: MACOS
+    value: macOS
+

package.json

@@ -1,7 +1,7 @@
 {
   "name": "azure-rest-api-specs",
   "devDependencies": {
-    "@azure-tools/typespec-apiview": "0.5.0",
+    "@azure-tools/typespec-apiview": "0.4.9",
     "@azure-tools/typespec-autorest": "0.49.0",
     "@azure-tools/typespec-azure-core": "0.49.0",
     "@azure-tools/typespec-azure-portal-core": "0.49.0",
@@ -16,7 +16,7 @@
     "@typespec/openapi": "0.63.0",
     "@typespec/openapi3": "0.63.0",
     "@typespec/prettier-plugin-typespec": "0.63.0",
-    "@typespec/rest": "0.63.1",
+    "@typespec/rest": "0.63.0",
     "@typespec/streams": "0.63.0",
     "@typespec/versioning": "0.63.0",
     "azure-rest-api-specs-eng-tools": "file:eng/tools",

specification/ai/ModelInference/examples/2024-05-01-preview/GetChatCompletions_AudioModality_Gen.json

@@ -0,0 +1,99 @@
+{
+  "title": "Audio modality chat completion",
+  "operationId": "GetChatCompletions",
+  "parameters": {
+    "api-version": "2024-05-01-preview",
+    "extra-parameters": "error",
+    "body": {
+      "modalities": [
+        "text",
+        "audio"
+      ],
+      "messages": [
+        {
+          "role": "system",
+          "content": "You are a helpful assistant"
+        },
+        {
+          "role": "user",
+          "content": [
+            {
+              "type": "audio_input",
+              "audio_input": {
+                "data": "<base64 encoded audio data>",
+                "mime_type": "audio/wav"
+              }
+            }
+          ]
+        },
+        {
+          "role": "assistant",
+          "content": null,
+          "audio": {
+            "id": "abcdef1234"
+          }
+        },
+        {
+          "role": "user",
+          "content": [
+            {
+              "type": "audio_input",
+              "audio_input": {
+                "data": "<base64 encoded audio data>",
+                "mime_type": "audio/wav"
+              }
+            }
+          ]
+        }
+      ],
+      "frequency_penalty": 0,
+      "presence_penalty": 0,
+      "temperature": 0,
+      "top_p": 0,
+      "seed": 21,
+      "model": "my-model-name"
+    }
+  },
+  "responses": {
+    "200": {
+      "body": {
+        "id": "kgousajxgzyhugvqekuswuqbk",
+        "object": "chat.completion",
+        "created": 1696522361,
+        "model": "my-model-name",
+        "usage": {
+          "completion_tokens": 19,
+          "prompt_tokens": 28,
+          "total_tokens": 16,
+          "completion_tokens_details": {
+            "audio_tokens": 5,
+            "reasoning_tokens": 0,
+            "total_tokens": 5
+          },
+          "prompt_tokens_details": {
+            "audio_tokens": 10,
+            "cached_tokens": 0
+          }
+        },
+        "choices": [
+          {
+            "index": 0,
+            "finish_reason": "stop",
+            "message": {
+              "role": "system",
+              "content": null,
+              "tool_calls": null,
+              "audio": {
+                "id": "abcdef1234",
+                "mime_type": "audio/wav",
+                "data": "<base64 encoded audio data>",
+                "expires_at": 1896522361,
+                "transcript": "This is a sample transcript"
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

specification/ai/ModelInference/examples/2024-05-01-preview/GetChatCompletions_MaximumSet_Gen.json

@@ -56,7 +56,7 @@
       "body": {
         "id": "kgousajxgzyhugvqekuswuqbk",
         "object": "chat.completion",
-        "created": 18,
+        "created": 1896522361,
         "model": "my-model-name",
         "usage": {
           "completion_tokens": 19,
@@ -65,7 +65,7 @@
         },
         "choices": [
           {
-            "index": 7,
+            "index": 0,
             "finish_reason": "stop",
             "message": {
               "role": "system",

specification/ai/ModelInference/models/chat_completions.tsp

@@ -438,7 +438,7 @@ model ChatRequestAssistantMessage extends ChatRequestMessage {
   @doc("""
       The audio generated by a previous response in a multi-turn conversation.
     """)
-  audio?: ChatCompletionsAudio;
+  audio?: ChatRequestAudioReference;
 }
 
 @doc("A request chat message representing requested output from a configured tool.")
@@ -455,6 +455,14 @@ model ChatRequestToolMessage extends ChatRequestMessage {
   toolCallId: string;
 }
 
+@doc("A reference to an audio response generated by the model.")
+model ChatRequestAudioReference {
+  @doc("""
+      Unique identifier for the audio response. This value corresponds to the id of a previous audio completion.
+    """)
+  id: string;
+}
+
 @doc("A representation of a chat message update as received in a streaming response.")
 model StreamingChatResponseMessageUpdate {
   @doc("The chat role associated with the message. If present, should always be 'assistant'")