feat: Stream response

package.json

@@ -19,6 +19,7 @@
     "wrangler": "^3.97.0"
   },
   "dependencies": {
+    "discord-api-types": "^0.37.112",
     "discord-interactions": "^4.1.0",
     "dotenv": "^16.4.7",
     "itty-router": "^5.0.18",

src/server.ts

@@ -8,6 +8,7 @@ import {
   InteractionType,
   verifyKey
 } from 'discord-interactions';
+import { APIInteraction } from 'discord-api-types/v10';
 import Together from 'together-ai';
 import { CHAT_COMMAND, PING_COMMAND, TEMPLATE_COMMAND } from './commands';
 
@@ -74,13 +75,33 @@ router.post('/', async (request, env) => {
         );
         const userPrompt = promptOption?.value;
 
-        const aiResponse = await sendToAIProvider(env.TOGETHER_API_KEY, userPrompt || "No input");
-        return new JsonResponse({
-          type: InteractionResponseType.CHANNEL_MESSAGE_WITH_SOURCE,
-          data: {
-            content: aiResponse
-          },
-        });
+        // Send an initial response to answer the command
+        const initialResponse = await fetch(
+          `https://discord.com/api/v10/interactions/${interaction.id}/${interaction.token}/callback`,
+          {
+            method: "POST",
+            headers: {
+              "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+              type: InteractionResponseType.DEFERRED_CHANNEL_MESSAGE_WITH_SOURCE,
+              content: "*Waiting...*"
+            }),
+          }
+        );
+
+        if (!initialResponse.ok) {
+          console.error("Failed to send initial response:", initialResponse.statusText);
+          return new JsonResponse(
+            { error: "Failed to send initial response" },
+            { status: 500 }
+          );
+        }
+
+        // Stream the AI response and update the message
+        await streamFromAI(interaction, userPrompt, env.TOGETHER_API_KEY);
+
+        return null;
       }
       default:
         return new JsonResponse({ error: 'Unknown Type' }, { status: 400 });
@@ -112,16 +133,75 @@ const server = {
   fetch: router.fetch
 };
 
-async function sendToAIProvider(api_key: string, prompt: string): Promise<string> {
-  const together = new Together({ apiKey: api_key });
-  const response = await together.chat.completions.create({
-    model: "meta-llama/Llama-Vision-Free",
-    messages: [{ role: "user", content: prompt }],
-  });
+async function streamFromAI(interaction: APIInteraction, api_key: string, prompt: string): Promise<void> {
+  // const together = new Together({ apiKey: api_key });
+  // const response = await together.chat.completions.create({
+  //   model: "meta-llama/Llama-Vision-Free",
+  //   messages: [{ role: "user", content: prompt }],
+  // });
+  //
+  // if (response.choices && response.choices[0] && response.choices[0].message) {
+  //   return response.choices[0].message.content || "No response";
+  // }
+  // return "No response";
+
+  try {
+    const together = new Together({ apiKey: api_key });
 
-  if (response.choices && response.choices[0] && response.choices[0].message) {
-    return response.choices[0].message.content || "No response";
+    const stream = await together.chat.completions.create({
+      model: "meta-llama/Llama-Vision-Free",
+      messages: [{ role: "user", content: prompt }],
+      stream: true, // Enable streaming
+    });
+
+    let aiResponse = "";
+
+    for await (const chunk of stream) {
+      aiResponse += chunk.choices[0].delta.content || "";
+
+      // Update the message with the current AI response
+      await fetch(
+        `https://discord.com/api/v10/webhooks/${interaction.application_id}/${interaction.token}/messages/@original`,
+        {
+          method: "PATCH",
+          headers: {
+            "Content-Type": "application/json",
+          },
+          body: JSON.stringify({
+            content: aiResponse || '' + '\n*Generating...*\n-# This message was generated by AI. Results may be wrong.',
+          }),
+        }
+      );
+    }
+    await fetch(
+      `https://discord.com/api/v10/webhooks/${interaction.application_id}/${interaction.token}/messages/@original`,
+      {
+        method: "PATCH",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          content: aiResponse || '' + '\n-# This message was generated by AI. Results may be wrong.',
+        }),
+      }
+    );
+  } catch (error) {
+    console.error("Error fetching AI response: ", error);
+
+    // Handle errors by updating the message
+    await fetch(
+      `https://discord.com/api/v10/webhooks/${interaction.application_id}/${interaction.token}/messages/@original`,
+      {
+        method: "PATCH",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify({
+          content: "An error occurred.\n" + error,
+        }),
+      }
+    );
   }
-  return "No response";
+  return;
 }
 export default server;