From 6543f33d544450b88ca35c1419272310b42d3ac4 Mon Sep 17 00:00:00 2001 From: Brian Hackett Date: Tue, 14 Jan 2025 13:07:23 -0800 Subject: [PATCH] Fixes for cloudflare deployment (#3) --- app/components/chat/BaseChat.tsx | 113 -------------- app/components/chat/Chat.client.tsx | 53 +------ app/lib/.server/llm/chat-anthropic.ts | 59 ++++++++ app/lib/.server/llm/stream-text.ts | 25 +++- app/lib/hooks/usePromptEnhancer.ts | 12 +- app/lib/replay/Recording.ts | 32 ++-- app/lib/replay/ReplayProtocolClient.ts | 10 ++ app/routes/api.chat.ts | 196 +++++++------------------ app/utils/chatStreamController.ts | 12 +- package.json | 1 + pnpm-lock.yaml | 133 +++++++++++++++++ 11 files changed, 307 insertions(+), 339 deletions(-) create mode 100644 app/lib/.server/llm/chat-anthropic.ts diff --git a/app/components/chat/BaseChat.tsx b/app/components/chat/BaseChat.tsx index e7b8e75d0..99e373415 100644 --- a/app/components/chat/BaseChat.tsx +++ b/app/components/chat/BaseChat.tsx @@ -9,7 +9,6 @@ import { Menu } from '~/components/sidebar/Menu.client'; import { IconButton } from '~/components/ui/IconButton'; import { Workbench } from '~/components/workbench/Workbench.client'; import { classNames } from '~/utils/classNames'; -import { MODEL_LIST, PROVIDER_LIST, initializeModelList } from '~/utils/constants'; import { Messages } from './Messages.client'; import { SendButton } from './SendButton.client'; import { APIKeyManager } from './APIKeyManager'; @@ -25,7 +24,6 @@ import GitCloneButton from './GitCloneButton'; import FilePreview from './FilePreview'; import { ModelSelector } from '~/components/chat/ModelSelector'; import { SpeechRecognitionButton } from '~/components/chat/SpeechRecognition'; -import type { IProviderSetting, ProviderInfo } from '~/types/model'; import { ScreenshotStateManager } from './ScreenshotStateManager'; import { toast } from 'react-toastify'; @@ -43,11 +41,6 @@ interface BaseChatProps { enhancingPrompt?: boolean; promptEnhanced?: boolean; input?: string; - model?: string; - setModel?: (model: string) => void; - provider?: ProviderInfo; - setProvider?: (provider: ProviderInfo) => void; - providerList?: ProviderInfo[]; handleStop?: () => void; sendMessage?: (event: React.UIEvent, messageInput?: string, simulation?: boolean) => void; handleInputChange?: (event: React.ChangeEvent) => void; @@ -69,11 +62,6 @@ export const BaseChat = React.forwardRef( showChat = true, chatStarted = false, isStreaming = false, - model, - setModel, - provider, - setProvider, - providerList, input = '', enhancingPrompt, handleInputChange, @@ -93,22 +81,6 @@ export const BaseChat = React.forwardRef( ref, ) => { const TEXTAREA_MAX_HEIGHT = chatStarted ? 400 : 200; - const [apiKeys, setApiKeys] = useState>(() => { - const savedKeys = Cookies.get('apiKeys'); - - if (savedKeys) { - try { - return JSON.parse(savedKeys); - } catch (error) { - console.error('Failed to parse API keys from cookies:', error); - return {}; - } - } - - return {}; - }); - const [modelList, setModelList] = useState(MODEL_LIST); - const [isModelSettingsCollapsed, setIsModelSettingsCollapsed] = useState(false); const [isListening, setIsListening] = useState(false); const [recognition, setRecognition] = useState(null); const [transcript, setTranscript] = useState(''); @@ -120,50 +92,6 @@ export const BaseChat = React.forwardRef( useEffect(() => { // Load API keys from cookies on component mount - let parsedApiKeys: Record | undefined = {}; - - try { - const storedApiKeys = Cookies.get('apiKeys'); - - if (storedApiKeys) { - const parsedKeys = JSON.parse(storedApiKeys); - - if (typeof parsedKeys === 'object' && parsedKeys !== null) { - setApiKeys(parsedKeys); - parsedApiKeys = parsedKeys; - } - } - } catch (error) { - console.error('Error loading API keys from cookies:', error); - - // Clear invalid cookie data - Cookies.remove('apiKeys'); - } - - let providerSettings: Record | undefined = undefined; - - try { - const savedProviderSettings = Cookies.get('providers'); - - if (savedProviderSettings) { - const parsedProviderSettings = JSON.parse(savedProviderSettings); - - if (typeof parsedProviderSettings === 'object' && parsedProviderSettings !== null) { - providerSettings = parsedProviderSettings; - } - } - } catch (error) { - console.error('Error loading Provider Settings from cookies:', error); - - // Clear invalid cookie data - Cookies.remove('providers'); - } - - initializeModelList({ apiKeys: parsedApiKeys, providerSettings }).then((modelList) => { - console.log('Model List: ', modelList); - setModelList(modelList); - }); - if (typeof window !== 'undefined' && ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window)) { const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition; const recognition = new SpeechRecognition(); @@ -351,31 +279,6 @@ export const BaseChat = React.forwardRef( -
-
- - {(providerList || []).length > 0 && provider && ( - { - const newApiKeys = { ...apiKeys, [provider.name]: key }; - setApiKeys(newApiKeys); - Cookies.set('apiKeys', JSON.stringify(newApiKeys)); - }} - /> - )} -
-
( show={input.length > 0 || isStreaming || uploadedFiles.length > 0} simulation={false} isStreaming={isStreaming} - disabled={!providerList || providerList.length === 0} onClick={(event) => { if (isStreaming) { handleStop?.(); @@ -492,7 +394,6 @@ export const BaseChat = React.forwardRef( show={(input.length > 0 || uploadedFiles.length > 0) && chatStarted} simulation={true} isStreaming={isStreaming} - disabled={!providerList || providerList.length === 0} onClick={(event) => { if (input.length > 0 || uploadedFiles.length > 0) { handleSendMessage?.(event, undefined, true); @@ -530,20 +431,6 @@ export const BaseChat = React.forwardRef( disabled={isStreaming} /> {chatStarted && {() => }} - setIsModelSettingsCollapsed(!isModelSettingsCollapsed)} - disabled={!providerList || providerList.length === 0} - > -
- {isModelSettingsCollapsed ? {model} : } -
{input.length > 3 ? (
diff --git a/app/components/chat/Chat.client.tsx b/app/components/chat/Chat.client.tsx index a0bf046dd..24c90c5eb 100644 --- a/app/components/chat/Chat.client.tsx +++ b/app/components/chat/Chat.client.tsx @@ -12,18 +12,16 @@ import { useMessageParser, usePromptEnhancer, useShortcuts, useSnapScroll } from import { description, useChatHistory } from '~/lib/persistence'; import { chatStore } from '~/lib/stores/chat'; import { workbenchStore } from '~/lib/stores/workbench'; -import { DEFAULT_MODEL, DEFAULT_PROVIDER, PROMPT_COOKIE_KEY, PROVIDER_LIST } from '~/utils/constants'; +import { PROMPT_COOKIE_KEY } from '~/utils/constants'; import { cubicEasingFn } from '~/utils/easings'; import { createScopedLogger, renderLogger } from '~/utils/logger'; import { BaseChat } from './BaseChat'; import Cookies from 'js-cookie'; import { debounce } from '~/utils/debounce'; import { useSettings } from '~/lib/hooks/useSettings'; -import type { ProviderInfo } from '~/types/model'; import { useSearchParams } from '@remix-run/react'; import { createSampler } from '~/utils/sampler'; import { saveProjectPrompt } from './Messages.client'; -import { uint8ArrayToBase64 } from '~/lib/replay/ReplayProtocolClient'; import type { SimulationPromptClientData } from '~/lib/replay/SimulationPrompt'; import { getIFrameSimulationData } from '~/lib/replay/Recording'; import { getCurrentIFrame } from '../workbench/Preview'; @@ -120,27 +118,15 @@ export const ChatImpl = memo( const [imageDataList, setImageDataList] = useState([]); // Move here const [searchParams, setSearchParams] = useSearchParams(); const files = useStore(workbenchStore.files); - const { activeProviders, promptId } = useSettings(); - - const [model, setModel] = useState(() => { - const savedModel = Cookies.get('selectedModel'); - return savedModel || DEFAULT_MODEL; - }); - const [provider, setProvider] = useState(() => { - const savedProvider = Cookies.get('selectedProvider'); - return (PROVIDER_LIST.find((p) => p.name === savedProvider) || DEFAULT_PROVIDER) as ProviderInfo; - }); + const { promptId } = useSettings(); const { showChat } = useStore(chatStore); const [animationScope, animate] = useAnimate(); - const [apiKeys, setApiKeys] = useState>({}); - const { messages, isLoading, input, handleInputChange, setInput, stop, append } = useChat({ api: '/api/chat', body: { - apiKeys, files, promptId, }, @@ -167,7 +153,6 @@ export const ChatImpl = memo( }); useEffect(() => { const prompt = searchParams.get('prompt'); - console.log(prompt, searchParams, model, provider); if (prompt) { setSearchParams({}); @@ -177,12 +162,12 @@ export const ChatImpl = memo( content: [ { type: 'text', - text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${prompt}`, + text: prompt, }, ] as any, // Type assertion to bypass compiler check }); } - }, [model, provider, searchParams]); + }, [searchParams]); const { enhancingPrompt, promptEnhanced, enhancePrompt, resetEnhancer } = usePromptEnhancer(); const { parsedMessages, parseMessages } = useMessageParser(); @@ -294,7 +279,7 @@ export const ChatImpl = memo( content: [ { type: 'text', - text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`, + text: _input, }, ...imageDataList.map((imageData) => ({ type: 'image', @@ -314,7 +299,7 @@ export const ChatImpl = memo( content: [ { type: 'text', - text: `[Model: ${model}]\n\n[Provider: ${provider.name}]\n\n${_input}`, + text: _input, }, ...imageDataList.map((imageData) => ({ type: 'image', @@ -363,24 +348,6 @@ export const ChatImpl = memo( const [messageRef, scrollRef] = useSnapScroll(); - useEffect(() => { - const storedApiKeys = Cookies.get('apiKeys'); - - if (storedApiKeys) { - setApiKeys(JSON.parse(storedApiKeys)); - } - }, []); - - const handleModelChange = (newModel: string) => { - setModel(newModel); - Cookies.set('selectedModel', newModel, { expires: 30 }); - }; - - const handleProviderChange = (newProvider: ProviderInfo) => { - setProvider(newProvider); - Cookies.set('selectedProvider', newProvider.name, { expires: 30 }); - }; - return ( { @@ -424,9 +386,6 @@ export const ChatImpl = memo( setInput(input); scrollTextArea(); }, - model, - provider, - apiKeys, ); }} uploadedFiles={uploadedFiles} diff --git a/app/lib/.server/llm/chat-anthropic.ts b/app/lib/.server/llm/chat-anthropic.ts new file mode 100644 index 000000000..c0a152411 --- /dev/null +++ b/app/lib/.server/llm/chat-anthropic.ts @@ -0,0 +1,59 @@ +import type { CoreMessage } from "ai"; +import Anthropic from "@anthropic-ai/sdk"; +import { ChatStreamController } from "~/utils/chatStreamController"; +import type { ContentBlockParam, MessageParam } from "@anthropic-ai/sdk/resources/messages/messages.mjs"; + +const MaxMessageTokens = 8192; + +function convertContentToAnthropic(content: any): ContentBlockParam[] { + if (typeof content === "string") { + return [{ type: "text", text: content }]; + } + if (Array.isArray(content)) { + return content.flatMap(convertContentToAnthropic); + } + if (content.type === "text" && typeof content.text === "string") { + return [{ type: "text", text: content.text }]; + } + console.log("AnthropicUnknownContent", JSON.stringify(content, null, 2)); + return []; +} + +export async function chatAnthropic(chatController: ChatStreamController, apiKey: string, systemPrompt: string, messages: CoreMessage[]) { + const anthropic = new Anthropic({ apiKey }); + + const messageParams: MessageParam[] = []; + + messageParams.push({ + role: "assistant", + content: systemPrompt, + }); + + for (const message of messages) { + const role = message.role == "user" ? "user" : "assistant"; + const content = convertContentToAnthropic(message.content); + messageParams.push({ + role, + content, + }); + } + + const response = await anthropic.messages.create({ + model: "claude-3-5-sonnet-20241022", + messages: messageParams, + max_tokens: MaxMessageTokens, + }); + + for (const content of response.content) { + if (content.type === "text") { + chatController.writeText(content.text); + } else { + console.log("AnthropicUnknownResponse", JSON.stringify(content, null, 2)); + } + } + + const tokens = response.usage.input_tokens + response.usage.output_tokens; + console.log("AnthropicTokens", tokens); + + chatController.writeUsage({ completionTokens: response.usage.output_tokens, promptTokens: response.usage.input_tokens }); +} diff --git a/app/lib/.server/llm/stream-text.ts b/app/lib/.server/llm/stream-text.ts index 47a2992ab..316cacbcb 100644 --- a/app/lib/.server/llm/stream-text.ts +++ b/app/lib/.server/llm/stream-text.ts @@ -142,16 +142,15 @@ function extractPropertiesFromMessage(message: Message): { model: string; provid return { model, provider, content: cleanedContent }; } -export async function streamText(props: { +export async function getStreamTextArguments(props: { messages: Messages; env: Env; - options?: StreamingOptions; apiKeys?: Record; files?: FileMap; providerSettings?: Record; promptId?: string; }) { - const { messages, env: serverEnv, options, apiKeys, files, providerSettings, promptId } = props; + const { messages, env: serverEnv, apiKeys, files, providerSettings, promptId } = props; // console.log({serverEnv}); @@ -202,9 +201,7 @@ export async function streamText(props: { const coreMessages = convertToCoreMessages(processedMessages as any); - console.log("QueryModel", JSON.stringify({ systemPrompt, coreMessages })); - - return _streamText({ + return { model: provider.getModelInstance({ model: currentModel, serverEnv, @@ -214,6 +211,18 @@ export async function streamText(props: { system: systemPrompt, maxTokens: dynamicMaxTokens, messages: coreMessages, - ...options, - }); + }; +} + +export async function streamText(props: { + messages: Messages; + env: Env; + options?: StreamingOptions; + apiKeys?: Record; + files?: FileMap; + providerSettings?: Record; + promptId?: string; +}) { + const args = await getStreamTextArguments(props); + return _streamText({ ...args, ...props.options }); } diff --git a/app/lib/hooks/usePromptEnhancer.ts b/app/lib/hooks/usePromptEnhancer.ts index 6275ef37e..5be878b96 100644 --- a/app/lib/hooks/usePromptEnhancer.ts +++ b/app/lib/hooks/usePromptEnhancer.ts @@ -1,5 +1,4 @@ import { useState } from 'react'; -import type { ProviderInfo } from '~/types/model'; import { createScopedLogger } from '~/utils/logger'; const logger = createScopedLogger('usePromptEnhancement'); @@ -15,24 +14,15 @@ export function usePromptEnhancer() { const enhancePrompt = async ( input: string, - setInput: (value: string) => void, - model: string, - provider: ProviderInfo, - apiKeys?: Record, + setInput: (value: string) => void ) => { setEnhancingPrompt(true); setPromptEnhanced(false); const requestBody: any = { message: input, - model, - provider, }; - if (apiKeys) { - requestBody.apiKeys = apiKeys; - } - const response = await fetch('/api/enhancer', { method: 'POST', body: JSON.stringify(requestBody), diff --git a/app/lib/replay/Recording.ts b/app/lib/replay/Recording.ts index 9f5c7930b..7846a233e 100644 --- a/app/lib/replay/Recording.ts +++ b/app/lib/replay/Recording.ts @@ -4,10 +4,11 @@ import { assert, stringToBase64, uint8ArrayToBase64 } from "./ReplayProtocolClie export interface SimulationResource { url: string; - requestBodyBase64: string; - responseBodyBase64: string; - responseStatus: number; - responseHeaders: Record; + requestBodyBase64?: string; + responseBodyBase64?: string; + responseStatus?: number; + responseHeaders?: Record; + error?: string; } enum SimulationInteractionKind { @@ -122,7 +123,7 @@ export async function getMouseData(iframe: HTMLIFrameElement, position: { x: num // Add handlers to the current iframe's window. function addRecordingMessageHandler(messageHandlerId: string) { - const resources: Map = new Map(); + const resources: SimulationResource[] = []; const interactions: SimulationInteraction[] = []; const indexedDBAccesses: IndexedDBAccess[] = []; const localStorageAccesses: LocalStorageAccess[] = []; @@ -131,10 +132,7 @@ function addRecordingMessageHandler(messageHandlerId: string) { function addTextResource(path: string, text: string) { const url = (new URL(path, window.location.href)).href; - if (resources.has(url)) { - return; - } - resources.set(url, { + resources.push({ url, requestBodyBase64: "", responseBodyBase64: stringToBase64(text), @@ -147,7 +145,7 @@ function addRecordingMessageHandler(messageHandlerId: string) { return { locationHref: window.location.href, documentUrl: window.location.href, - resources: Array.from(resources.values()), + resources, interactions, indexedDBAccesses, localStorageAccesses, @@ -475,10 +473,18 @@ function addRecordingMessageHandler(messageHandlerId: string) { const baseFetch = window.fetch; window.fetch = async (info, options) => { - const rv = await baseFetch(info, options); const url = info instanceof Request ? info.url : info.toString(); - responseToURL.set(rv, url); - return createProxy(rv); + try { + const rv = await baseFetch(info, options); + responseToURL.set(rv, url); + return createProxy(rv); + } catch (error) { + resources.push({ + url, + error: String(error), + }); + throw error; + } }; } diff --git a/app/lib/replay/ReplayProtocolClient.ts b/app/lib/replay/ReplayProtocolClient.ts index 7c3284a26..df379f8f0 100644 --- a/app/lib/replay/ReplayProtocolClient.ts +++ b/app/lib/replay/ReplayProtocolClient.ts @@ -6,6 +6,16 @@ export function assert(condition: any, message: string = "Assertion failed!"): a } } +export function defer(): { promise: Promise; resolve: (value: T) => void; reject: (reason?: any) => void } { + let resolve: (value: T) => void; + let reject: (reason?: any) => void; + const promise = new Promise((_resolve, _reject) => { + resolve = _resolve; + reject = _reject; + }); + return { promise, resolve: resolve!, reject: reject! }; +} + export function uint8ArrayToBase64(data: Uint8Array) { let str = ""; for (const byte of data) { diff --git a/app/routes/api.chat.ts b/app/routes/api.chat.ts index 29dabd48b..0bface90f 100644 --- a/app/routes/api.chat.ts +++ b/app/routes/api.chat.ts @@ -1,36 +1,14 @@ import { type ActionFunctionArgs } from '@remix-run/cloudflare'; -import { createDataStream } from 'ai'; -import { MAX_RESPONSE_SEGMENTS, MAX_TOKENS } from '~/lib/.server/llm/constants'; -import { CONTINUE_PROMPT } from '~/lib/common/prompts/prompts'; -import { streamText, type Messages, type StreamingOptions } from '~/lib/.server/llm/stream-text'; -import SwitchableStream from '~/lib/.server/llm/switchable-stream'; -import type { IProviderSetting } from '~/types/model'; import { type SimulationChatMessage, type SimulationPromptClientData, performSimulationPrompt } from '~/lib/replay/SimulationPrompt'; import { ChatStreamController } from '~/utils/chatStreamController'; import { assert } from '~/lib/replay/ReplayProtocolClient'; +import { getStreamTextArguments, type Messages } from '~/lib/.server/llm/stream-text'; +import { chatAnthropic } from '~/lib/.server/llm/chat-anthropic'; export async function action(args: ActionFunctionArgs) { return chatAction(args); } -function parseCookies(cookieHeader: string): Record { - const cookies: Record = {}; - - const items = cookieHeader.split(';').map((cookie) => cookie.trim()); - - items.forEach((item) => { - const [name, ...rest] = item.split('='); - - if (name && rest) { - const decodedName = decodeURIComponent(name.trim()); - const decodedValue = decodeURIComponent(rest.join('=').trim()); - cookies[decodedName] = decodedValue; - } - }); - - return cookies; -} - function extractMessageContent(baseContent: any): string { let content = baseContent; @@ -68,138 +46,66 @@ async function chatAction({ context, request }: ActionFunctionArgs) { simulationClientData?: SimulationPromptClientData; }>(); - console.log("SimulationClientData", simulationClientData); - - const cookieHeader = request.headers.get('Cookie'); - const apiKeys = JSON.parse(parseCookies(cookieHeader || '').apiKeys || '{}'); - const providerSettings: Record = JSON.parse( - parseCookies(cookieHeader || '').providers || '{}', - ); + let finished: (v?: any) => void; + context.cloudflare.ctx.waitUntil(new Promise((resolve) => finished = resolve)); - const stream = new SwitchableStream(); - - const cumulativeUsage = { - completionTokens: 0, - promptTokens: 0, - totalTokens: 0, - }; + console.log("SimulationClientData", simulationClientData); try { - if (simulationClientData) { - const chatHistory: SimulationChatMessage[] = []; - for (const { role, content } of messages) { - chatHistory.push({ role, content: extractMessageContent(content) }); - } - const lastHistoryMessage = chatHistory.pop(); - assert(lastHistoryMessage?.role == "user", "Last message in chat history must be a user message"); - const userPrompt = lastHistoryMessage.content; - - const anthropicApiKey = process.env.ANTHROPIC_API_KEY; - if (!anthropicApiKey) { - throw new Error("Anthropic API key is not set"); - } - - const { message, fileChanges } = await performSimulationPrompt(simulationClientData, userPrompt, chatHistory, anthropicApiKey); - - const resultStream = new ReadableStream({ - async start(controller) { - const chatController = new ChatStreamController(controller); - - chatController.writeText(message + "\n"); - chatController.writeFileChanges("Update Files", fileChanges); - - /* - chatController.writeText("Hello World\n"); - chatController.writeText("Hello World 2\n"); - chatController.writeText("Hello\n World 3\n"); - chatController.writeFileChanges("Rewrite Files", [{filePath: "src/services/llm.ts", contents: "FILE_CONTENTS_FIXME" }]); - chatController.writeAnnotation("usage", { completionTokens: 10, promptTokens: 20, totalTokens: 30 }); - */ - - controller.close(); - setTimeout(() => stream.close(), 1000); - }, - }); - - stream.switchSource(resultStream); + const { system, messages: coreMessages } = await getStreamTextArguments({ + messages, + env: context.cloudflare.env, + apiKeys: {}, + files, + providerSettings: undefined, + promptId, + }); - return new Response(stream.readable, { - status: 200, - headers: { - contentType: 'text/plain; charset=utf-8', - }, - }); + const anthropicApiKey = context.cloudflare.env.ANTHROPIC_API_KEY; + if (!anthropicApiKey) { + throw new Error("Anthropic API key is not set"); } - const options: StreamingOptions = { - toolChoice: 'none', - onFinish: async ({ text: content, finishReason, usage }) => { - console.log("QueryModelFinished", usage, content); - - if (usage) { - cumulativeUsage.completionTokens += usage.completionTokens || 0; - cumulativeUsage.promptTokens += usage.promptTokens || 0; - cumulativeUsage.totalTokens += usage.totalTokens || 0; - } - - if (finishReason !== 'length') { - return stream - .switchSource( - createDataStream({ - async execute(dataStream) { - dataStream.writeMessageAnnotation({ - type: 'usage', - value: { - completionTokens: cumulativeUsage.completionTokens, - promptTokens: cumulativeUsage.promptTokens, - totalTokens: cumulativeUsage.totalTokens, - }, - }); - }, - onError: (error: any) => `Custom error: ${error.message}`, - }), - ) - .then(() => stream.close()); + const resultStream = new ReadableStream({ + async start(controller) { + const chatController = new ChatStreamController(controller); + + /* + chatController.writeText("Hello World\n"); + chatController.writeText("Hello World 2\n"); + chatController.writeText("Hello\n World 3\n"); + chatController.writeFileChanges("Rewrite Files", [{filePath: "src/services/llm.ts", contents: "FILE_CONTENTS_FIXME" }]); + chatController.writeAnnotation("usage", { completionTokens: 10, promptTokens: 20, totalTokens: 30 }); + */ + + try { + if (simulationClientData) { + const chatHistory: SimulationChatMessage[] = []; + for (const { role, content } of messages) { + chatHistory.push({ role, content: extractMessageContent(content) }); + } + const lastHistoryMessage = chatHistory.pop(); + assert(lastHistoryMessage?.role == "user", "Last message in chat history must be a user message"); + const userPrompt = lastHistoryMessage.content; + + const { message, fileChanges } = await performSimulationPrompt(simulationClientData, userPrompt, chatHistory, anthropicApiKey); + + chatController.writeText(message + "\n"); + chatController.writeFileChanges("Update Files", fileChanges); + } else { + await chatAnthropic(chatController, anthropicApiKey, system, coreMessages); + } + } catch (error: any) { + console.error(error); + chatController.writeText("Error: " + error.message); } - if (stream.switches >= MAX_RESPONSE_SEGMENTS) { - throw Error('Cannot continue message: Maximum segments reached'); - } - - const switchesLeft = MAX_RESPONSE_SEGMENTS - stream.switches; - - console.log(`Reached max token limit (${MAX_TOKENS}): Continuing message (${switchesLeft} switches left)`); - - messages.push({ role: 'assistant', content }); - messages.push({ role: 'user', content: CONTINUE_PROMPT }); - - const result = await streamText({ - messages, - env: context.cloudflare.env, - options, - apiKeys, - files, - providerSettings, - promptId, - }); - - return stream.switchSource(result.toDataStream()); + controller.close(); + setTimeout(finished, 1000); }, - }; - - const result = await streamText({ - messages, - env: context.cloudflare.env, - options, - apiKeys, - files, - providerSettings, - promptId, }); - stream.switchSource(result.toDataStream()); - - return new Response(stream.readable, { + return new Response(resultStream, { status: 200, headers: { contentType: 'text/plain; charset=utf-8', diff --git a/app/utils/chatStreamController.ts b/app/utils/chatStreamController.ts index 0c8289c32..59d7bb55d 100644 --- a/app/utils/chatStreamController.ts +++ b/app/utils/chatStreamController.ts @@ -10,13 +10,16 @@ export interface ChatFileChange { export class ChatStreamController { private controller: ReadableStreamDefaultController; + private encoder: TextEncoder; constructor(controller: ReadableStreamDefaultController) { this.controller = controller; + this.encoder = new TextEncoder(); } writeText(text: string) { - this.controller.enqueue(`0:${JSON.stringify(text)}\n`); + const data = this.encoder.encode(`0:${JSON.stringify(text)}\n`); + this.controller.enqueue(data); } writeFileChanges(title: string, fileChanges: ChatFileChange[]) { @@ -29,6 +32,11 @@ export class ChatStreamController { } writeAnnotation(type: string, value: any) { - this.controller.enqueue(`8:[{"type":"${type}","value":${JSON.stringify(value)}}]\n`); + const data = this.encoder.encode(`8:[{"type":"${type}","value":${JSON.stringify(value)}}]\n`); + this.controller.enqueue(data); + } + + writeUsage({ completionTokens, promptTokens }: { completionTokens: number, promptTokens: number }) { + this.writeAnnotation("usage", { completionTokens, promptTokens, totalTokens: completionTokens + promptTokens }); } } diff --git a/package.json b/package.json index 79fc803be..87bde603f 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@ai-sdk/google": "^0.0.52", "@ai-sdk/mistral": "^0.0.43", "@ai-sdk/openai": "^0.0.66", + "@anthropic-ai/sdk": "^0.33.1", "@codemirror/autocomplete": "^6.18.3", "@codemirror/commands": "^6.7.1", "@codemirror/lang-cpp": "^6.0.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 961254e97..771e4a21b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@ai-sdk/openai': specifier: ^0.0.66 version: 0.0.66(zod@3.23.8) + '@anthropic-ai/sdk': + specifier: ^0.33.1 + version: 0.33.1 '@codemirror/autocomplete': specifier: ^6.18.3 version: 6.18.3(@codemirror/language@6.10.6)(@codemirror/state@6.4.1)(@codemirror/view@6.35.0)(@lezer/common@1.2.3) @@ -435,6 +438,9 @@ packages: '@antfu/utils@0.7.10': resolution: {integrity: sha512-+562v9k4aI80m1+VuMHehNJWLOFjBnXn3tdOitzD0il5b7smkSBal4+a3oKiQTbrwMmN/TBUMDvbdoWDehgOww==} + '@anthropic-ai/sdk@0.33.1': + resolution: {integrity: sha512-VrlbxiAdVRGuKP2UQlCnsShDHJKWepzvfRCkZMpU+oaUdKLpOfmylLMRojGrAgebV+kDtPjewCVP0laHXg+vsA==} + '@babel/code-frame@7.26.2': resolution: {integrity: sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==} engines: {node: '>=6.9.0'} @@ -2126,9 +2132,15 @@ packages: '@types/ms@0.7.34': resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/node-fetch@2.6.12': + resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} + '@types/node-forge@1.3.11': resolution: {integrity: sha512-FQx220y22OKNTqaByeBGqHWYz4cl94tpcxeFdvBo3wjG6XPBuZ0BNgNZRV5J5TFmmcsJ4IzsLkmGRiQbnYsBEQ==} + '@types/node@18.19.70': + resolution: {integrity: sha512-RE+K0+KZoEpDUbGGctnGdkrLFwi1eYKTlIHNl2Um98mUkGsm1u2Ff6Ltd0e8DktTtC98uy7rSj+hO8t/QuLoVQ==} + '@types/node@22.10.1': resolution: {integrity: sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==} @@ -2396,6 +2408,10 @@ packages: engines: {node: '>=0.4.0'} hasBin: true + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + aggregate-error@3.1.0: resolution: {integrity: sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==} engines: {node: '>=8'} @@ -2468,6 +2484,9 @@ packages: async-lock@1.4.1: resolution: {integrity: sha512-Az2ZTpuytrtqENulXwO3GGv1Bztugx6TT37NIo7imr/Qo0gsYiGtSdBa2B6fsXhTpVZDNfu1Qn3pk531e3q+nQ==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + available-typed-arrays@1.0.7: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} @@ -2692,6 +2711,10 @@ packages: colorjs.io@0.5.2: resolution: {integrity: sha512-twmVoizEW7ylZSN32OgKdXRmo1qg+wT5/6C3xu5b9QsWzSFAhHLn2xd8ro0diCsKfCj1RdaTP/nrcW+vAoQPIw==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -2869,6 +2892,10 @@ packages: defu@6.1.4: resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} @@ -3246,10 +3273,21 @@ packages: resolution: {integrity: sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.1: + resolution: {integrity: sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==} + engines: {node: '>= 6'} + format@0.2.2: resolution: {integrity: sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==} engines: {node: '>=0.4.x'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -3463,6 +3501,9 @@ packages: resolution: {integrity: sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==} engines: {node: '>=10.17.0'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + husky@9.1.7: resolution: {integrity: sha512-5gs5ytaNjBrh5Ow3zrvdUUY+0VxIuWVL4i9irt6friV+BqdCfmV11CQTWMiBYWHbXhco+J1kHfTOUkePhCDvMA==} engines: {node: '>=18'} @@ -4232,6 +4273,15 @@ packages: node-fetch-native@1.6.4: resolution: {integrity: sha512-IhOigYzAKHd244OC0JIMIUrjzctirCmPkaIfhDeGcEETWof5zKYUW7e7MYvChGWh/4CJeXEgsRyGzuF334rOOQ==} + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + node-fetch@3.3.2: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -5304,6 +5354,9 @@ packages: resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==} engines: {node: '>=6'} + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -5377,6 +5430,9 @@ packages: unconfig@0.5.5: resolution: {integrity: sha512-VQZ5PT9HDX+qag0XdgQi8tJepPhXiR/yVOkn707gJDKo31lGjRilPREiQJ9Z6zd/Ugpv6ZvO5VxVIcatldYcNQ==} + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} @@ -5667,6 +5723,16 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + which-typed-array@1.1.16: resolution: {integrity: sha512-g+N+GAWiRj66DngFwHvISJd+ITsyphZvD1vChfVg6cEdnzy53GzB3oy0fUNlvhz7H7+MiqhYr26qxQShCpKTTQ==} engines: {node: '>= 0.4'} @@ -5904,6 +5970,18 @@ snapshots: '@antfu/utils@0.7.10': {} + '@anthropic-ai/sdk@0.33.1': + dependencies: + '@types/node': 18.19.70 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + '@babel/code-frame@7.26.2': dependencies: '@babel/helper-validator-identifier': 7.25.9 @@ -7562,10 +7640,19 @@ snapshots: '@types/ms@0.7.34': {} + '@types/node-fetch@2.6.12': + dependencies: + '@types/node': 22.10.1 + form-data: 4.0.1 + '@types/node-forge@1.3.11': dependencies: '@types/node': 22.10.1 + '@types/node@18.19.70': + dependencies: + undici-types: 5.26.5 + '@types/node@22.10.1': dependencies: undici-types: 6.20.0 @@ -7975,6 +8062,10 @@ snapshots: acorn@8.14.0: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + aggregate-error@3.1.0: dependencies: clean-stack: 2.2.0 @@ -8049,6 +8140,8 @@ snapshots: async-lock@1.4.1: {} + asynckit@0.4.0: {} + available-typed-arrays@1.0.7: dependencies: possible-typed-array-names: 1.0.0 @@ -8307,6 +8400,10 @@ snapshots: colorjs.io@0.5.2: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + comma-separated-tokens@2.0.3: {} common-tags@1.8.2: {} @@ -8455,6 +8552,8 @@ snapshots: defu@6.1.4: {} + delayed-stream@1.0.0: {} + depd@2.0.0: {} dequal@2.0.3: {} @@ -8963,8 +9062,21 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + + form-data@4.0.1: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + format@0.2.2: {} + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -9249,6 +9361,10 @@ snapshots: human-signals@2.1.0: {} + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + husky@9.1.7: {} iconv-lite@0.4.24: @@ -10354,6 +10470,10 @@ snapshots: node-fetch-native@1.6.4: {} + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + node-fetch@3.3.2: dependencies: data-uri-to-buffer: 4.0.1 @@ -11497,6 +11617,8 @@ snapshots: totalist@3.0.1: {} + tr46@0.0.3: {} + trim-lines@3.0.1: {} trough@2.2.0: {} @@ -11562,6 +11684,8 @@ snapshots: transitivePeerDependencies: - supports-color + undici-types@5.26.5: {} + undici-types@6.20.0: {} undici@5.28.4: @@ -11920,6 +12044,15 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + which-typed-array@1.1.16: dependencies: available-typed-arrays: 1.0.7