diff --git a/CHANGELOG.md b/CHANGELOG.md index a1099fac0..2a8057feb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change +## 3.4.0 + +- feat(ai-anthropic): 添加新组件 `@celljs/ai-anthropic`，支持 Anthropic 模型及其 API + ## 3.3.0 - feat(ai-ollama): 添加新组件 `@celljs/ai-ollama`，支持 Ollama 通用能力 diff --git a/ai-packages/ai-anthropic/.eslintrc.js b/ai-packages/ai-anthropic/.eslintrc.js new file mode 100644 index 000000000..5c01e5e50 --- /dev/null +++ b/ai-packages/ai-anthropic/.eslintrc.js @@ -0,0 +1,10 @@ +/** @type {import('eslint').Linter.Config} */ +module.exports = { + extends: [ + require.resolve('@celljs/component/configs/build.eslintrc.json') + ], + parserOptions: { + tsconfigRootDir: __dirname, + project: 'tsconfig.json' + } +}; \ No newline at end of file diff --git a/ai-packages/ai-anthropic/README.md b/ai-packages/ai-anthropic/README.md new file mode 100644 index 000000000..a8d31429b --- /dev/null +++ b/ai-packages/ai-anthropic/README.md @@ -0,0 +1,82 @@ +# Cell - AI Anthropic Component + +## 概览 + +AI Anthropic 模块是一个用于与 Anthropic API 交互的库，提供了生成聊天响应和嵌入向量的功能。通过简单易用的 API 接口，支持消息的创建、请求的发送和响应的处理。是 @celljs/ai-core 模块中所有模型服务接口抽象的一种实现。 + +## 特性 + +- 生成聊天响应 +- 生成嵌入向量 +- 支持流式响应 +- 支持多种模型参数配置 + +## 安装 + +使用 npm 安装 AI Anthropic 模块： + +```bash +npm install @celljs/ai-anthropic +``` + +或者使用 yarn： + +```bash +yarn add @celljs/ai-anthropic +``` + +## 快速开始 + +以下是一个简单的示例，展示如何使用 AI Anthropic 模块生成聊天响应： + +```typescript +import { AssistantMessage, PromptTemplate } from '@celljs/ai-core'; +import { OllamChatModel, OllamModel, } from '@celljs/ai-anthropic'; +import { Component Autowired } from '@celljs/core'; + +@Component() +export class AnthropicDemo { + @Autowired(AnthropicChatModel) + private anthropicChatModel: AnthropicChatModel; + + @Autowired(PromptTemplate) + private promptTemplate: PromptTemplate; + + /** + * Chat with Anthropic + */ + async chat() { + const prompt = await this.promptTemplate.create( + 'Hello {name}', + { + chatOptions: { model: AnthropicModel.CLAUDE_3_5_SONNET }, + variables: { name: 'Anthropic' } + } + ); + const response = await this.anthropicChatModel.call(prompt); + console.log(response.result.output); + } + + /** + * Stream chat response + */ + async stream() { + const prompt = await this.promptTemplate.create( + 'Hello {name}', + { + chatOptions: { model: AnthropicModel.CLAUDE_3_5_SONNET }, + variables: { name: 'Anthropic' } + } + ); + const response$ = await this.anthropicChatModel.stream(prompt); + response$.subscribe({ + next: response => console.log(response.result.output), + complete: () => console.log('Chat completed!') + }); + } +} +``` + +## 许可证 + +本项目采用 MIT 许可证。 diff --git a/ai-packages/ai-anthropic/package.json b/ai-packages/ai-anthropic/package.json new file mode 100644 index 000000000..88adf1e23 --- /dev/null +++ b/ai-packages/ai-anthropic/package.json @@ -0,0 +1,46 @@ +{ + "name": "@celljs/ai-anthropic", + "version": "3.3.0", + "description": "Authropic models support", + "main": "lib/common/index.js", + "typings": "lib/common/index.d.ts", + "dependencies": { + "@celljs/ai-core": "3.3.0", + "@celljs/core": "3.3.0", + "@celljs/http": "3.3.0", + "class-transformer": "^0.5.1", + "rxjs": "^6.6.0", + "tslib": "^2.8.0" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "cell-component" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/cellbang/cell.git" + }, + "bugs": { + "url": "https://github.com/cellbang/cell/issues" + }, + "homepage": "https://github.com/cellbang/cell", + "files": [ + "lib", + "src", + "cell.yml" + ], + "scripts": { + "lint": "cell-component lint", + "build": "cell-component build", + "watch": "cell-component watch", + "clean": "cell-component clean", + "test": "cell-component test:js" + }, + "devDependencies": { + "@celljs/component": "3.3.0" + }, + "gitHead": "bbf636b21ea1a347affcc05a5f6f58b35bedef6d" +} diff --git a/ai-packages/ai-anthropic/src/common/api/anthropic-api.spec.ts b/ai-packages/ai-anthropic/src/common/api/anthropic-api.spec.ts new file mode 100644 index 000000000..ef39dfac6 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/anthropic-api.spec.ts @@ -0,0 +1,165 @@ +import { expect } from 'chai'; +import { ChatRequest } from './chat-request'; +import { RestOperations } from '@celljs/http'; +import { createContainer } from '../test/test-container'; +import { AnthropicAPI } from './api-protocol'; +import { AnthropicModel } from './anthropic-model'; +import { ContentBlock, ContentBlockType } from './content-block'; +import { Role, AnthropicMessage } from './message'; +import { ChatResponse } from './chat-response'; +import '../index'; + +const container = createContainer(); + +describe('AnthropicAPIImpl', () => { + let anthropicAPI: AnthropicAPI; + + beforeEach(() => { + container.rebind(RestOperations).toConstantValue({ + post: async (url: string, data: any, config: any) => { + if (data.stream) { + const chatResponse = { + id: 'msg_123', + type: 'message', + role: Role.ASSISTANT, + content: [ + { + type: ContentBlockType.TEXT, + text: 'How can I assist you today?' + } + ], + model: data.model, + stop_reason: 'end_turn', + stop_sequence: undefined, + usage: { + input_tokens: 10, + output_tokens: 20 + } + }; + const steam = new ReadableStream({ + start(controller) { + controller.enqueue('data: ' + JSON.stringify({ + type: 'message_start', + data: chatResponse + }) + '\n\n'); + controller.close(); + } + }); + return { data: steam }; + } + + return { + data: { + id: 'msg_123', + type: 'message', + role: Role.ASSISTANT, + content: [ + { + type: ContentBlockType.TEXT, + text: 'How can I assist you today?' + } + ], + model: data.model, + stop_reason: 'end_turn', + stop_sequence: undefined, + usage: { + input_tokens: 10, + output_tokens: 20 + } + } + }; + } + }); + anthropicAPI = container.get(AnthropicAPI); + }); + + describe('chat', () => { + it('should return a ChatResponse', async () => { + const messages = [ + new AnthropicMessage( + [new ContentBlock(ContentBlockType.TEXT, undefined, 'Hello')], + Role.USER + ) + ]; + const chatRequest = ChatRequest.builder() + .withModel(AnthropicModel.CLAUDE_3_HAIKU) + .withMessages(messages) + .withMaxTokens(1000) + .withStream(false) + .build(); + + const response = await anthropicAPI.chat(chatRequest); + expect(response).to.be.instanceOf(ChatResponse); + expect(response.model).to.equal(AnthropicModel.CLAUDE_3_HAIKU); + }); + + it('should throw an error if stream mode is enabled', async () => { + const messages = [ + new AnthropicMessage( + [new ContentBlock(ContentBlockType.TEXT, undefined, 'Hello')], + Role.USER + ) + ]; + const chatRequest = ChatRequest.builder() + .withModel(AnthropicModel.CLAUDE_3_HAIKU) + .withMessages(messages) + .withMaxTokens(1000) + .withStream(true) + .build(); + + try { + await anthropicAPI.chat(chatRequest); + expect.fail('Should have thrown an error'); + } catch (e) { + expect(e.message).to.equal('Request must set the stream property to false.'); + } + }); + }); + + describe('streamingChat', () => { + it('should return an Observable of ChatResponse', async () => { + const messages = [ + new AnthropicMessage( + [new ContentBlock(ContentBlockType.TEXT, undefined, 'Hello')], + Role.USER + ) + ]; + const chatRequest = ChatRequest.builder() + .withModel(AnthropicModel.CLAUDE_3_HAIKU) + .withMessages(messages) + .withMaxTokens(1000) + .withStream(true) + .build(); + + const response$ = await anthropicAPI.streamingChat(chatRequest); + response$.subscribe({ + next: response => { + expect(response).to.be.instanceOf(ChatResponse); + expect(response.model).to.equal(AnthropicModel.CLAUDE_3_HAIKU); + } + }); + }); + + it('should throw an error if stream mode is disabled', async () => { + const messages = [ + new AnthropicMessage( + [new ContentBlock(ContentBlockType.TEXT, undefined, 'Hello')], + Role.USER + ) + ]; + const chatRequest = ChatRequest.builder() + .withModel(AnthropicModel.CLAUDE_3_HAIKU) + .withMessages(messages) + .withMaxTokens(1000) + .withStream(false) + .build(); + + try { + await anthropicAPI.streamingChat(chatRequest); + expect.fail('Should have thrown an error'); + } catch (e) { + expect(e.message).to.equal('Request must set the stream property to true.'); + } + }); + }); +}); diff --git a/ai-packages/ai-anthropic/src/common/api/anthropic-api.ts b/ai-packages/ai-anthropic/src/common/api/anthropic-api.ts new file mode 100644 index 000000000..be3fe47b6 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/anthropic-api.ts @@ -0,0 +1,90 @@ +import { Assert, Autowired, Component, Optional, Value } from '@celljs/core'; +import { Observable } from 'rxjs'; +import { map, filter, groupBy, concatMap, reduce } from 'rxjs/operators'; +import { RestOperations } from '@celljs/http'; +import { ChatRequest } from './chat-request'; +import { ChatResponse } from './chat-response'; +import { AnthropicAPI, AnthropicAPIOptions } from './api-protocol'; +import { instanceToPlain, plainToInstance } from 'class-transformer'; +import { SSEUtil, StreamEvent } from '@celljs/ai-core'; +import { EventType, AnthropicStreamEvent, ToolUseAggregationEvent } from './event'; +import { StreamUtil } from '../utils'; + +@Component(AnthropicAPI) +export class AnthropicAPIImpl implements AnthropicAPI { + + @Autowired(RestOperations) + protected readonly restOperations: RestOperations; + + @Value('cell.ai.anthropic.api') + protected readonly options?: AnthropicAPIOptions; + + @Autowired(AnthropicAPIOptions) + @Optional() + protected readonly defaultOptions?: AnthropicAPIOptions; + + protected get baseUrl(): string { + return this.options?.baseUrl ?? this.defaultOptions?.baseUrl ?? 'https://api.anthropic.com'; + } + + protected get apiKey(): string | undefined { + return this.options?.apiKey ?? this.defaultOptions?.apiKey; + } + + protected get anthropicVersion(): string { + return this.options?.anthropicVersion ?? this.defaultOptions?.anthropicVersion ?? '2023-06-01'; + } + + async chat(chatRequest: ChatRequest): Promise { + Assert.isTrue(!chatRequest.stream, 'Request must set the stream property to false.'); + const { data } = await this.restOperations + .post('/v1/messages', instanceToPlain(chatRequest), { baseURL: this.baseUrl }); + + return plainToInstance(ChatResponse, data); + } + async streamingChat(chatRequest: ChatRequest): Promise> { + Assert.isTrue(chatRequest.stream, 'Request must set the stream property to true.'); + const { data } = await this.restOperations + .post( + '/v1/messages', + instanceToPlain(chatRequest), + { + baseURL: this.baseUrl, + headers: { + 'Accept': 'text/event-stream', + }, + responseType: 'stream' + } + ); + let isInsideTool = false; + return SSEUtil.toObservable>(data) + .pipe( + map(item => plainToInstance(AnthropicStreamEvent, item)), + filter(item => item.data.type !== EventType.PING), + map(item => { + if (StreamUtil.isToolUseStart(item.data)) { + isInsideTool = true; + } + return item; + }), + groupBy(item => { + if (isInsideTool && StreamUtil.isToolUseFinish(item.data)) { + isInsideTool = false; + return true; + } + return !isInsideTool; + }), + concatMap(group => + group.pipe( + reduce( + (acc, curr) => StreamUtil.mergeToolUseEvents(acc, curr.data), + new ToolUseAggregationEvent() + ) + ) + ), + map(event => StreamUtil.eventToChatResponse(event)), + filter(response => !!response.type) + ); + + } +} diff --git a/ai-packages/ai-anthropic/src/common/api/anthropic-model.ts b/ai-packages/ai-anthropic/src/common/api/anthropic-model.ts new file mode 100644 index 000000000..67ced9c18 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/anthropic-model.ts @@ -0,0 +1,42 @@ +/** + * Helper class for the Anthropic model. + */ +export enum AnthropicModel { + + /** + * The claude-3-5-sonnet-20241022 model. + */ + CLAUDE_3_5_SONNET = 'claude-3-5-sonnet-latest', + + /** + * The CLAUDE_3_OPUS + */ + CLAUDE_3_OPUS = 'claude-3-opus-latest', + + /** + * The CLAUDE_3_SONNET + */ + CLAUDE_3_SONNET = 'claude-3-sonnet-20240229', + + /** + * The CLAUDE 3.5 HAIKU + */ + CLAUDE_3_5_HAIKU = 'claude-3-5-haiku-latest', + + /** + * The CLAUDE_3_HAIKU + */ + CLAUDE_3_HAIKU = 'claude-3-haiku-20240307', + + // Legacy models + /** + * The CLAUDE_2_1 + */ + CLAUDE_2_1 = 'claude-2.1', + + /** + * The CLAUDE_2_0 + */ + CLAUDE_2 = 'claude-2.0' + +} diff --git a/ai-packages/ai-anthropic/src/common/api/api-protocol.ts b/ai-packages/ai-anthropic/src/common/api/api-protocol.ts new file mode 100644 index 000000000..ff0a6ce6b --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/api-protocol.ts @@ -0,0 +1,45 @@ +import { Observable } from 'rxjs'; +import { ChatRequest } from './chat-request'; +import { ChatResponse } from './chat-response'; + +export const AnthropicAPI = Symbol('AnthropicAPI'); +export const AnthropicAPIOptions = Symbol('AnthropicAPIOptions'); + +/** + * Anthropic API options. + */ +export interface AnthropicAPIOptions { + /** + * Base URL of the Anthropic API. + */ + baseUrl?: string; + + /** + * API key to authenticate with the Anthropic API. + */ + apiKey?: string; + + /** + * Version of the Anthropic API. + */ + anthropicVersion?: string; +} + +export interface AnthropicAPI { + /** + * Generate the next message in a chat with a provided model. + * This is a streaming endpoint (controlled by the 'stream' request property), so + * there will be a series of responses. The final response object will include + * statistics and additional data from the request. + * @param chatRequest Chat request. + * @return Chat response. + */ + chat(chatRequest: ChatRequest): Promise; + /** + * Streaming response for the chat completion request. + * @param chatRequest Chat request. The request must set the stream property to true. + * @return Chat response as a {@link Flux} stream. + */ + streamingChat(chatRequest: ChatRequest): Promise>; + +} diff --git a/ai-packages/ai-anthropic/src/common/api/chat-request.ts b/ai-packages/ai-anthropic/src/common/api/chat-request.ts new file mode 100644 index 000000000..8d14baf72 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/chat-request.ts @@ -0,0 +1,280 @@ +import { Type, Expose } from 'class-transformer'; +import { AnthropicMessage } from './message'; + +/** + * Tool description. + */ +export class Tool { + /** + * The name of the tool. + */ + name: string; + + /** + * A description of the tool. + */ + description: string; + + /** + * The input schema of the tool. + */ + inputSchema: Record; + + constructor(name: string, description: string, inputSchema: Record) { + this.name = name; + this.description = description; + this.inputSchema = inputSchema; + } +} + +/** + * Metadata about the request. + */ +export class Metadata { + /** + * An external identifier for the user who is associated with the + * request. This should be a uuid, hash value, or other opaque identifier. + * Anthropic may use this id to help detect abuse. Do not include any identifying + * information such as name, email address, or phone number. + */ + @Expose({ name: 'user_id' }) + userId: string; + + constructor(userId: string) { + this.userId = userId; + } +} + +/** + * Chat completion request object. + */ +export class ChatRequest { + + /** + * The model that will complete your prompt. See the list of + * models for + * additional details and options. + */ + model: string; + + /** + * Input messages. + */ + @Type(() => AnthropicMessage) + messages: AnthropicMessage[]; + + /** + * System prompt. A system prompt is a way of providing context and + * instructions to Claude, such as specifying a particular goal or role. See our + * guide to system + * prompts. + */ + system?: string; + + /** + * The maximum number of tokens to generate before stopping. Note + * that our models may stop before reaching this maximum. This parameter only + * specifies the absolute maximum number of tokens to generate. Different models have + * different maximum values for this parameter. + */ + @Expose({ name: 'max_tokens' }) + maxTokens: number; + + /** + * An object describing metadata about the request. + */ + metadata?: Metadata; + + /** + * Custom text sequences that will cause the model to stop + * generating. Our models will normally stop when they have naturally completed their + * turn, which will result in a response stop_reason of "end_turn". If you want the + * model to stop generating when it encounters custom strings of text, you can use the + * stop_sequences parameter. If the model encounters one of the custom sequences, the + * response stop_reason value will be "stop_sequence" and the response stop_sequence + * value will contain the matched stop sequence. + */ + @Expose({ name: 'stop_sequences' }) + stopSequences?: string[]; + + /** + * Whether to incrementally stream the response using server-sent + * events. + */ + stream: boolean; + + /** + * Amount of randomness injected into the response.Defaults to 1.0. + * Ranges from 0.0 to 1.0. Use temperature closer to 0.0 for analytical / multiple + * choice, and closer to 1.0 for creative and generative tasks. Note that even with + * temperature of 0.0, the results will not be fully deterministic. + */ + temperature?: number; + + /** + * Use nucleus sampling. In nucleus sampling, we compute the cumulative + * distribution over all the options for each subsequent token in decreasing + * probability order and cut it off once it reaches a particular probability specified + * by top_p. You should either alter temperature or top_p, but not both. Recommended + * for advanced use cases only. You usually only need to use temperature. + */ + @Expose({ name: 'top_p' }) + topP?: number; + + /** + * Only sample from the top K options for each subsequent token. Used to + * remove "long tail" low probability responses. Learn more technical details here. + * Recommended for advanced use cases only. You usually only need to use temperature. + */ + @Expose({ name: 'top_k' }) + topK?: number; + + /** + * Definitions of tools that the model may use. If provided the model may + * return tool_use content blocks that represent the model's use of those tools. You + * can then run those tools using the tool input generated by the model and then + * optionally return results back to the model using tool_result content blocks. + */ + @Type(() => Tool) + tools?: Tool[]; + + constructor( + model: string, + messages: AnthropicMessage[], + system: string | undefined, + maxTokens: number, + temperature: number | undefined, + stream: boolean, + metadata?: Metadata, + stopSequences?: string[], + topP?: number, + topK?: number, + tools?: Tool[], + ) { + this.model = model; + this.messages = messages; + this.system = system; + this.maxTokens = maxTokens; + this.metadata = metadata; + this.stopSequences = stopSequences; + this.stream = stream; + this.temperature = temperature; + this.topP = topP; + this.topK = topK; + this.tools = tools; + } + + /** + * Builder class for ChatRequest + */ + static builder(): ChatRequestBuilder { + return new ChatRequestBuilder(); + } + + static from(request: ChatRequest): ChatRequestBuilder { + return new ChatRequestBuilder(request); + } +} + +/** + * Builder interface for ChatRequest + */ +export class ChatRequestBuilder { + private model: string; + private messages: AnthropicMessage[]; + private system?: string; + private maxTokens: number; + private metadata?: Metadata; + private stopSequences?: string[]; + private stream: boolean = false; + private temperature?: number; + private topP?: number; + private topK?: number; + private tools?: Tool[]; + + constructor(request?: ChatRequest) { + if (request) { + this.model = request.model; + this.messages = request.messages; + this.system = request.system; + this.maxTokens = request.maxTokens; + this.metadata = request.metadata; + this.stopSequences = request.stopSequences; + this.stream = request.stream; + this.temperature = request.temperature; + this.topP = request.topP; + this.topK = request.topK; + } + } + + withMessages(messages: AnthropicMessage[]): ChatRequestBuilder { + this.messages = messages; + return this; + } + + withStream(stream: boolean): ChatRequestBuilder { + this.stream = stream; + return this; + } + + withModel(model: string): ChatRequestBuilder { + this.model = model; + return this; + } + + withSystem(system: string): ChatRequestBuilder { + this.system = system; + return this; + } + + withStopSequences(stopSequences: string[]): ChatRequestBuilder { + this.stopSequences = stopSequences; + return this; + } + + withMaxTokens(maxTokens: number): ChatRequestBuilder { + this.maxTokens = maxTokens; + return this; + } + + withTemperature(temperature: number): ChatRequestBuilder { + this.temperature = temperature; + return this; + } + + withTopP(topP: number): ChatRequestBuilder { + this.topP = topP; + return this; + } + + withTopK(topK: number): ChatRequestBuilder { + this.topK = topK; + return this; + } + + withMetadata(metadata: Metadata): ChatRequestBuilder { + this.metadata = metadata; + return this; + } + + withTools(tools: Tool[]): ChatRequestBuilder { + this.tools = tools; + return this; + } + + build(): ChatRequest { + return new ChatRequest( + this.model, + this.messages, + this.system, + this.maxTokens, + this.temperature, + this.stream, + this.metadata, + this.stopSequences, + this.topP, + this.topK, + this.tools + ); + } +} diff --git a/ai-packages/ai-anthropic/src/common/api/chat-response.ts b/ai-packages/ai-anthropic/src/common/api/chat-response.ts new file mode 100644 index 000000000..c4745e08d --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/chat-response.ts @@ -0,0 +1,161 @@ +import { Expose, Type } from 'class-transformer'; +import { Role } from './message'; +import { Assert } from '@celljs/core'; +import { ContentBlock } from './content-block'; +import { Usage } from './usage'; + +/** + * Chat completion response object. + */ +export class ChatResponse { + + /** + * Unique object identifier. The format and length of IDs may change over + * time. + */ + id: string; + + /** + * Object type. For Messages, this is always "message". + */ + type: string; + + /** + * Conversational role of the generated message. This will always be + * "assistant". + */ + role: Role; + + /** + * Content generated by the model. This is an array of content blocks. + */ + @Type(() => ContentBlock) + content: ContentBlock[]; + + /** + * The model that handled the request. + */ + model: string; + + /** + * The reason the model stopped generating tokens. This will be one + * of "end_turn", "max_tokens", "stop_sequence", "tool_use", or "timeout". + */ + @Expose({ name: 'stop_reason' }) + stopReason: string; + + /** + * Which custom stop sequence was generated, if any. + */ + @Expose({ name: 'stop_sequence' }) + stopSequence?: string; + + /** + * Input and output token usage. + */ + usage: Usage; + + constructor( + id: string, + type: string, + role: Role, + content: ContentBlock[], + model: string, + stopReason: string, + stopSequence: string | undefined, + usage: any + ) { + this.id = id; + this.type = type; + this.role = role; + this.content = content; + this.model = model; + this.stopReason = stopReason; + this.stopSequence = stopSequence; + this.usage = usage; + } + + static builder(): ChatResponseBuilder { + return new ChatResponseBuilder(); + } +} + +export class ChatResponseBuilder { + private type: string; + private id: string; + private role: Role; + private content: ContentBlock[]; + private model: string; + private stopReason: string; + private stopSequence?: string; + private usage: Usage; + + constructor() { + } + + withType(type: string): ChatResponseBuilder { + this.type = type; + return this; + } + + withId(id: string): ChatResponseBuilder { + this.id = id; + return this; + } + + withRole(role: Role): ChatResponseBuilder { + this.role = role; + return this; + } + + withContent(content: ContentBlock[]): ChatResponseBuilder { + this.content = content; + return this; + } + + withModel(model: string): ChatResponseBuilder { + this.model = model; + return this; + } + + withStopReason(stopReason: string): ChatResponseBuilder { + this.stopReason = stopReason; + return this; + } + + withStopSequence(stopSequence: string): ChatResponseBuilder { + this.stopSequence = stopSequence; + return this; + } + + withUsage(usage: Usage): ChatResponseBuilder { + this.usage = usage; + return this; + } + + withOutputTokens(outputTokens: number): ChatResponseBuilder { + Assert.isTrue(!!this.usage, 'Usage must be set before setting output tokens'); + this.usage.outputTokens = outputTokens; + return this; + } + + build(): ChatResponse { + return new ChatResponse(this.id, this.type, this.role, this.content, this.model, this.stopReason, this.stopSequence, this.usage); + } +} + +export class ChatResponseBuilderReference { + + private static instance: ChatResponseBuilder; + + static get(): ChatResponseBuilder { + if (!this.instance) { + this.instance = new ChatResponseBuilder(); + } + return this.instance; + } + + static reset(): void { + this.instance = new ChatResponseBuilder(); + } +} diff --git a/ai-packages/ai-anthropic/src/common/api/content-block.ts b/ai-packages/ai-anthropic/src/common/api/content-block.ts new file mode 100644 index 000000000..84c692101 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/content-block.ts @@ -0,0 +1,155 @@ +import { Expose } from 'class-transformer'; + +/** + * The type of the content block. + */ +export enum ContentBlockType { + /** + * Tool request + */ + TOOL_USE = 'tool_use', + + /** + * Send tool result back to LLM. + */ + TOOL_RESULT = 'tool_result', + + /** + * Text message. + */ + TEXT = 'text', + + /** + * Text delta message. Returned from the streaming response. + */ + TEXT_DELTA = 'text_delta', + + /** + * Tool use input partial JSON delta streaming. + */ + INPUT_JSON_DELTA = 'input_json_delta', + + /** + * Image message. + */ + IMAGE = 'image', + + /** + * Document message. + */ + DOCUMENT = 'document', + +} + +/** + * The source of the media content. (Applicable for "image" types only) + */ +export class Source { + + /** + * The type of the media content. Only "base64" is supported at the moment. + */ + type: string; + + /** + * The media type of the content. For example, "image/png" or "image/jpeg". + */ + @Expose({ name: 'media_type' }) + mediaType: string; + + /** + * The base64-encoded data of the content. + */ + data: string; + + constructor(type: string, mediaType: string, data: string) { + this.type = type; + this.mediaType = mediaType; + this.data = data; + } + + /** + * Create source + * @param mediaType The media type of the content. + * @param data The content data. + */ + public static from(mediaType: string, data: string): Source { + return new Source('base64', mediaType, data); + } + +} + +/** + * The content block of the message. + */ +export class ContentBlock { + + /** + * The content type can be "text", "image", "tool_use", "tool_result" or "text_delta". + */ + type: ContentBlockType; + + /** + * The source of the media content. Applicable for "image" types only. + */ + source?: Source; + + /** + * The text of the message. Applicable for "text" types only. + */ + text?: string; + + /** + * The index of the content block. Applicable only for streaming responses. + */ + index?: number; + + /** + * The id of the tool use. Applicable only for tool_use response. + */ + id?: string; + + /** + * The name of the tool use. Applicable only for tool_use response. + */ + name?: string; + + /** + * The input of the tool use. Applicable only for tool_use response. + */ + input?: Record; + + /** + * The id of the tool use. Applicable only for tool_result response. + */ + @Expose({ name: 'tool_use_id' }) + toolUseId?: string; + + /** + * The content of the tool result. Applicable only for tool_result response. + */ + content?: string; + + constructor( + type: ContentBlockType, + source?: Source, + text?: string, + index?: number, + id?: string, + name?: string, + input?: Record, + toolUseId?: string, + content?: string + ) { + this.type = type; + this.source = source; + this.text = text; + this.index = index; + this.id = id; + this.name = name; + this.input = input; + this.toolUseId = toolUseId; + this.content = content; + } + +} diff --git a/ai-packages/ai-anthropic/src/common/api/event.ts b/ai-packages/ai-anthropic/src/common/api/event.ts new file mode 100644 index 000000000..7832c6140 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/event.ts @@ -0,0 +1,376 @@ +import { Type } from 'class-transformer'; +import { ChatResponse } from './chat-response'; +import { StreamEvent } from '@celljs/ai-core'; +import { ContentBlockType } from './content-block'; + +/** + * The event type of the streamed chunk. + */ +export enum EventType { + /** + * Message start event. Contains a Message object with empty content. + */ + MESSAGE_START = 'message_start', + + /** + * Message delta event, indicating top-level changes to the final Message object. + */ + MESSAGE_DELTA = 'message_delta', + + /** + * A final message stop event. + */ + MESSAGE_STOP = 'message_stop', + + /** + * Content block start event. + */ + CONTENT_BLOCK_START = 'content_block_start', + + /** + * Content block delta event. + */ + CONTENT_BLOCK_DELTA = 'content_block_delta', + + /** + * A final content block stop event. + */ + CONTENT_BLOCK_STOP = 'content_block_stop', + + /** + * Error event. + */ + ERROR = 'error', + + /** + * Ping event. + */ + PING = 'ping', + + /** + * Artificially created event to aggregate tool use events. + */ + TOOL_USE_AGGREGATE = 'tool_use_aggregate' + +} + +/** + * Anthropic event. + */ +export class AnthropicEvent { + type: EventType; +} + +/** + * Content block body. + */ +export class ContentBlockBody { + /** + * The type of the content block. e.g. "text", "tool_use". + */ + type: string; +} + +/** + * Text content block delta. + */ +export class ContentBlockText extends ContentBlockBody { + + /* + * The text of the message. Applicable for "text" types only. + */ + text: string; +} + +/** + * Tool use content block. + */ +export class ContentBlockToolUse extends ContentBlockBody { + /* + * The id of the tool use. Applicable only for tool_use response. + */ + id: string; + + /* + * The name of the tool use. Applicable only for tool_use response. + */ + name: string; + + /* + * The input of the tool use. Applicable only for tool_use response. + */ + input: Record; + + constructor(type: ContentBlockType, id: string, name: string, input: Record) { + super(); + this.type = type; + this.id = id; + this.name = name; + this.input = input; + } +} + +export class AnthropicEventWithContentBlock extends AnthropicEvent { + /** + * The content block body. + */ + @Type(() => ContentBlockBody, { + discriminator: { + property: 'type', + subTypes: [ + { value: ContentBlockText, name: ContentBlockType.TEXT }, + { value: ContentBlockToolUse, name: ContentBlockType.TOOL_USE } + ] + } + }) + contentBlock: ContentBlockBody; +} + +/** + * Special event used to aggregate multiple tool use events into a single event with + * list of aggregated ContentBlockToolUse. + */ +export class ToolUseAggregationEvent extends AnthropicEvent { + /* + * The index of the content block. Applicable only for streaming responses. + */ + index?: number; + + /* + * The id of the tool use. Applicable only for tool_use response. + */ + id?: string; + + /* + * The name of the tool use. Applicable only for tool_use response. + */ + name?: string; + + /* + * The partial JSON content. + */ + partialJson: string = ''; + + /* + * The tool content blocks. + */ + toolContentBlocks: ContentBlockToolUse[] = []; + + isEmpty(): boolean { + return (this.index === undefined || !this.id || !this.name || !this.partialJson); + } + + appendPartialJson(partialJson: string): ToolUseAggregationEvent { + this.partialJson = this.partialJson + partialJson; + return this; + } + + squashIntoContentBlock(): void { + const map = (this.partialJson) ? JSON.parse(this.partialJson) : {}; + this.toolContentBlocks.push(new ContentBlockToolUse(ContentBlockType.TOOL_USE, this.id!, this.name!, map)); + this.index = undefined; + this.id = undefined; + this.name = undefined; + this.partialJson = ''; + } +} + +/** + * Content block delta body. + */ +export class ContentBlockDeltaBody { + /* + * The type of the content block. e.g. "text", "input_json". + */ + type: string; +} + +/** + * Text content block delta. + */ +export class ContentBlockDeltaText extends ContentBlockDeltaBody { + /* + * The text of the message. Applicable for "text" types only. + */ + text: string; +} + +/** + * JSON content block delta. + */ +export class ContentBlockDeltaJson extends ContentBlockDeltaBody { + /* + * The partial JSON content. + */ + partialJson: string; +} + +/** + * Content block stop event. + */ +export class ContentBlockStopEvent extends AnthropicEvent { + /* + * The index of the content block. Applicable only for streaming responses. + */ + index: number; +} + +/** + * Content block start event. + */ +export class ContentBlockStartEvent extends AnthropicEventWithContentBlock { + + /* + * The index of the content block. Applicable only for streaming responses. + */ + index: number; +} + +/** + * Content block delta event. + */ +export class ContentBlockDeltaEvent extends AnthropicEvent { + + /* + * The index of the content block. Applicable only for streaming responses. + */ + index: number; + + /* + * The content block delta body. + */ + @Type(() => ContentBlockDeltaBody, { + discriminator: { + property: 'type', + subTypes: [ + { value: ContentBlockDeltaText, name: ContentBlockType.TEXT_DELTA }, + { value: ContentBlockDeltaJson, name: ContentBlockType.INPUT_JSON_DELTA } + ] + } + }) + delta: ContentBlockDeltaBody; + +} + +/** + * Message start event. + */ +export class MessageStartEvent { + /* + * The event type. + */ + type: EventType; + + /* + * The message body. + */ + message: ChatResponse; + + constructor(type: EventType, message: ChatResponse) { + this.type = type; + this.message = message; + } +} + +/** + * Message delta event. + */ +export class MessageDeltaEvent extends AnthropicEvent { + + /* + * The message delta body. + */ + delta: MessageDelta; + + /* + * The message delta usage. + */ + usage: MessageDeltaUsage; + +} + +/** + * Message delta. + */ +export class MessageDelta { + /* + * The stop reason. + */ + stopReason: string; + + /* + * The stop sequence. + */ + stopSequence: string; +} + +/** + * Message delta usage. + */ +export class MessageDeltaUsage { + /* + * The output tokens. + */ + outputTokens: number; +} + +/** + * Message stop event. + */ +export class MessageStopEvent extends AnthropicEvent { + +} + +/** + * Error event. + */ +export class ErrorEvent extends AnthropicEvent { + + /* + * The error body. + */ + error: Error; +} + +/** + * Error body. + */ +export class Error { + /* + * The error type. + */ + type: string; + + /* + * The error message. + */ + message: string; +} + +/** + * Ping event. + */ +export class PingEvent extends AnthropicEvent { + +} + +export class AnthropicStreamEvent implements StreamEvent { + event?: EventType; + @Type(() => AnthropicEvent, { + discriminator: { + property: 'type', + subTypes: [ + { value: ContentBlockStartEvent, name: EventType.CONTENT_BLOCK_START }, + { value: ContentBlockDeltaEvent, name: EventType.CONTENT_BLOCK_DELTA }, + { value: ContentBlockStopEvent, name: EventType.CONTENT_BLOCK_STOP }, + { value: PingEvent, name: EventType.PING }, + { value: ErrorEvent, name: EventType.ERROR }, + { value: MessageStartEvent, name: EventType.MESSAGE_START }, + { value: MessageDeltaEvent, name: EventType.MESSAGE_DELTA }, + { value: MessageStopEvent, name: EventType.MESSAGE_STOP } + ] + } + }) + data: AnthropicEvent; + raw: string[]; +} diff --git a/ai-packages/ai-anthropic/src/common/api/index.ts b/ai-packages/ai-anthropic/src/common/api/index.ts new file mode 100644 index 000000000..8a544e755 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/index.ts @@ -0,0 +1,9 @@ +export * from './api-protocol'; +export * from './chat-request'; +export * from './chat-response'; +export * from './anthropic-api'; +export * from './event'; +export * from './message'; +export * from './usage'; +export * from './content-block'; +export * from './anthropic-model'; diff --git a/ai-packages/ai-anthropic/src/common/api/message.ts b/ai-packages/ai-anthropic/src/common/api/message.ts new file mode 100644 index 000000000..ded11c6fc --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/message.ts @@ -0,0 +1,49 @@ +import { Type } from 'class-transformer'; +import { ContentBlock } from './content-block'; + +/** + * The role of the message in the conversation. + */ +export enum Role { + /** + * User message type. + */ + USER = 'user', + /** + * Assistant message type. Usually the response from the model. + */ + ASSISTANT = 'assistant', +} + +/** + * Input messages. + * + * Our models are trained to operate on alternating user and assistant conversational + * turns. When creating a new Message, you specify the prior conversational turns with + * the messages parameter, and the model then generates the next Message in the + * conversation. Each input message must be an object with a role and content. You can + * specify a single user-role message, or you can include multiple user and assistant + * messages. The first message must always use the user role. If the final message + * uses the assistant role, the response content will continue immediately from the + * content in that message. This can be used to constrain part of the model's + * response. + */ +export class AnthropicMessage { + /** + * The contents of the message. Can be of one of String or MultiModalContent + * types. + */ + @Type(() => ContentBlock) + content: ContentBlock[]; + + /** + * The role of the messages author. Could be one of the {@link Role} + * types. + */ + role: Role; + + constructor(content: ContentBlock[], role: Role) { + this.content = content; + this.role = role; + } +} diff --git a/ai-packages/ai-anthropic/src/common/api/usage.ts b/ai-packages/ai-anthropic/src/common/api/usage.ts new file mode 100644 index 000000000..9895f22da --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/api/usage.ts @@ -0,0 +1,24 @@ +import { Expose } from 'class-transformer'; + +/** + * Usage statistics. + */ +export class Usage { + + /** + * The number of input tokens which were used. + */ + @Expose({ name: 'input_tokens' }) + inputTokens: number; + + /** + * The number of output tokens which were used. + */ + @Expose({ name: 'output_tokens' }) + outputTokens: number; + + constructor(inputTokens: number, outputTokens: number) { + this.inputTokens = inputTokens; + this.outputTokens = outputTokens; + } +} diff --git a/ai-packages/ai-anthropic/src/common/index.ts b/ai-packages/ai-anthropic/src/common/index.ts new file mode 100644 index 000000000..bed443678 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/index.ts @@ -0,0 +1,2 @@ +export * from './api'; +export * from './model'; diff --git a/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.spec.ts b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.spec.ts new file mode 100644 index 000000000..268960f3c --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.spec.ts @@ -0,0 +1,67 @@ +import { expect } from 'chai'; +import '../index'; +import { AnthropicChatModel } from './anthropic-chat-model'; +import { AssistantMessage, PromptTemplate } from '@celljs/ai-core'; +import { createContainer } from '../test/test-container'; +import { AnthropicModel } from '../api/anthropic-model'; +import { MockAnthropicAPI } from '../test/mock-anthropic-api'; +import { AnthropicAPI } from '../api/api-protocol'; + +const container = createContainer(); + +describe('AnthropicChatModel', () => { + let anthropicChatModel: AnthropicChatModel; + let promptTemplate: PromptTemplate; + + before(() => { + container.rebind(AnthropicAPI).to(MockAnthropicAPI).inSingletonScope(); + anthropicChatModel = container.get(AnthropicChatModel); + promptTemplate = container.get(PromptTemplate); + }); + + describe('call', () => { + it('should return a ChatResponse for a given prompt', async () => { + const prompt = await promptTemplate.create('Hello', { chatOptions: { model: AnthropicModel.CLAUDE_3_5_SONNET } }); + const response = await anthropicChatModel.call(prompt); + expect(response).to.have.property('result'); + expect(response.result.output).to.be.instanceOf(AssistantMessage); + }); + + it('should throw an error if the model is not set', async () => { + const prompt = await promptTemplate.create('Hello', { chatOptions: { model: '' } }); + try { + await anthropicChatModel.call(prompt); + } catch (e) { + expect(e).to.be.instanceOf(Error); + expect(e.message).to.equal('Model is not set!'); + } + }); + }); + + describe('stream', () => { + it('should return an Observable of ChatResponse for a given prompt', done => { + (async () => { + const prompt = await promptTemplate.create('Hello', { chatOptions: { model: AnthropicModel.CLAUDE_3_5_SONNET } }); + const response$ = await anthropicChatModel.stream(prompt); + response$.subscribe({ + next: response => { + expect(response).to.have.property('result'); + expect(response.result.output).to.be.instanceOf(AssistantMessage); + }, + complete: () => done() + }); + })(); + }); + + it('should throw an error if the model is not set', async () => { + const prompt = await promptTemplate.create('Hello', { chatOptions: { model: '' } }); + try { + await anthropicChatModel.stream(prompt); + } catch (e) { + expect(e).to.be.instanceOf(Error); + expect(e.message).to.equal('Model is not set!'); + } + }); + + }); +}); diff --git a/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.ts b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.ts new file mode 100644 index 000000000..b659bad18 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-model.ts @@ -0,0 +1,187 @@ +import { + AssistantMessage, + ChatGenerationMetadata, + ChatModel, + ChatResponse, + ChatResponseMetadata, + FunctionCallback, + Generation, + Media, + MessageType, + Prompt, + ToolCall, + ToolResponseMessage, + Usage, + UserMessage +} from '@celljs/ai-core'; +import { Autowired, Component, IllegalArgumentError, Logger } from '@celljs/core'; +import { Observable } from 'rxjs'; +import { AnthropicAPI, ChatResponse as AnthriopicChatResponse, AnthropicMessage, ChatRequest, ContentBlock, ContentBlockType, Role, Source, Tool } from '../api'; +import { AnthropicChatOptions } from './anthropic-chat-options'; +import { map } from 'rxjs/operators'; + +@Component(ChatModel) +export class AnthropicChatModel implements ChatModel { + @Autowired(AnthropicChatOptions) + protected readonly defaultOptions: AnthropicChatOptions; + + @Autowired(AnthropicAPI) + protected readonly chatApi: AnthropicAPI; + + @Autowired(Logger) + protected readonly logger: Logger; + + protected getContentBlockTypeByMedia(media: Media): ContentBlockType { + if (media.mediaType.startsWith('image')) { + return ContentBlockType.IMAGE; + } else if (media.mediaType.startsWith('pdf')) { + return ContentBlockType.DOCUMENT; + } + throw new IllegalArgumentError(`Unsupported media type: ${media.mediaType}. Supported types are: images (image/*) and PDF documents (application/pdf)`); + } + + protected fromMediaData(mediaData: any): string { + if (mediaData instanceof Uint8Array) { + return btoa(String.fromCharCode(...mediaData)); + } else if (typeof mediaData === 'string') { + return mediaData; + } else { + throw new IllegalArgumentError(`Unsupported media data type: ${typeof mediaData}`); + } + } + + protected resolveFunctionCallbacks(functions: Set): FunctionCallback[] { + return []; + } + + protected getFunctionTools(functions: Set): Tool[] { + return this.resolveFunctionCallbacks(functions).map(func => new Tool(func.name, func.description, JSON.parse(func.inputTypeSchema))); + } + + protected createRequest(prompt: Prompt, stream: boolean): ChatRequest { + const userMessages = prompt.instructions + .filter(message => message.messageType !== MessageType.SYSTEM) + .map(message => { + if (message.messageType === MessageType.USER) { + const contents = [new ContentBlock(ContentBlockType.TEXT, undefined, message.content)]; + if (message instanceof UserMessage) { + if (message.media.length > 0) { + const mediaContent = message.media.map(media => { + const contentBlockType = this.getContentBlockTypeByMedia(media); + const source = new Source('base64', media.mediaType, this.fromMediaData(media.data)); + return new ContentBlock(contentBlockType, source); + }); + contents.push(...mediaContent); + } + } + return new AnthropicMessage(contents, message.messageType === MessageType.USER ? Role.USER : Role.ASSISTANT); + } else if (message.messageType === MessageType.ASSISTANT) { + const assistantMessage = message as AssistantMessage; + const contentBlocks = []; + if (message.content) { + contentBlocks.push(new ContentBlock(ContentBlockType.TEXT, undefined, message.content)); + } + if (assistantMessage.toolCalls.length > 0) { + for (const toolCall of assistantMessage.toolCalls) { + contentBlocks.push( + new ContentBlock(ContentBlockType.TOOL_USE, undefined, undefined, undefined, toolCall.id, toolCall.name, JSON.parse(toolCall.arguments)) + ); + } + } + return new AnthropicMessage(contentBlocks, Role.ASSISTANT); + } else if (message.messageType === MessageType.TOOL) { + const toolResponses = (message as ToolResponseMessage).responses.map( + toolResponse => new ContentBlock(ContentBlockType.TOOL_RESULT, undefined, undefined, undefined, toolResponse.id, toolResponse.responseData) + ); + return new AnthropicMessage(toolResponses, Role.USER); + } else { + throw new IllegalArgumentError(`Unsupported message type: ${message.messageType}`); + } + }); + + const systemPrompt = prompt.instructions + .filter(m => m.messageType === MessageType.SYSTEM) + .map(m => m.content) + .join('\n'); + + let request = new ChatRequest(this.defaultOptions.model, userMessages, systemPrompt, this.defaultOptions.maxTokens, this.defaultOptions.temperature, stream); + + if (prompt.options) { + request = Object.assign(request, prompt.options); + } + + const functionsForThisRequest: Set = new Set(); + + if (this.defaultOptions.functions.size > 0) { + this.defaultOptions.functions.forEach(func => functionsForThisRequest.add(func)); + } + + if (functionsForThisRequest.size > 0) { + const tools = this.getFunctionTools(functionsForThisRequest); + request.tools = tools; + } + + return request; + } + + protected from(response: AnthriopicChatResponse): ChatResponseMetadata { + return ChatResponseMetadata.builder() + .id(response.id) + .model(response.model) + .usage(Usage.from(response.usage.inputTokens, response.usage.outputTokens)) + .keyValue('stop-reason', response.stopReason) + .keyValue('stop-sequence', response.stopSequence) + .keyValue('type', response.type) + .build(); + } + + protected toChatResponse(response?: AnthriopicChatResponse): ChatResponse { + if (response === undefined) { + this.logger.warn('Null chat completion returned'); + return ChatResponse.from([]); + } + + const generations = response.content + .filter(content => content.type !== ContentBlockType.TOOL_USE) + .map(content => Generation.from(new AssistantMessage(content.text), ChatGenerationMetadata.from(response.stopReason))); + + const allGenerations = [...generations]; + + if (response.stopReason && generations.length === 0) { + const generation = Generation.from(new AssistantMessage(), ChatGenerationMetadata.from(response.stopReason)); + allGenerations.push(generation); + } + + const toolToUseList = response.content.filter(c => c.type === ContentBlockType.TOOL_USE); + if (toolToUseList.length > 0) { + const toolCalls: ToolCall[] = []; + for (const toolToUse of toolToUseList) { + toolCalls.push({ + id: toolToUse.id!, + type: 'function', + name: toolToUse.name!, + arguments: JSON.stringify(toolToUse.input) + }); + } + + const assistantMessage = new AssistantMessage('', [], {}, toolCalls); + const toolCallGeneration = Generation.from(assistantMessage, ChatGenerationMetadata.from(response.stopReason)); + allGenerations.push(toolCallGeneration); + } + + return ChatResponse.from(allGenerations, this.from(response)); + } + + async call(prompt: Prompt): Promise { + const request = this.createRequest(prompt, false); + const chatResponse = await this.chatApi.chat(request); + return this.toChatResponse(chatResponse); + } + + async stream(prompt: Prompt): Promise> { + const request = this.createRequest(prompt, true); + const chatResponse = await this.chatApi.streamingChat(request); + return chatResponse.pipe(map(chunk => this.toChatResponse(chunk))); + } + +} diff --git a/ai-packages/ai-anthropic/src/common/model/anthropic-chat-options.ts b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-options.ts new file mode 100644 index 000000000..57210ee40 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/model/anthropic-chat-options.ts @@ -0,0 +1,152 @@ +import { FunctionCallback, FunctionCallingOptions } from '@celljs/ai-core'; +import { Exclude, Expose } from 'class-transformer'; +import { Metadata } from '../api/chat-request'; +import { Constant } from '@celljs/core'; + +/** + * The options to be used when sending a chat request to the Anthropic API. + */ +@Constant(AnthropicChatOptions, new AnthropicChatOptions()) +export class AnthropicChatOptions implements FunctionCallingOptions { + + model: string; + + @Expose({ name: 'max_tokens' }) + maxTokens: number; + + metadata: Metadata; + + @Expose({ name: 'stop_sequences' }) + stopSequences: string[]; + + @Expose({ name: 'temperature' }) + temperature: number; + + @Expose({ name: 'top_p' }) + topP: number; + + @Expose({ name: 'top_k' }) + topK: number; + + /** + * Tool Function Callbacks to register with the ChatModel. For Prompt + * Options the functionCallbacks are automatically enabled for the duration of the + * prompt execution. For Default Options the functionCallbacks are registered but + * disabled by default. Use the enableFunctions to set the functions from the registry + * to be used by the ChatModel chat completion requests. + */ + @Exclude() + functionCallbacks: FunctionCallback[] = []; + + /** + * List of functions, identified by their names, to configure for function calling in + * the chat completion requests. Functions with those names must exist in the + * functionCallbacks registry. The {@link #functionCallbacks} from the PromptOptions + * are automatically enabled for the duration of the prompt execution + * + * Note that function enabled with the default options are enabled for all chat + * completion requests. This could impact the token count and the billing. If the + * functions is set in a prompt options, then the enabled functions are only active + * for the duration of this prompt execution + */ + @Exclude() + functions: Set = new Set(); + @Exclude() + proxyToolCalls: boolean = false; + + @Exclude() + toolContext: Record = {}; + + static builder(): AnthropicChatOptionsBuilder { + return new AnthropicChatOptionsBuilder(); + } + + static fromOptions(fromOptions: AnthropicChatOptions): AnthropicChatOptions { + return AnthropicChatOptions.builder() + .model(fromOptions.model) + .maxTokens(fromOptions.maxTokens) + .metadata(fromOptions.metadata) + .stopSequences(fromOptions.stopSequences) + .temperature(fromOptions.temperature) + .topP(fromOptions.topP) + .topK(fromOptions.topK) + .functionCallbacks(fromOptions.functionCallbacks) + .functions(fromOptions.functions) + .proxyToolCalls(fromOptions.proxyToolCalls) + .toolContext(fromOptions.toolContext) + .build(); + } +} + +export class AnthropicChatOptionsBuilder { + + private options: AnthropicChatOptions = new AnthropicChatOptions(); + + model(model: string): AnthropicChatOptionsBuilder { + this.options.model = model; + return this; + } + + maxTokens(maxTokens: number): AnthropicChatOptionsBuilder { + this.options.maxTokens = maxTokens; + return this; + } + + metadata(metadata: Metadata): AnthropicChatOptionsBuilder { + this.options.metadata = metadata; + return this; + } + + stopSequences(stopSequences: string[]): AnthropicChatOptionsBuilder { + this.options.stopSequences = stopSequences; + return this; + } + + temperature(temperature: number): AnthropicChatOptionsBuilder { + this.options.temperature = temperature; + return this; + } + + topP(topP: number): AnthropicChatOptionsBuilder { + this.options.topP = topP; + return this; + } + + topK(topK: number): AnthropicChatOptionsBuilder { + this.options.topK = topK; + return this; + } + + functionCallbacks(functionCallbacks: FunctionCallback[]): AnthropicChatOptionsBuilder { + this.options.functionCallbacks = functionCallbacks; + return this; + } + + functions(functions: Set): AnthropicChatOptionsBuilder { + this.options.functions = functions; + return this; + } + + function(functionName: string): AnthropicChatOptionsBuilder { + this.options.functions.add(functionName); + return this; + } + + proxyToolCalls(proxyToolCalls: boolean): AnthropicChatOptionsBuilder { + this.options.proxyToolCalls = proxyToolCalls; + return this; + } + + toolContext(toolContext: Record): AnthropicChatOptionsBuilder { + if (this.options.toolContext === undefined) { + this.options.toolContext = toolContext; + } else { + this.options.toolContext = { ...this.options.toolContext, ...toolContext }; + } + return this; + } + + build(): AnthropicChatOptions { + return this.options; + } +} diff --git a/ai-packages/ai-anthropic/src/common/model/index.ts b/ai-packages/ai-anthropic/src/common/model/index.ts new file mode 100644 index 000000000..b32ae4903 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/model/index.ts @@ -0,0 +1,2 @@ +export * from './anthropic-chat-model'; +export * from './anthropic-chat-options'; diff --git a/ai-packages/ai-anthropic/src/common/module.ts b/ai-packages/ai-anthropic/src/common/module.ts new file mode 100644 index 000000000..f9e6e06cf --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/module.ts @@ -0,0 +1,4 @@ +import { autoBind } from '@celljs/core'; +import './index'; + +export default autoBind(); diff --git a/ai-packages/ai-anthropic/src/common/test/mock-anthropic-api.ts b/ai-packages/ai-anthropic/src/common/test/mock-anthropic-api.ts new file mode 100644 index 000000000..265b2bba5 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/test/mock-anthropic-api.ts @@ -0,0 +1,62 @@ +import { Component } from '@celljs/core'; +import { Observable, from } from 'rxjs'; +import { AnthropicAPI } from '../api/api-protocol'; +import { ChatRequest } from '../api/chat-request'; +import { ChatResponse } from '../api/chat-response'; +import { ContentBlockType } from '../api/content-block'; +import { Role } from '../api/message'; + +@Component(AnthropicAPI) +export class MockAnthropicAPI implements AnthropicAPI { + async chat(chatRequest: ChatRequest): Promise { + if (chatRequest.stream) { + throw new Error('Request must set the stream property to false.'); + } + + return new ChatResponse( + 'msg_123', + 'message', + Role.ASSISTANT, + [ + { + type: ContentBlockType.TEXT, + text: 'How can I assist you today?' + } + ], + chatRequest.model, + 'end_turn', + undefined, + { + inputTokens: 10, + outputTokens: 20 + } + ); + } + + async streamingChat(chatRequest: ChatRequest): Promise> { + if (!chatRequest.stream) { + throw new Error('Request must set the stream property to true.'); + } + + const response = new ChatResponse( + 'msg_123', + 'message', + Role.ASSISTANT, + [ + { + type: ContentBlockType.TEXT, + text: 'How can I assist you today?' + } + ], + chatRequest.model, + 'end_turn', + undefined, + { + inputTokens: 10, + outputTokens: 20 + } + ); + + return from([response]); + } +} diff --git a/ai-packages/ai-anthropic/src/common/test/test-container.ts b/ai-packages/ai-anthropic/src/common/test/test-container.ts new file mode 100644 index 000000000..e6d1c1c25 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/test/test-container.ts @@ -0,0 +1,12 @@ +import { coreTestModule } from '@celljs/core/lib/common/test/test-module'; +import httpModule from '@celljs/http/lib/common/module'; +import { autoBind, ContainerFactory, ContainerProvider } from '@celljs/core'; +import '../index'; + +const ollamaTestModule = autoBind(); + +export const createContainer = () => { + const container = ContainerFactory.create(coreTestModule, httpModule, ollamaTestModule); + ContainerProvider.set(container); + return container; +}; diff --git a/ai-packages/ai-anthropic/src/common/utils/index.ts b/ai-packages/ai-anthropic/src/common/utils/index.ts new file mode 100644 index 000000000..a3f4a5d48 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/utils/index.ts @@ -0,0 +1 @@ +export * from './stream-util'; diff --git a/ai-packages/ai-anthropic/src/common/utils/stream-util.ts b/ai-packages/ai-anthropic/src/common/utils/stream-util.ts new file mode 100644 index 000000000..c6b90d9d5 --- /dev/null +++ b/ai-packages/ai-anthropic/src/common/utils/stream-util.ts @@ -0,0 +1,142 @@ +import { Assert } from '@celljs/core'; +import { ChatResponse, ChatResponseBuilderReference } from '../api/chat-response'; +import { + ContentBlockDeltaEvent, + ContentBlockDeltaJson, + ContentBlockDeltaText, + ContentBlockStartEvent, + ContentBlockText, + ContentBlockToolUse, + EventType, + MessageDeltaEvent, + MessageStartEvent, + ToolUseAggregationEvent, + AnthropicEvent +} from '../api/event'; +import { ContentBlock, ContentBlockType } from '../api/content-block'; + +export abstract class StreamUtil { + static isToolUseStart(event: AnthropicEvent): boolean { + if (!event || !event.type || event.type !== EventType.CONTENT_BLOCK_START) { + return false; + } + return (event as ContentBlockStartEvent).contentBlock.type === ContentBlockType.TOOL_USE; + } + + static isToolUseFinish(event: AnthropicEvent): boolean { + if (!event || !event.type || event.type !== EventType.CONTENT_BLOCK_STOP) { + return false; + } + return true; + } + + static mergeToolUseEvents(previousEvent: AnthropicEvent, event: AnthropicEvent): AnthropicEvent { + if (!previousEvent || !event) { + return event; + } + + const eventAggregator = previousEvent as ToolUseAggregationEvent; + + if (event.type === EventType.CONTENT_BLOCK_START) { + const contentBlockStart = event as ContentBlockStartEvent; + + if (contentBlockStart.contentBlock.type === ContentBlockType.TOOL_USE) { + const cbToolUse = contentBlockStart.contentBlock as ContentBlockToolUse; + + eventAggregator.id = cbToolUse.id; + eventAggregator.name = cbToolUse.name; + eventAggregator.index = contentBlockStart.index; + eventAggregator.partialJson = ''; + } + } else if (event.type === EventType.CONTENT_BLOCK_DELTA) { + const contentBlockDelta = event as ContentBlockDeltaEvent; + Assert.isTrue( + contentBlockDelta.delta.type === ContentBlockType.INPUT_JSON_DELTA, + 'The json content block delta should have been aggregated. Unsupported content block type: ' + contentBlockDelta.delta.type + ); + + if (contentBlockDelta.delta.type === ContentBlockType.INPUT_JSON_DELTA) { + return eventAggregator.appendPartialJson((contentBlockDelta.delta as ContentBlockDeltaJson).partialJson); + } + } else if (event.type === EventType.CONTENT_BLOCK_STOP) { + if (!eventAggregator.isEmpty()) { + eventAggregator.squashIntoContentBlock(); + return eventAggregator; + } + } + + return event; + } + + static eventToChatResponse(event: AnthropicEvent): ChatResponse { + if (event.type === EventType.MESSAGE_START) { + ChatResponseBuilderReference.reset(); + const messageStartEvent = event as MessageStartEvent; + ChatResponseBuilderReference.get() + .withType(event.type) + .withId(messageStartEvent.message.id) + .withRole(messageStartEvent.message.role) + .withModel(messageStartEvent.message.model) + .withUsage(messageStartEvent.message.usage) + .withContent([]); + + } else if (event.type === EventType.TOOL_USE_AGGREGATE) { + const eventToolUseBuilder = event as ToolUseAggregationEvent; + + if (eventToolUseBuilder.toolContentBlocks && eventToolUseBuilder.toolContentBlocks.length > 0) { + const content = eventToolUseBuilder.toolContentBlocks.map( + toolToUse => new ContentBlock(ContentBlockType.TOOL_USE, undefined, undefined, undefined, toolToUse.id, toolToUse.name, toolToUse.input) + ); + ChatResponseBuilderReference.get().withContent(content); + + } + } else if (event.type === EventType.CONTENT_BLOCK_START) { + const contentBlockStartEvent = event as ContentBlockStartEvent; + + Assert.isTrue( + contentBlockStartEvent.contentBlock.type === ContentBlockType.TEXT, + 'The json content block should have been aggregated. Unsupported content block type: ' + contentBlockStartEvent.contentBlock.type + ); + + if (contentBlockStartEvent.contentBlock.type === ContentBlockType.TEXT) { + const contentBlockText = contentBlockStartEvent.contentBlock as ContentBlockText; + ChatResponseBuilderReference + .get() + .withType(event.type) + .withContent([new ContentBlock(ContentBlockType.TEXT, undefined, contentBlockText.text, contentBlockStartEvent.index)]); + } + } else if (event.type === EventType.CONTENT_BLOCK_DELTA) { + const contentBlockDeltaEvent = event as ContentBlockDeltaEvent; + + if (contentBlockDeltaEvent.delta.type === ContentBlockType.TEXT_DELTA) { + const deltaTxt = contentBlockDeltaEvent.delta as ContentBlockDeltaText; + ChatResponseBuilderReference + .get() + .withType(event.type) + .withContent([new ContentBlock(ContentBlockType.TEXT_DELTA, undefined, deltaTxt.text, contentBlockDeltaEvent.index)]); + } + } else if (event.type === EventType.MESSAGE_DELTA) { + ChatResponseBuilderReference.get().withType(event.type); + + const messageDeltaEvent = event as MessageDeltaEvent; + + if (messageDeltaEvent.delta.stopReason) { + ChatResponseBuilderReference.get().withStopReason(messageDeltaEvent.delta.stopReason); + } + + if (messageDeltaEvent.delta.stopSequence) { + ChatResponseBuilderReference.get().withStopSequence(messageDeltaEvent.delta.stopSequence); + } + + if (messageDeltaEvent.usage) { + ChatResponseBuilderReference.get().withOutputTokens(messageDeltaEvent.usage.outputTokens); + } + } else if (event.type === EventType.MESSAGE_STOP) { + // pass through + } else { + ChatResponseBuilderReference.get().withType(event.type).withContent([]); + } + + return ChatResponseBuilderReference.get().build(); + } +} diff --git a/ai-packages/ai-anthropic/src/package.spec.ts b/ai-packages/ai-anthropic/src/package.spec.ts new file mode 100644 index 000000000..98ccffbf9 --- /dev/null +++ b/ai-packages/ai-anthropic/src/package.spec.ts @@ -0,0 +1,4 @@ +describe('ai-authropic package', () => { + + it('support ai-authropic coverage statistics', () => {}); +}); diff --git a/ai-packages/ai-anthropic/tsconfig.json b/ai-packages/ai-anthropic/tsconfig.json new file mode 100644 index 000000000..dd53f24cd --- /dev/null +++ b/ai-packages/ai-anthropic/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "@celljs/component/configs/base.tsconfig.json", + "compilerOptions": { + "composite": true, + "rootDir": "src", + "outDir": "lib" + }, + "include": [ + "src" + ], + "references": [ + { + "path": "../../packages/core" + }, + { + "path": "../../packages/http" + }, + { + "path": "../ai-core" + } + ] +} diff --git a/ai-packages/ai-anthropic/typedoc.json b/ai-packages/ai-anthropic/typedoc.json new file mode 100644 index 000000000..7c070f7c2 --- /dev/null +++ b/ai-packages/ai-anthropic/typedoc.json @@ -0,0 +1,8 @@ +{ + "extends": [ + "../../configs/typedoc.base.jsonc" + ], + "entryPoints": [ + "src/common/index.ts" + ], +} \ No newline at end of file diff --git a/ai-packages/ai-core/src/common/chat/metadata/metadata-protocol.ts b/ai-packages/ai-core/src/common/chat/metadata/metadata-protocol.ts index 1d2912e7e..b2b34a612 100644 --- a/ai-packages/ai-core/src/common/chat/metadata/metadata-protocol.ts +++ b/ai-packages/ai-core/src/common/chat/metadata/metadata-protocol.ts @@ -26,6 +26,12 @@ export interface Usage { } +export namespace Usage { + export function from(promptTokens: number, generationTokens: number): Usage { + return { promptTokens, generationTokens, totalTokens: promptTokens + generationTokens }; + } +} + /** * Abstract Data Type (ADT) encapsulating metadata from an AI provider's API rate limits * granted to the API key in use and the API key's current balance. @@ -118,26 +124,92 @@ export namespace PromptMetadata { } export interface ChatResponseMetadata extends ResponseMetadata { - readonly id?: string; - readonly model?: string; + id?: string; + model?: string; /** * AI provider specific metadata on rate limits. * @see RateLimit */ - readonly rateLimit: RateLimit; + rateLimit: RateLimit; /** * AI provider specific metadata on API usage. * @see Usage */ - readonly usage: Usage; + usage: Usage; - readonly promptMetadata: PromptMetadata; + promptMetadata: PromptMetadata; + +} + +export class ChatResponseMetadataBuilder { + + private readonly chatResponseMetadata: ChatResponseMetadata = { + id: '', + model: '', + rateLimit: { + requestsLimit: 0, + requestsRemaining: 0, + requestsReset: 0, + tokensLimit: 0, + tokensRemaining: 0, + tokensReset: 0 + }, + usage: { + promptTokens: 0, + generationTokens: 0, + totalTokens: 0 + }, + promptMetadata: [] as PromptMetadata, + extra: {} + }; + + id(id: string): ChatResponseMetadataBuilder { + this.chatResponseMetadata.id = id; + return this; + } + + model(model: string): ChatResponseMetadataBuilder { + this.chatResponseMetadata.model = model; + return this; + } + + rateLimit(rateLimit: RateLimit): ChatResponseMetadataBuilder { + this.chatResponseMetadata.rateLimit = rateLimit; + return this; + } + + usage(usage: Usage): ChatResponseMetadataBuilder { + this.chatResponseMetadata.usage = usage; + return this; + } + + promptMetadata(promptMetadata: PromptMetadata): ChatResponseMetadataBuilder { + this.chatResponseMetadata.promptMetadata = promptMetadata; + return this; + } + + keyValue(key: string, value: any): ChatResponseMetadataBuilder { + if (!key) { + throw new IllegalArgumentError('Key must not be empty'); + } + if (value) { + this.chatResponseMetadata.extra[key] = value; + } + + return this; + } + + build(): ChatResponseMetadata { + return this.chatResponseMetadata; + } } export namespace ChatResponseMetadata { - export const EMPTY = { + export const EMPTY: ChatResponseMetadata = { + id: '', + model: '', rateLimit: { requestsLimit: 0, requestsRemaining: 0, @@ -151,6 +223,11 @@ export namespace ChatResponseMetadata { generationTokens: 0, totalTokens: 0 }, - promptMetadata: [] as PromptMetadata - } as ChatResponseMetadata; + promptMetadata: [] as PromptMetadata, + extra: {} + }; + + export function builder(): ChatResponseMetadataBuilder { + return new ChatResponseMetadataBuilder(); + } } diff --git a/ai-packages/ai-core/src/common/chat/model/chat-protocol.ts b/ai-packages/ai-core/src/common/chat/model/chat-protocol.ts index a7de38cd5..cc7a0a1a1 100644 --- a/ai-packages/ai-core/src/common/chat/model/chat-protocol.ts +++ b/ai-packages/ai-core/src/common/chat/model/chat-protocol.ts @@ -1,5 +1,6 @@ import { Model, ModelResponse, ModelResult, StreamingModel } from '../../model/model-protocol'; import { AssistantMessage } from '../message'; +import { ChatGenerationMetadata, ChatResponseMetadata } from '../metadata'; import { Prompt } from '../prompt/prompt-protocol'; export const StreamingChatModel = Symbol('StreamingChatModel'); @@ -16,6 +17,18 @@ export interface ChatResponse extends ModelResponse { } +export namespace Generation { + export function from(output: AssistantMessage, metadata = ChatGenerationMetadata.EMPTY): Generation { + return { output, metadata }; + } +} + +export namespace ChatResponse { + export function from(results: Generation[], metadata = ChatResponseMetadata.EMPTY): ChatResponse { + return { result: results[0], results, metadata }; + } +} + export interface StreamingChatModel extends StreamingModel { } diff --git a/ai-packages/ai-core/src/common/model/function/function-protocol.ts b/ai-packages/ai-core/src/common/model/function/function-protocol.ts index 6c88f61d6..5e4b644ed 100644 --- a/ai-packages/ai-core/src/common/model/function/function-protocol.ts +++ b/ai-packages/ai-core/src/common/model/function/function-protocol.ts @@ -52,6 +52,12 @@ export interface FunctionCallingOptions { functions: Set; } +export namespace FunctionCallingOptions { + export function is(obj: any): obj is FunctionCallingOptions { + return obj && obj.functionCallbacks !== undefined && obj.functions !== undefined; + } +} + export interface PortableFunctionCallingOptions extends FunctionCallingOptions, ChatOptions { } diff --git a/ai-packages/ai-core/src/common/sse/sse-decoder.ts b/ai-packages/ai-core/src/common/sse/sse-decoder.ts index 15d4a2ea3..0e5deae95 100644 --- a/ai-packages/ai-core/src/common/sse/sse-decoder.ts +++ b/ai-packages/ai-core/src/common/sse/sse-decoder.ts @@ -1,4 +1,4 @@ -import { EventDecoder, ServerSentEvent } from './sse-protocol'; +import { EventDecoder, StreamEvent } from './sse-protocol'; /** * Server-sent event. @@ -14,8 +14,8 @@ export class SSEDecoder implements EventDecoder { this.chunks = []; } - protected buildServerSentEvent(): ServerSentEvent { - const sse: ServerSentEvent = { + protected buildServerSentEvent(): StreamEvent { + const sse: StreamEvent = { event: this.event, data: this.data.join('\n'), raw: this.chunks, @@ -28,7 +28,7 @@ export class SSEDecoder implements EventDecoder { return sse; } - decode(line: string): ServerSentEvent | undefined { + decode(line: string): StreamEvent | undefined { if (line.endsWith('\r')) { line = line.substring(0, line.length - 1); } diff --git a/ai-packages/ai-core/src/common/sse/sse-protocol.ts b/ai-packages/ai-core/src/common/sse/sse-protocol.ts index 36ec44183..bc3ad89e0 100644 --- a/ai-packages/ai-core/src/common/sse/sse-protocol.ts +++ b/ai-packages/ai-core/src/common/sse/sse-protocol.ts @@ -3,9 +3,9 @@ import { Bytes } from '@celljs/core'; /** * Server-sent event. */ -export interface ServerSentEvent { +export interface StreamEvent { event?: string; - data: string; + data: Data; raw: string[]; }; @@ -13,7 +13,7 @@ export interface ServerSentEvent { * Decoder for server-sent events. */ export interface EventDecoder { - decode(line: string): ServerSentEvent | undefined; + decode(line: string): StreamEvent | undefined; } /** diff --git a/ai-packages/ai-core/src/common/utils/sse-util.ts b/ai-packages/ai-core/src/common/utils/sse-util.ts index e529be946..6e8421eba 100644 --- a/ai-packages/ai-core/src/common/utils/sse-util.ts +++ b/ai-packages/ai-core/src/common/utils/sse-util.ts @@ -1,5 +1,5 @@ import { Observable, Subscriber } from 'rxjs'; -import { ServerSentEvent, SSEDecoder, LineDecoderImpl } from '../sse'; +import { StreamEvent, SSEDecoder, LineDecoderImpl } from '../sse'; import { Bytes } from '@celljs/core'; /** @@ -7,8 +7,8 @@ import { Bytes } from '@celljs/core'; */ export class SSEUtil { - static toObservable(readableStream: ReadableStream, controller?: AbortController): Observable { - return new Observable(subscriber => { + static toObservable(readableStream: ReadableStream, controller?: AbortController): Observable> { + return new Observable>(subscriber => { (async () => { const decoder = new SSEDecoder(); const lineDecoder = new LineDecoderImpl(); @@ -48,7 +48,7 @@ export class SSEUtil { } - private static handleSSE(sse: ServerSentEvent | undefined, subscriber: Subscriber, controller?: AbortController): void { + private static handleSSE(sse: StreamEvent | undefined, subscriber: Subscriber>, controller?: AbortController): void { if (sse) { if (sse.data.startsWith('[DONE]')) { subscriber.complete(); @@ -72,7 +72,11 @@ export class SSEUtil { return; } - subscriber.next(data); + subscriber.next({ + event: sse.event, + data, + raw: sse.raw + }); if (data.done === true) { subscriber.complete(); diff --git a/ai-packages/ai-ollama/README.md b/ai-packages/ai-ollama/README.md index 7175580a0..7eec3386b 100644 --- a/ai-packages/ai-ollama/README.md +++ b/ai-packages/ai-ollama/README.md @@ -31,13 +31,13 @@ yarn add @celljs/ai-ollama ```typescript import { AssistantMessage, PromptTemplate } from '@celljs/ai-core'; -import { OllamChatModel, OllamModel, } from '@celljs/ai-ollama'; +import { OllamaChatModel, OllamaModel, } from '@celljs/ai-ollama'; import { Component Autowired } from '@celljs/core'; @Component() export class OllamaDemo { - @Autowired(OllamChatModel) - private ollamChatModel: OllamChatModel; + @Autowired(OllamaChatModel) + private ollamaChatModel: OllamaChatModel; @Autowired(OllamaEmbeddingModel) private ollamaEmbeddingModel: OllamaEmbeddingModel; @@ -52,11 +52,11 @@ export class OllamaDemo { const prompt = await this.promptTemplate.create( 'Hello {name}', { - chatOptions: { model: OllamModel.LLAMA3_2 }, + chatOptions: { model: OllamaModel.LLAMA3_2 }, variables: { name: 'Ollama' } } ); - const response = await this.ollamChatModel.call(prompt); + const response = await this.ollamaChatModel.call(prompt); console.log(response.result.output); } @@ -67,11 +67,11 @@ export class OllamaDemo { const prompt = await this.promptTemplate.create( 'Hello {name}', { - chatOptions: { model: OllamModel.LLAMA3_2 }, + chatOptions: { model: OllamaModel.LLAMA3_2 }, variables: { name: 'Ollama' } } ); - const response$ = await this.ollamChatModel.stream(prompt); + const response$ = await this.ollamaChatModel.stream(prompt); response$.subscribe({ next: response => console.log(response.result.output), complete: () => console.log('Chat completed!') @@ -84,7 +84,7 @@ export class OllamaDemo { async embed() { const response = await this.ollamaEmbeddingModel.call({ inputs: ['text to embed'], - options: { model: OllamModel.LLAMA3_2 } + options: { model: OllamaModel.LLAMA3_2 } }); console.log(response.result.embeddings); } diff --git a/ai-packages/ai-ollama/src/common/api/chat-request.ts b/ai-packages/ai-ollama/src/common/api/chat-request.ts index 7a59bac7d..31617f357 100644 --- a/ai-packages/ai-ollama/src/common/api/chat-request.ts +++ b/ai-packages/ai-ollama/src/common/api/chat-request.ts @@ -5,21 +5,22 @@ import { Assert } from '@celljs/core'; /** * Function definition. - * - * @param name The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes. - * @param description A description of what the function does, used by the model to choose when and how to call - * the function. - * @param parameters The parameters the functions accepts, described as a JSON Schema object. To describe a - * function that accepts no parameters, provide the value {"type": "object", "properties": {}}. */ export class FunctionDefinition { - @Expose() + /** + * The name of the function to be called. Must be a-z, A-Z, 0-9, or contain underscores and dashes. + */ name: string; - @Expose() + /** + * A description of what the function does, used by the model to choose when and how to call the function. + */ description: string; - @Expose() + /** + * The parameters the functions accepts, described as a JSON Schema object. To describe a function that + * accepts no parameters, provide the value {"type": "object", "properties": {}}. + */ parameters: Record; constructor(description: string, name: string, jsonSchema: string) { @@ -31,15 +32,16 @@ export class FunctionDefinition { /** * Represents a tool the model may call. Currently, only functions are supported as a tool. - * - * @param type The type of the tool. Currently, only 'function' is supported. - * @param function The function definition. */ export class Tool { - @Expose() + /** + * The type of the tool. Currently, only 'function' is supported. + */ type: ToolType; - @Expose() + /** + * The function definition. + */ @Type(() => FunctionDefinition) function: FunctionDefinition; @@ -59,40 +61,47 @@ export enum ToolType { /** * Chat request object. * - * @param model The model to use for completion. It should be a name familiar to Ollama from the [Library](https://ollama.com/library). - * @param messages The list of messages in the chat. This can be used to keep a chat memory. - * @param stream Whether to stream the response. If false, the response will be returned as a single response object rather than a stream of objects. - * @param format The format to return the response in. Currently, the only accepted value is "json". - * @param keepAlive Controls how long the model will stay loaded into memory following this request (default: 5m). - * @param tools List of tools the model has access to. - * @param options Model-specific options. For example, "temperature" can be set through this field, if the model supports it. - * You can use the `OllamaOptions` builder to create the options then `OllamaOptions.toMap()` to convert the options into a map. - * * @see [Chat Completion API](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-chat-completion) * @see [Ollama Types](https://github.com/ollama/ollama/blob/main/api/types.go) */ export class ChatRequest { - @Expose() + /** + * The model to use for completion. It should be a name familiar to Ollama from the [Library](https://ollama.com/library). + */ model: string; - @Expose() + /** + * The list of messages in the chat. This can be used to keep a chat memory. + */ @Type(() => Message) messages: Message[]; - @Expose() + /** + * Whether to stream the response. If false, the response will be returned as a single response object rather than a stream of objects. + */ stream: boolean; - @Expose() + /** + * The format to return the response in. Currently, the only accepted value is "json". + */ format: string; + /** + * Controls how long the model will stay loaded into memory following this request (default: 5m). + */ @Expose({ name: 'keep_alive' }) keepAlive: string; - @Expose() + /** + * List of tools the model has access to. + */ @Type(() => Tool) tools: Tool[]; - @Expose() + /** + * Model-specific options. For example, "temperature" can be set through this field, if the model supports it. + * You can use the `OllamaOptions` builder to create the options then `OllamaOptions.toMap()` to convert the options into a map. + */ options: Record; constructor( diff --git a/ai-packages/ai-ollama/src/common/api/chat-response.ts b/ai-packages/ai-ollama/src/common/api/chat-response.ts index 8d1849465..483c5bd76 100644 --- a/ai-packages/ai-ollama/src/common/api/chat-response.ts +++ b/ai-packages/ai-ollama/src/common/api/chat-response.ts @@ -4,57 +4,74 @@ import { Message } from './message'; /** * Ollama chat response object. * - * @param model The model used for generating the response. - * @param createdAt The timestamp of the response generation. - * @param message The response {@link Message} with {@link Message.Role#ASSISTANT}. - * @param doneReason The reason the model stopped generating text. - * @param done Whether this is the final response. For streaming response only the - * last message is marked as done. If true, this response may be followed by another - * response with the following, additional fields: context, prompt_eval_count, - * prompt_eval_duration, eval_count, eval_duration. - * @param totalDuration Time spent generating the response. - * @param loadDuration Time spent loading the model. - * @param promptEvalCount Number of tokens in the prompt. - * @param promptEvalDuration Time spent evaluating the prompt. - * @param evalCount Number of tokens in the response. - * @param evalDuration Time spent generating the response. - * * @see [Chat Completion API](https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-chat-completion) * @see [Ollama Types](https://github.com/ollama/ollama/blob/main/api/types.go) */ export class ChatResponse { - @Expose() + /** + * The model used for generating the response. + */ model: string; + /** + * The timestamp of the response generation. + */ @Expose({ name: 'created_at' }) @Type(() => Date) createdAt: Date; - @Expose() + /** + * The response {@link Message} with {@link Message.Role#ASSISTANT}. + */ @Type(() => Message) message: Message; + /** + * The reason the model stopped generating text. + */ @Expose({ name: 'done_reason' }) doneReason: string; - @Expose() + /** + * Whether this is the final response. For streaming response only the last message is marked as done. + * If true, this response may be followed by another response with the following, + * additional fields: context, prompt_eval_count, prompt_eval_duration, eval_count, eval_duration. + */ done: boolean; + /** + * Time spent generating the response. + */ @Expose({ name: 'total_duration' }) totalDuration: number; + /** + * Time spent loading the model. + */ @Expose({ name: 'load_duration' }) loadDuration: number; + /** + * Number of tokens in the prompt. + */ @Expose({ name: 'prompt_eval_count' }) promptEvalCount?: number; + /** + * Time spent evaluating the prompt. + */ @Expose({ name: 'prompt_eval_duration' }) promptEvalDuration: number; + /** + * Number of tokens in the response. + */ @Expose({ name: 'eval_count' }) evalCount?: number; + /** + * Time spent generating the response. + */ @Expose({ name: 'eval_duration' }) evalDuration: number; diff --git a/ai-packages/ai-ollama/src/common/api/embeddings-request.ts b/ai-packages/ai-ollama/src/common/api/embeddings-request.ts index 538e3096a..727587ff2 100644 --- a/ai-packages/ai-ollama/src/common/api/embeddings-request.ts +++ b/ai-packages/ai-ollama/src/common/api/embeddings-request.ts @@ -2,28 +2,33 @@ import { Expose } from 'class-transformer'; /** * Generate embeddings from a model. - * - * @param model The name of model to generate embeddings from. - * @param input The text or list of text to generate embeddings for. - * @param keepAlive Controls how long the model will stay loaded into memory following the request (default: 5m). - * @param options Additional model parameters listed in the documentation for the - * @param truncate Truncates the end of each input to fit within context length. - * Returns error if false and context length is exceeded. Defaults to true. */ export class EmbeddingsRequest { - @Expose() + /** + * The name of model to generate embeddings from. + */ model: string; - @Expose() + /** + * The text or list of text to generate embeddings for. + */ input: string[]; + /** + * Controls how long the model will stay loaded into memory following the request (default: 5m). + */ @Expose({ name: 'keep_alive' }) keepAlive?: string; - @Expose() + /** + * Additional model parameters listed in the documentation for the + */ options?: Record; - @Expose() + /** + * Truncates the end of each input to fit within context length. + * Returns error if false and context length is exceeded. Defaults to true. + */ truncate?: boolean; constructor( diff --git a/ai-packages/ai-ollama/src/common/api/embeddings-response.ts b/ai-packages/ai-ollama/src/common/api/embeddings-response.ts index fa2861b4e..90d887ed7 100644 --- a/ai-packages/ai-ollama/src/common/api/embeddings-response.ts +++ b/ai-packages/ai-ollama/src/common/api/embeddings-response.ts @@ -1,16 +1,16 @@ -import { Expose } from 'class-transformer'; - /** * The response object returned from the /embedding endpoint. - * @param model The model used for generating the embeddings. - * @param embeddings The list of embeddings generated from the model. - * Each embedding (list of doubles) corresponds to a single input text. */ export class EmbeddingsResponse { - @Expose() + /** + * The model used for generating the embeddings. + */ model: string; - @Expose() + /** + * The list of embeddings generated from the model. + * Each embedding (list of doubles) corresponds to a single input text. + */ embeddings: number[][]; constructor( diff --git a/ai-packages/ai-ollama/src/common/api/ollama-api.ts b/ai-packages/ai-ollama/src/common/api/ollama-api.ts index 65f935bb4..f5b65564e 100644 --- a/ai-packages/ai-ollama/src/common/api/ollama-api.ts +++ b/ai-packages/ai-ollama/src/common/api/ollama-api.ts @@ -41,7 +41,7 @@ export class OllamaAPIImpl implements OllamaAPI { } ); return SSEUtil.toObservable(data).pipe( - map(item => plainToInstance(ChatResponse, item)) + map(item => plainToInstance(ChatResponse, item.data)) ); } diff --git a/ai-packages/ai-ollama/src/common/api/ollama-options.ts b/ai-packages/ai-ollama/src/common/api/ollama-options.ts index 672455253..aa4f96e49 100644 --- a/ai-packages/ai-ollama/src/common/api/ollama-options.ts +++ b/ai-packages/ai-ollama/src/common/api/ollama-options.ts @@ -114,7 +114,6 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * specific number will make the model generate the same text for the same prompt. * (Default: -1) */ - @Expose({ name: 'seed' }) seed?: number; /** @@ -165,7 +164,6 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * The temperature of the model. Increasing the temperature will * make the model answer more creatively. (Default: 0.8) */ - @Expose({ name: 'temperature' }) temperature?: number; /** @@ -192,7 +190,6 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * Enable Mirostat sampling for controlling perplexity. (default: 0, 0 * = disabled, 1 = Mirostat, 2 = Mirostat 2.0) */ - @Expose({ name: 'mirostat' }) mirostat?: number; /** @@ -221,7 +218,6 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * LLM will stop generating text and return. Multiple stop patterns may be set by * specifying multiple separate stop parameters in a modelfile. */ - @Expose({ name: 'stop' }) @Type(() => String) stop?: string[]; @@ -230,14 +226,12 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * Used to allow overriding the model name with prompt options. * Part of Chat completion parameters. */ - @Expose({ name: 'model' }) model?: string; /** * Sets the desired format of output from the LLM. The only valid values are null or "json". * Part of Chat completion advanced parameters. */ - @Expose({ name: 'format' }) format?: string; /** @@ -252,7 +246,6 @@ export class OllamaOptions implements FunctionCallingOptions, ChatOptions, Embed * Truncates the end of each input to fit within context length. Returns error if false and context length is exceeded. * Defaults to true. */ - @Expose({ name: 'truncate' }) truncate?: boolean; /** diff --git a/tsconfig.json b/tsconfig.json index fd79698d6..b30cad7e8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,6 +6,9 @@ }, "include": [], "references": [ + { + "path": "ai-packages/ai-anthropic" + }, { "path": "ai-packages/ai-core" },