Skip to content

Commit

Permalink
fix: tools
Browse files Browse the repository at this point in the history
  • Loading branch information
@veasion
veasion committed Dec 20, 2024
1 parent 2187f0e commit 65f593d
Show file tree
Hide file tree
Showing 6 changed files with 413 additions and 76 deletions.
83 changes: 8 additions & 75 deletions src/background/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,84 +1,17 @@
import { Eko } from "ekoai";
import { ExecutionContext } from "ekoai/types";
import { tools, utils } from "ekoai/extension";

let eko = new Eko("claude-3-5-sonnet-20241022");
let context = {
variables: new Map<string, unknown>(),
tools: [],
} as any as ExecutionContext;

async function testWebSearch() {
let webSearch = new tools.WebSearch();
let result = await webSearch.execute(context, { query: "谢扬" });
console.log("result: ", result);
return result;
}

async function testTabManagement(action: string) {
let tabManagement = new tools.TabManagement();
let result = await tabManagement.execute(context, { action });
console.log("result: ", action, result);
return result;
}

async function exportFile() {
let exportFile = new tools.ExportFile();
let result = await exportFile.execute(context, {
fileType: "csv",
content: "Name, Price\niPhone, $1000\nnotebook, $1.02",
});
console.log("result: ", result);
return result;
}

async function extractContent() {
let extract = new tools.ExtractContent();
let result = await extract.execute(context, {});
console.log("result: ", result);
return result;
}

async function openUrl(url: string, newWindow: boolean) {
let openUrl = new tools.OpenUrl();
let result = await openUrl.execute(context, { url, newWindow });
console.log("result: ", result);
return result;
}

async function screenshot() {
let screenshot = new tools.Screenshot();
let result = await screenshot.execute(context, {});
console.log("result: ", result);
return result;
}

async function computerWeb(
action: string,
coordinate?: [number, number],
text?: string
) {
let tabId = await utils.getCurrentTabId();
context.variables.set("tabId", tabId);
let pageSize = await utils.getPageSize(tabId);
let computer = new tools.ComputerWeb(pageSize);
let result = await computer.execute(context, { action, coordinate, text });
console.log("result: ", result);
return result;
}
import { testTools } from "./test_tools";
import { testWebSearchWithLLM } from "./test_llm_search";
import { testWebSearchWithComputer } from "./test_llm_computer";
import { testWebSearchWithWorkflow } from "./test_llm_workflow";

chrome.runtime.onMessage.addListener(async function (
request,
sender,
sendResponse
) {
if (request.type == "run") {
await testWebSearch();
// await testTabManagement("current_tab");
// await testTabManagement("tab_all");
// await exportFile();
// await extractContent();
// await openUrl('https://www.google.com', true);
// await screenshot();
// await testTools();
await testWebSearchWithLLM();
// await testWebSearchWithComputer();
// await testWebSearchWithWorkflow();
}
});
143 changes: 143 additions & 0 deletions src/background/test_llm_computer.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
import { ClaudeProvider } from "ekoai";
import {
ExecutionContext,
Message,
LLMParameters,
ToolDefinition,
Tool,
} from "ekoai/types";
import { tools, getLLMConfig } from "ekoai/extension";

export async function testWebSearchWithComputer() {
let apiKey = (await getLLMConfig()).apiKey;
if (!apiKey) {
throw Error("Please configure apiKey");
}
let llmProvider = new ClaudeProvider(apiKey);
let context = {
variables: new Map<string, unknown>(),
tools: [],
} as any as ExecutionContext;

let messages: Message[] = [
{
role: "user",
content: "搜索谢扬信息，汇总成表格导出",
},
];
let params: LLMParameters = {
maxTokens: 4096,
toolChoice: {
type: "auto",
},
tools: [
new tools.TabManagement() as ToolDefinition,
new tools.OpenUrl() as ToolDefinition,
new tools.ComputerWeb() as ToolDefinition,
new tools.ExportFile() as ToolDefinition,
new tools.ExtractContent() as ToolDefinition,
],
};
let toolMap: { [key: string]: Tool } = {};
for (let i = 0; i < params.tools.length; i++) {
let tool = params.tools[i];
toolMap[tool.name] = tool as Tool;
}
do {
printLog(messages[messages.length - 1]);
console.log("Requesting...");
let result = await llmProvider.generateText(messages, params);
messages.push({
role: "assistant",
content: result.content,
});
printLog(messages[messages.length - 1]);
let toolCalls = result.toolCalls;
if (!toolCalls || toolCalls.length == 0) {
break;
}
let user_content = [];
for (let i = 0; i < toolCalls.length; i++) {
let toolCall = toolCalls[i];
let tool = toolMap[toolCall.name];
let result = (await tool.execute(context, toolCall.input)) as any;
if (result.image && result.image.type == "base64") {
user_content.push({
type: "tool_result",
tool_use_id: toolCall.id,
content: [
{
type: "image",
source: result.image,
},
],
});
} else {
user_content.push({
type: "tool_result",
tool_use_id: toolCall.id,
content: [
{
type: "text",
text: JSON.stringify(result),
},
],
});
}
}
messages.push({
role: "user",
content: user_content,
});
} while (true);
}

function printLog(message: Message) {
let sb = message.role + ":\n";
let content = message.content;
if (typeof content == "string") {
sb += content;
} else {
for (let i = 0; i < content.length; i++) {
let _content = content[i] as any;
if (_content.type == "text") {
sb += _content.text + "\n";
} else if (_content.type == "tool_use") {
sb +=
"tool_use: [" +
_content.name +
"] > " +
JSON.stringify(_content.input) +
"\n";
} else if (_content.type == "tool_result") {
let tool_content = [];
if (typeof _content.content == "string") {
tool_content.push({ type: "text", text: _content.content });
} else {
tool_content = _content.content;
}
sb += "tool_result: ";
for (let j = 0; j < tool_content.length; j++) {
let __content = tool_content[j];
if (__content.type == "text") {
sb += "\n" + __content.text;
} else {
let source = __content.source;
sb +=
__content.type +
" > data:" +
source.media_type +
";" +
source.type +
"," +
source.data.substring(0, 10) +
"...(" +
source.data.length +
")";
}
}
}
}
}
console.log(sb.trim());
}
140 changes: 140 additions & 0 deletions src/background/test_llm_search.ts
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import { ClaudeProvider } from "ekoai";
import {
ExecutionContext,
Message,
LLMParameters,
ToolDefinition,
Tool,
} from "ekoai/types";
import { tools, getLLMConfig } from "ekoai/extension";

export async function testWebSearchWithLLM() {
let apiKey = (await getLLMConfig()).apiKey;
if (!apiKey) {
throw Error("Please configure apiKey");
}
let llmProvider = new ClaudeProvider(apiKey);
let context = {
variables: new Map<string, unknown>(),
tools: [],
} as any as ExecutionContext;

let messages: Message[] = [
{
role: "user",
content: "搜索谢扬信息，汇总成表格导出",
},
];
let params: LLMParameters = {
maxTokens: 4096,
toolChoice: {
type: "auto",
},
tools: [
new tools.WebSearch() as ToolDefinition,
new tools.ExportFile() as ToolDefinition,
],
};
let toolMap: { [key: string]: Tool } = {};
for (let i = 0; i < params.tools.length; i++) {
let tool = params.tools[i];
toolMap[tool.name] = tool as Tool;
}
do {
printLog(messages[messages.length - 1]);
console.log("Requesting...");
let result = await llmProvider.generateText(messages, params);
messages.push({
role: "assistant",
content: result.content,
});
printLog(messages[messages.length - 1]);
let toolCalls = result.toolCalls;
if (!toolCalls || toolCalls.length == 0) {
break;
}
let user_content = [];
for (let i = 0; i < toolCalls.length; i++) {
let toolCall = toolCalls[i];
let tool = toolMap[toolCall.name];
let result = (await tool.execute(context, toolCall.input)) as any;
if (result.image && result.image.type == "base64") {
user_content.push({
type: "tool_result",
tool_use_id: toolCall.id,
content: [
{
type: "image",
source: result.image,
},
],
});
} else {
user_content.push({
type: "tool_result",
tool_use_id: toolCall.id,
content: [
{
type: "text",
text: JSON.stringify(result),
},
],
});
}
}
messages.push({
role: "user",
content: user_content,
});
} while (true);
}

function printLog(message: Message) {
let sb = message.role + ":\n";
let content = message.content;
if (typeof content == "string") {
sb += content;
} else {
for (let i = 0; i < content.length; i++) {
let _content = content[i] as any;
if (_content.type == "text") {
sb += _content.text + "\n";
} else if (_content.type == "tool_use") {
sb +=
"tool_use: [" +
_content.name +
"] > " +
JSON.stringify(_content.input) +
"\n";
} else if (_content.type == "tool_result") {
let tool_content = [];
if (typeof _content.content == "string") {
tool_content.push({ type: "text", text: _content.content });
} else {
tool_content = _content.content;
}
sb += "tool_result: ";
for (let j = 0; j < tool_content.length; j++) {
let __content = tool_content[j];
if (__content.type == "text") {
sb += "\n" + __content.text;
} else {
let source = __content.source;
sb +=
__content.type +
" > data:" +
source.media_type +
";" +
source.type +
"," +
source.data.substring(0, 10) +
"...(" +
source.data.length +
")";
}
}
}
}
}
console.log(sb.trim());
}
Loading

0 comments on commit 65f593d

Please sign in to comment.