diff --git a/README.md b/README.md index 4fa30df..202b94e 100644 --- a/README.md +++ b/README.md @@ -7,5 +7,4 @@ Eko Keeps Operating - Empowering language to transform human words into action - npm install npm run build - ``` \ No newline at end of file diff --git a/public/manifest.json b/public/manifest.json index 4ee0513..4a2efac 100644 --- a/public/manifest.json +++ b/public/manifest.json @@ -7,8 +7,12 @@ "type": "module", "service_worker": "js/background.js" }, + "icons": { + "16": "icon.png", + "48": "icon.png", + "128": "icon.png" + }, "action": { - "default_icon": "icon.png", "default_popup": "popup.html" }, "options_ui": { diff --git a/src/background/tools/computer_use.ts b/src/background/tools/computer_use.ts new file mode 100644 index 0000000..af4e3be --- /dev/null +++ b/src/background/tools/computer_use.ts @@ -0,0 +1,295 @@ +import { Tool, InputSchema } from "../../types/action.types"; +import * as utils from "../utils"; + +/** + * Computer Use + */ +export class ComputerUse implements Tool { + name: string; + description: string; + input_schema: InputSchema; + windowId?: number; + tabId?: number; + + constructor(size: [number, number]) { + this.name = "computer_use"; + this.description = `Use a mouse and keyboard to interact with a computer, and take screenshots. +* This is a browser GUI interface where you do not have access to the address bar or bookmarks. You must operate the browser using inputs like screenshots, mouse, keyboard, etc. +* Some operations may take time to process, so you may need to wait and take successive screenshots to see the results of your actions. E.g. if you clicked submit button, but it didn't work, try taking another screenshot. +* The screen's resolution is ${size[0]}x${size[1]}. +* Whenever you intend to move the cursor to click on an element, you should consult a screenshot to determine the coordinates of the element before moving the cursor. +* If you tried clicking on a button or link but it failed to load, even after waiting, try adjusting your cursor position so that the tip of the cursor visually falls on the element that you want to click. +* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element.`; + this.input_schema = { + type: "object", + properties: { + action: { + type: "string", + description: `The action to perform. The available actions are: +* \`key\`: Press a key or key-combination on the keyboard. +- This supports pyautogui hotkey syntax. +- Multiple keys are combined using the "+" symbol. +- Examples: "a", "enter", "ctrl+s", "command+shift+a", "num0". +* \`type\`: Type a string of text on the keyboard. +* \`cursor_position\`: Get the current (x, y) pixel coordinate of the cursor on the screen. +* \`mouse_move\`: Move the cursor to a specified (x, y) pixel coordinate on the screen. +* \`left_click\`: Click the left mouse button. +* \`left_click_drag\`: Click and drag the cursor to a specified (x, y) pixel coordinate on the screen. +* \`right_click\`: Click the right mouse button. +* \`double_click\`: Double-click the left mouse button. +* \`screenshot\`: Take a screenshot of the screen. +* \`scroll\`: Performs a scroll of the mouse scroll wheel, The coordinate parameter is ineffective, each time a scroll operation is performed.`, + enum: [ + "key", + "type", + "mouse_move", + "left_click", + "left_click_drag", + "right_click", + "double_click", + "screenshot", + "cursor_position", + "scroll", + ], + }, + coordinate: { + type: "array", + description: + "(x, y): The x (pixels from the left edge) and y (pixels from the top edge) coordinates to move the mouse to.", + }, + text: { + type: "string", + description: "Required only by `action=type` and `action=key`", + }, + }, + required: ["action"], + }; + } + + /** + * computer + * + * @param {*} params { action: 'mouse_move', coordinate: [100, 200] } + * @returns { success: true, coordinate?: [], image?: { type: 'base64', media_type: 'image/jpeg', data: '/9j...' } } + */ + async execute(params: unknown): Promise { + if ( + typeof params !== "object" || + params === null || + !("action" in params) + ) { + throw new Error( + 'Invalid parameters. Expected an object with a "action" property.' + ); + } + let { action, coordinate, text } = params as any; + let tabId = await this.getTabId(); + let result; + switch (action as string) { + case "key": + result = await key(tabId, text, coordinate); + break; + case "type": + result = await type(tabId, text, coordinate); + break; + case "mouse_move": + result = await mouse_move(tabId, coordinate); + break; + case "left_click": + result = await left_click(tabId, coordinate); + break; + case "left_click_drag": + result = await left_click_drag(tabId, coordinate); + break; + case "right_click": + result = await right_click(tabId, coordinate); + break; + case "double_click": + result = await double_click(tabId, coordinate); + break; + case "screenshot": + result = await screenshot(this.windowId); + break; + case "cursor_position": + result = await cursor_position(tabId); + break; + case "scroll": + result = await scroll(tabId, coordinate); + break; + default: + throw Error( + `Invalid parameters. The "${action}" value is not included in the "action" enumeration.` + ); + } + return { success: true, ...result }; + } + + async getTabId(): Promise { + let tabId = this.tabId; + if (!tabId) { + tabId = await utils.getCurrentTabId(); + } + return tabId as number; + } +} + +export async function key( + tabId: number, + key: string, + coordinate?: [number, number] +) { + if (!coordinate) { + coordinate = (await cursor_position(tabId)).coordinate; + } + await mouse_move(tabId, coordinate); + let mapping: { [key: string]: string } = { + space: " ", + escape: "esc", + return: "enter", + page_up: "pageup", + page_down: "pagedown", + back_space: "backspace", + }; + let keys = key.replace(/\s+/g, " ").split(" "); + for (let i = 0; i < keys.length; i++) { + let _key = keys[i]; + if (_key.indexOf("+") > -1) { + let mapped_keys = _key.split("+").map((k) => mapping[k] || k); + await runComputeruseCommand("hotkey", mapped_keys); + } else { + let mapped_key = mapping[_key] || _key; + await runComputeruseCommand("press", [mapped_key]); + } + await utils.sleep(100); + } +} + +export async function type( + tabId: number, + text: string, + coordinate?: [number, number] +) { + if (coordinate) { + await mouse_move(tabId, coordinate); + } + await runComputeruseCommand("write", [text]); +} + +export async function mouse_move(tabId: number, coordinate: [number, number]) { + await runComputeruseCommand("moveTo", coordinate); +} + +export async function left_click(tabId: number, coordinate?: [number, number]) { + if (!coordinate) { + coordinate = (await cursor_position(tabId)).coordinate; + } + await runComputeruseCommand("click", [ + coordinate[0], + coordinate[1], + 1, + 0, + "left", + ]); +} + +export async function left_click_drag( + tabId: number, + coordinate: [number, number] +) { + await runComputeruseCommand("dragTo", [coordinate[0], coordinate[1], 0]); +} + +export async function right_click( + tabId: number, + coordinate?: [number, number] +) { + if (!coordinate) { + coordinate = (await cursor_position(tabId)).coordinate; + } + await runComputeruseCommand("click", [ + coordinate[0], + coordinate[1], + 1, + 0, + "right", + ]); +} + +export async function double_click( + tabId: number, + coordinate?: [number, number] +) { + if (!coordinate) { + coordinate = (await cursor_position(tabId)).coordinate; + } + await runComputeruseCommand("click", [ + coordinate[0], + coordinate[1], + 2, + 0, + "left", + ]); +} + +export async function screenshot(windowId?: number): Promise<{ + image: { + type: "base64"; + media_type: "image/png" | "image/jpeg"; + data: string; + }; +}> { + let screenshot = (await runComputeruseCommand("screenshot")).result; + let dataUrl = screenshot.startsWith("data:") + ? screenshot + : "data:image/png;base64," + screenshot; + /* + if (!windowId) { + const window = await chrome.windows.getCurrent(); + windowId = window.id; + } + let dataUrl = await chrome.tabs.captureVisibleTab(windowId as number, { + format: "jpeg", // jpeg / png + quality: 80, // 0-100 + }); + */ + let data = dataUrl.substring(dataUrl.indexOf("base64,") + 7); + return { + image: { + type: "base64", + media_type: dataUrl.indexOf("png") > -1 ? "image/png" : "image/jpeg", + data: data, + }, + }; +} + +export async function cursor_position(tabId: number): Promise<{ + coordinate: [number, number]; +}> { + /* + let result: any = await chrome.tabs.sendMessage(tabId, { + type: "computer:cursor_position", + }); + return { coordinate: result.coordinate as [number, number] }; + */ + let response = await runComputeruseCommand("position"); + return response.result; +} + +export async function size(): Promise<[number, number]> { + let response = await runComputeruseCommand("size"); + return response.result; +} + +export async function scroll(tabId: number, coordinate?: [number, number]) { + await runComputeruseCommand("scroll", [2]); +} + +export async function runComputeruseCommand( + func: string, + args?: Array +): Promise<{ result: any }> { + return (await chrome.runtime.sendMessage("gcgnhflnikdhbgielkialpbfcdpifcml", { + func, + args, + })) as any as { result: any }; +} diff --git a/src/background/tools/screenshot.ts b/src/background/tools/screenshot.ts deleted file mode 100644 index e69de29..0000000 diff --git a/src/background/tools/web_search.ts b/src/background/tools/web_search.ts index 4316713..b6c9db6 100644 --- a/src/background/tools/web_search.ts +++ b/src/background/tools/web_search.ts @@ -20,7 +20,7 @@ export class WebSearch implements Tool { description: "search for keywords", }, maxResults: { - type: "number", + type: "integer", description: "Maximum search results, default 5", }, }, diff --git a/src/background/utils.ts b/src/background/utils.ts index 2e207ec..b26e775 100644 --- a/src/background/utils.ts +++ b/src/background/utils.ts @@ -9,6 +9,25 @@ export function getCurrentTabId(): Promise { }); } +export async function getPageSize(): Promise<{ width: number; height: number }> { + let tabId = await getCurrentTabId(); + let injectionResult = await chrome.scripting.executeScript({ + target: { tabId: tabId as number }, + func: () => [ + window.innerWidth || + document.documentElement.clientWidth || + document.body.clientWidth, + window.innerHeight || + document.documentElement.clientHeight || + document.body.clientHeight, + ], + }); + return { + width: injectionResult[0].result[0] as number, + height: injectionResult[0].result[1] as number, + }; +} + export function sleep(time: number): Promise { return new Promise((resolve) => setTimeout(() => resolve(), time)); } diff --git a/src/content/index.ts b/src/content/index.ts new file mode 100644 index 0000000..653f12c --- /dev/null +++ b/src/content/index.ts @@ -0,0 +1,32 @@ +declare const eko: any; + +chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { + (async () => { + try { + switch (request.type) { + case "page:getDetailLinks": { + let result = await eko.getDetailLinks(request.search); + sendResponse(result); + break; + } + case "page:getContent": { + let result = await eko.getContent(request.search); + sendResponse(result); + break; + } + case "computer:cursor_position": { + sendResponse({ coordinate: [eko.lastMouseX, eko.lastMouseY] }); + break; + } + } + } catch (e) { + console.log("onMessage error", e); + } + })(); + return true; +}); + +document.addEventListener("mousemove", (event) => { + eko.lastMouseX = event.clientX; + eko.lastMouseY = event.clientY; +}); diff --git a/src/content/index.tsx b/src/content/index.tsx deleted file mode 100644 index 91051d8..0000000 --- a/src/content/index.tsx +++ /dev/null @@ -1,23 +0,0 @@ -declare const eko: any; - -chrome.runtime.onMessage.addListener(function (request, sender, sendResponse) { - (async () => { - try { - switch(request.type) { - case 'page:getDetailLinks': { - let result = await eko.getDetailLinks(request.search) - sendResponse(result) - break - } - case 'page:getContent': { - let result = await eko.getContent(request.search) - sendResponse(result) - break - } - } - } catch (e) { - console.log('onMessage error', e) - } - })() - return true -}) diff --git a/src/popup/index.tsx b/src/popup/index.tsx index 700566f..3359527 100644 --- a/src/popup/index.tsx +++ b/src/popup/index.tsx @@ -1,51 +1,180 @@ -import React, { useEffect, useState } from "react"; import { createRoot } from "react-dom/client"; +import React, { useState, useEffect } from "react"; +import { + Layout, + Button, + Table, + Modal, + Form, + Input, + message, + Space, +} from "antd"; +import { PlusOutlined } from "@ant-design/icons"; +import { ColumnsType } from "antd/es/table"; -const Popup = () => { - const [count, setCount] = useState(0); - const [currentURL, setCurrentURL] = useState(); +interface UserScript { + id: number; + name: string; + code: string; + enabled: boolean; +} - useEffect(() => { - chrome.action.setBadgeText({ text: count.toString() }); - }, [count]); +interface ScriptFormValues { + name: string; + code: string; +} + +const { Header, Content } = Layout; + +const ScriptManager: React.FC = () => { + const [scripts, setScripts] = useState([]); + const [isAddModalVisible, setIsAddModalVisible] = useState(false); + const [currentScript, setCurrentScript] = useState(null); + const [form] = Form.useForm(); useEffect(() => { - chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { - setCurrentURL(tabs[0].url); + chrome.storage.sync.get(["userScripts"], (result) => { + if (result.userScripts) { + setScripts(result.userScripts); + } }); }, []); - const changeBackground = () => { - chrome.tabs.query({ active: true, currentWindow: true }, function (tabs) { - const tab = tabs[0]; - if (tab.id) { - chrome.tabs.sendMessage( - tab.id, - { - color: "#555555", - }, - (msg) => { - console.log("result message:", msg); - } - ); - } + const saveScripts = (updatedScripts: UserScript[]) => { + chrome.storage.sync.set({ userScripts: updatedScripts }, () => { + setScripts(updatedScripts); + message.success("Save Success!"); + }); + }; + + const handleAddScript = (values: ScriptFormValues) => { + const newScript: UserScript = { + id: Date.now(), + name: values.name, + code: values.code, + enabled: true, + }; + const updatedScripts = [...scripts, newScript]; + saveScripts(updatedScripts); + setIsAddModalVisible(false); + form.resetFields(); + }; + + const handleEditScript = (script: UserScript) => { + setCurrentScript(script); + form.setFieldsValue({ + name: script.name, + code: script.code, }); + setIsAddModalVisible(true); }; + const handleUpdateScript = (values: ScriptFormValues) => { + if (!currentScript) return; + + const updatedScripts = scripts.map((script) => + script.id === currentScript.id + ? { ...script, name: values.name, code: values.code } + : script + ); + saveScripts(updatedScripts); + setIsAddModalVisible(false); + setCurrentScript(null); + form.resetFields(); + }; + + const handleDeleteScript = (scriptId: number) => { + const updatedScripts = scripts.filter((script) => script.id !== scriptId); + saveScripts(updatedScripts); + }; + + const columns: ColumnsType = [ + { + title: "DSL Name", + dataIndex: "name", + key: "name", + align: 'center' + }, + { + title: "Actions", + key: "actions", + align: 'center', + width: 100, + render: (text, record: UserScript) => ( + + alert("eko")}>Run + handleEditScript(record)}>Edit + handleDeleteScript(record.id)}>Delete + + ), + }, + ]; + return ( - <> -
    -
  • Current URL: {currentURL}
    • -
  • Current Time: {new Date().toLocaleTimeString()}
    • -
- - - +

Eko DSL Manager

+ + + + + + + + + + { + setIsAddModalVisible(false); + setCurrentScript(null); + form.resetFields(); + }} + onOk={() => form.submit()} + > +
+ + + + + + + + + + ); }; @@ -53,6 +182,6 @@ const root = createRoot(document.getElementById("root")!); root.render( - + ); diff --git a/src/types/action.types.ts b/src/types/action.types.ts index 3f56c8e..db5f368 100644 --- a/src/types/action.types.ts +++ b/src/types/action.types.ts @@ -16,6 +16,9 @@ export interface Tool { } export interface Propertie { - type: 'string' | 'number' | 'bool'; - description?: string + type: 'string' | 'integer' | 'boolean' | 'array' | 'object'; + description?: string; + items?: InputSchema; + enum?: Array; + properties?: Properties; } diff --git a/webpack/webpack.common.js b/webpack/webpack.common.js index 4456ae5..668f3e2 100644 --- a/webpack/webpack.common.js +++ b/webpack/webpack.common.js @@ -8,7 +8,7 @@ module.exports = { popup: path.join(srcDir, 'popup/index.tsx'), options: path.join(srcDir, 'options/index.tsx'), background: path.join(srcDir, 'background/index.ts'), - content_script: path.join(srcDir, 'content/index.tsx') + content_script: path.join(srcDir, 'content/index.ts') }, output: { path: path.join(__dirname, "../dist/js"),