From 438c80b7587264759f936a5dc66ad4914c230afe Mon Sep 17 00:00:00 2001 From: f-trycua Date: Mon, 26 Jan 2026 15:56:22 -0800 Subject: [PATCH] refactor: move computer tool to cua-computer plugin Move GUI automation from core to a standalone plugin. This keeps the core lean and lets users opt-in when they need computer-use capabilities. Plugin provides: - Screenshot capture - Mouse clicks (left, right, double) - Keyboard input (type, key, hotkey) - Scrolling and dragging - Cursor position/screen size queries Enable via config: ```yaml plugins: cua-computer: serverUrl: "http://localhost:8000" ``` --- extensions/cua-computer/clawdbot.plugin.json | 22 +++++++++++++ .../cua-computer}/computer-server-client.ts | 0 .../cua-computer}/computer-tool.ts | 8 ++--- extensions/cua-computer/index.ts | 33 +++++++++++++++++++ extensions/cua-computer/package.json | 6 ++++ src/agents/clawdbot-tools.ts | 7 ---- 6 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 extensions/cua-computer/clawdbot.plugin.json rename {src/agents/tools => extensions/cua-computer}/computer-server-client.ts (100%) rename {src/agents/tools => extensions/cua-computer}/computer-tool.ts (97%) create mode 100644 extensions/cua-computer/index.ts create mode 100644 extensions/cua-computer/package.json diff --git a/extensions/cua-computer/clawdbot.plugin.json b/extensions/cua-computer/clawdbot.plugin.json new file mode 100644 index 000000000..bbccc202d --- /dev/null +++ b/extensions/cua-computer/clawdbot.plugin.json @@ -0,0 +1,22 @@ +{ + "id": "cua-computer", + "name": "Cua Computer", + "description": "GUI automation via cua-computer-server - screenshots, clicks, typing, scrolling", + "version": "0.1.0", + "configSchema": { + "type": "object", + "properties": { + "serverUrl": { + "type": "string", + "description": "computer-server URL (default: http://localhost:8000)" + } + } + }, + "uiHints": { + "serverUrl": { + "label": "Server URL", + "placeholder": "http://localhost:8000", + "help": "URL of the cua-computer-server instance" + } + } +} diff --git a/src/agents/tools/computer-server-client.ts b/extensions/cua-computer/computer-server-client.ts similarity index 100% rename from src/agents/tools/computer-server-client.ts rename to extensions/cua-computer/computer-server-client.ts diff --git a/src/agents/tools/computer-tool.ts b/extensions/cua-computer/computer-tool.ts similarity index 97% rename from src/agents/tools/computer-tool.ts rename to extensions/cua-computer/computer-tool.ts index 719df358d..875f922d9 100644 --- a/src/agents/tools/computer-tool.ts +++ b/extensions/cua-computer/computer-tool.ts @@ -9,10 +9,10 @@ import { Type } from "@sinclair/typebox"; -import type { ClawdbotConfig } from "../../config/config.js"; -import { stringEnum } from "../schema/typebox.js"; -import type { AnyAgentTool } from "./common.js"; -import { imageResult, jsonResult, readNumberParam, readStringParam } from "./common.js"; +import type { ClawdbotConfig } from "../../src/config/config.js"; +import { stringEnum } from "../../src/agents/schema/typebox.js"; +import type { AnyAgentTool } from "../../src/agents/tools/common.js"; +import { imageResult, jsonResult, readNumberParam, readStringParam } from "../../src/agents/tools/common.js"; import { ComputerServerClient, ComputerServerError } from "./computer-server-client.js"; const COMPUTER_ACTIONS = [ diff --git a/extensions/cua-computer/index.ts b/extensions/cua-computer/index.ts new file mode 100644 index 000000000..fd3408371 --- /dev/null +++ b/extensions/cua-computer/index.ts @@ -0,0 +1,33 @@ +/** + * Cua Computer Plugin + * + * Provides GUI automation via cua-computer-server - screenshots, clicks, typing, scrolling. + * + * @see https://github.com/trycua/cua/tree/main/libs/python/computer-server + */ + +import type { ClawdbotPluginDefinition } from "../../src/plugins/types.js"; +import { createComputerTool } from "./computer-tool.js"; + +interface CuaComputerConfig { + serverUrl?: string; +} + +const plugin: ClawdbotPluginDefinition = { + id: "cua-computer", + name: "Cua Computer", + description: "GUI automation via cua-computer-server", + + register(api) { + const config = api.pluginConfig as CuaComputerConfig | undefined; + + api.registerTool( + createComputerTool({ + defaultServerUrl: config?.serverUrl, + config: api.config, + }), + ); + }, +}; + +export default plugin; diff --git a/extensions/cua-computer/package.json b/extensions/cua-computer/package.json new file mode 100644 index 000000000..6563fcaaa --- /dev/null +++ b/extensions/cua-computer/package.json @@ -0,0 +1,6 @@ +{ + "name": "@clawdbot/plugin-cua-computer", + "version": "0.1.0", + "type": "module", + "main": "index.ts" +} diff --git a/src/agents/clawdbot-tools.ts b/src/agents/clawdbot-tools.ts index 96571efd9..91de31937 100644 --- a/src/agents/clawdbot-tools.ts +++ b/src/agents/clawdbot-tools.ts @@ -6,7 +6,6 @@ import { createAgentsListTool } from "./tools/agents-list-tool.js"; import { createBrowserTool } from "./tools/browser-tool.js"; import { createCanvasTool } from "./tools/canvas-tool.js"; import type { AnyAgentTool } from "./tools/common.js"; -import { createComputerTool } from "./tools/computer-tool.js"; import { createCronTool } from "./tools/cron-tool.js"; import { createGatewayTool } from "./tools/gateway-tool.js"; import { createImageTool } from "./tools/image-tool.js"; @@ -26,8 +25,6 @@ export function createClawdbotTools(options?: { allowedControlUrls?: string[]; allowedControlHosts?: string[]; allowedControlPorts?: number[]; - /** URL of the computer-server for GUI automation (default: http://localhost:8000) */ - computerServerUrl?: string; agentSessionKey?: string; agentChannel?: GatewayMessageChannel; agentAccountId?: string; @@ -82,10 +79,6 @@ export function createClawdbotTools(options?: { allowedControlHosts: options?.allowedControlHosts, allowedControlPorts: options?.allowedControlPorts, }), - createComputerTool({ - defaultServerUrl: options?.computerServerUrl, - config: options?.config, - }), createCanvasTool(), createNodesTool({ agentSessionKey: options?.agentSessionKey,