refactor: move computer tool to cua-computer plugin
Move GUI automation from core to a standalone plugin. This keeps the
core lean and lets users opt-in when they need computer-use capabilities.
Plugin provides:
- Screenshot capture
- Mouse clicks (left, right, double)
- Keyboard input (type, key, hotkey)
- Scrolling and dragging
- Cursor position/screen size queries
Enable via config:
```yaml
plugins:
cua-computer:
serverUrl: "http://localhost:8000"
```
This commit is contained in:
parent
e8b9963437
commit
438c80b758
22
extensions/cua-computer/clawdbot.plugin.json
Normal file
22
extensions/cua-computer/clawdbot.plugin.json
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
"id": "cua-computer",
|
||||||
|
"name": "Cua Computer",
|
||||||
|
"description": "GUI automation via cua-computer-server - screenshots, clicks, typing, scrolling",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"configSchema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"serverUrl": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "computer-server URL (default: http://localhost:8000)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"uiHints": {
|
||||||
|
"serverUrl": {
|
||||||
|
"label": "Server URL",
|
||||||
|
"placeholder": "http://localhost:8000",
|
||||||
|
"help": "URL of the cua-computer-server instance"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -9,10 +9,10 @@
|
|||||||
|
|
||||||
import { Type } from "@sinclair/typebox";
|
import { Type } from "@sinclair/typebox";
|
||||||
|
|
||||||
import type { ClawdbotConfig } from "../../config/config.js";
|
import type { ClawdbotConfig } from "../../src/config/config.js";
|
||||||
import { stringEnum } from "../schema/typebox.js";
|
import { stringEnum } from "../../src/agents/schema/typebox.js";
|
||||||
import type { AnyAgentTool } from "./common.js";
|
import type { AnyAgentTool } from "../../src/agents/tools/common.js";
|
||||||
import { imageResult, jsonResult, readNumberParam, readStringParam } from "./common.js";
|
import { imageResult, jsonResult, readNumberParam, readStringParam } from "../../src/agents/tools/common.js";
|
||||||
import { ComputerServerClient, ComputerServerError } from "./computer-server-client.js";
|
import { ComputerServerClient, ComputerServerError } from "./computer-server-client.js";
|
||||||
|
|
||||||
const COMPUTER_ACTIONS = [
|
const COMPUTER_ACTIONS = [
|
||||||
33
extensions/cua-computer/index.ts
Normal file
33
extensions/cua-computer/index.ts
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
/**
|
||||||
|
* Cua Computer Plugin
|
||||||
|
*
|
||||||
|
* Provides GUI automation via cua-computer-server - screenshots, clicks, typing, scrolling.
|
||||||
|
*
|
||||||
|
* @see https://github.com/trycua/cua/tree/main/libs/python/computer-server
|
||||||
|
*/
|
||||||
|
|
||||||
|
import type { ClawdbotPluginDefinition } from "../../src/plugins/types.js";
|
||||||
|
import { createComputerTool } from "./computer-tool.js";
|
||||||
|
|
||||||
|
interface CuaComputerConfig {
|
||||||
|
serverUrl?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const plugin: ClawdbotPluginDefinition = {
|
||||||
|
id: "cua-computer",
|
||||||
|
name: "Cua Computer",
|
||||||
|
description: "GUI automation via cua-computer-server",
|
||||||
|
|
||||||
|
register(api) {
|
||||||
|
const config = api.pluginConfig as CuaComputerConfig | undefined;
|
||||||
|
|
||||||
|
api.registerTool(
|
||||||
|
createComputerTool({
|
||||||
|
defaultServerUrl: config?.serverUrl,
|
||||||
|
config: api.config,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
export default plugin;
|
||||||
6
extensions/cua-computer/package.json
Normal file
6
extensions/cua-computer/package.json
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
{
|
||||||
|
"name": "@clawdbot/plugin-cua-computer",
|
||||||
|
"version": "0.1.0",
|
||||||
|
"type": "module",
|
||||||
|
"main": "index.ts"
|
||||||
|
}
|
||||||
@ -6,7 +6,6 @@ import { createAgentsListTool } from "./tools/agents-list-tool.js";
|
|||||||
import { createBrowserTool } from "./tools/browser-tool.js";
|
import { createBrowserTool } from "./tools/browser-tool.js";
|
||||||
import { createCanvasTool } from "./tools/canvas-tool.js";
|
import { createCanvasTool } from "./tools/canvas-tool.js";
|
||||||
import type { AnyAgentTool } from "./tools/common.js";
|
import type { AnyAgentTool } from "./tools/common.js";
|
||||||
import { createComputerTool } from "./tools/computer-tool.js";
|
|
||||||
import { createCronTool } from "./tools/cron-tool.js";
|
import { createCronTool } from "./tools/cron-tool.js";
|
||||||
import { createGatewayTool } from "./tools/gateway-tool.js";
|
import { createGatewayTool } from "./tools/gateway-tool.js";
|
||||||
import { createImageTool } from "./tools/image-tool.js";
|
import { createImageTool } from "./tools/image-tool.js";
|
||||||
@ -26,8 +25,6 @@ export function createClawdbotTools(options?: {
|
|||||||
allowedControlUrls?: string[];
|
allowedControlUrls?: string[];
|
||||||
allowedControlHosts?: string[];
|
allowedControlHosts?: string[];
|
||||||
allowedControlPorts?: number[];
|
allowedControlPorts?: number[];
|
||||||
/** URL of the computer-server for GUI automation (default: http://localhost:8000) */
|
|
||||||
computerServerUrl?: string;
|
|
||||||
agentSessionKey?: string;
|
agentSessionKey?: string;
|
||||||
agentChannel?: GatewayMessageChannel;
|
agentChannel?: GatewayMessageChannel;
|
||||||
agentAccountId?: string;
|
agentAccountId?: string;
|
||||||
@ -82,10 +79,6 @@ export function createClawdbotTools(options?: {
|
|||||||
allowedControlHosts: options?.allowedControlHosts,
|
allowedControlHosts: options?.allowedControlHosts,
|
||||||
allowedControlPorts: options?.allowedControlPorts,
|
allowedControlPorts: options?.allowedControlPorts,
|
||||||
}),
|
}),
|
||||||
createComputerTool({
|
|
||||||
defaultServerUrl: options?.computerServerUrl,
|
|
||||||
config: options?.config,
|
|
||||||
}),
|
|
||||||
createCanvasTool(),
|
createCanvasTool(),
|
||||||
createNodesTool({
|
createNodesTool({
|
||||||
agentSessionKey: options?.agentSessionKey,
|
agentSessionKey: options?.agentSessionKey,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user