import crypto from "node:crypto"; import fs from "node:fs/promises"; import { Type } from "@sinclair/typebox"; import { writeBase64ToFile } from "../../cli/nodes-camera.js"; import { canvasSnapshotTempPath, parseCanvasSnapshotPayload } from "../../cli/nodes-canvas.js"; import { imageMimeFromFormat } from "../../media/mime.js"; import { optionalStringEnum, stringEnum } from "../schema/typebox.js"; import { type AnyAgentTool, imageResult, jsonResult, readStringParam } from "./common.js"; import { callGatewayTool, type GatewayCallOptions } from "./gateway.js"; import { resolveNodeId } from "./nodes-utils.js"; const CANVAS_ACTIONS = [ "present", "hide", "navigate", "eval", "snapshot", "a2ui_push", "a2ui_reset", ] as const; const CANVAS_SNAPSHOT_FORMATS = ["png", "jpg", "jpeg"] as const; // Flattened schema: runtime validates per-action requirements. const CanvasToolSchema = Type.Object({ action: stringEnum(CANVAS_ACTIONS), gatewayUrl: Type.Optional(Type.String()), gatewayToken: Type.Optional(Type.String()), timeoutMs: Type.Optional(Type.Number()), node: Type.Optional(Type.String()), // present target: Type.Optional(Type.String()), x: Type.Optional(Type.Number()), y: Type.Optional(Type.Number()), width: Type.Optional(Type.Number()), height: Type.Optional(Type.Number()), // navigate url: Type.Optional(Type.String()), // eval javaScript: Type.Optional(Type.String()), // snapshot outputFormat: optionalStringEnum(CANVAS_SNAPSHOT_FORMATS), maxWidth: Type.Optional(Type.Number()), quality: Type.Optional(Type.Number()), delayMs: Type.Optional(Type.Number()), // a2ui_push jsonl: Type.Optional(Type.String()), jsonlPath: Type.Optional(Type.String()), }); export function createCanvasTool(): AnyAgentTool { return { label: "Canvas", name: "canvas", description: "Control node canvases (present/hide/navigate/eval/snapshot/A2UI). Use snapshot to capture the rendered UI.", parameters: CanvasToolSchema, execute: async (_toolCallId, args) => { const params = args as Record; const action = readStringParam(params, "action", { required: true }); const gatewayOpts: GatewayCallOptions = { gatewayUrl: readStringParam(params, "gatewayUrl", { trim: false }), gatewayToken: readStringParam(params, "gatewayToken", { trim: false }), timeoutMs: typeof params.timeoutMs === "number" ? params.timeoutMs : undefined, }; const nodeId = await resolveNodeId( gatewayOpts, readStringParam(params, "node", { trim: true }), true, ); const invoke = async (command: string, invokeParams?: Record) => await callGatewayTool("node.invoke", gatewayOpts, { nodeId, command, params: invokeParams, idempotencyKey: crypto.randomUUID(), }); switch (action) { case "present": { const placement = { x: typeof params.x === "number" ? params.x : undefined, y: typeof params.y === "number" ? params.y : undefined, width: typeof params.width === "number" ? params.width : undefined, height: typeof params.height === "number" ? params.height : undefined, }; const invokeParams: Record = {}; if (typeof params.target === "string" && params.target.trim()) { invokeParams.url = params.target.trim(); } if ( Number.isFinite(placement.x) || Number.isFinite(placement.y) || Number.isFinite(placement.width) || Number.isFinite(placement.height) ) { invokeParams.placement = placement; } await invoke("canvas.present", invokeParams); return jsonResult({ ok: true }); } case "hide": await invoke("canvas.hide", undefined); return jsonResult({ ok: true }); case "navigate": { const url = readStringParam(params, "url", { required: true }); await invoke("canvas.navigate", { url }); return jsonResult({ ok: true }); } case "eval": { const javaScript = readStringParam(params, "javaScript", { required: true, }); const raw = (await invoke("canvas.eval", { javaScript })) as { payload?: { result?: string }; }; const result = raw?.payload?.result; if (result) { return { content: [{ type: "text", text: result }], details: { result }, }; } return jsonResult({ ok: true }); } case "snapshot": { const formatRaw = typeof params.outputFormat === "string" ? params.outputFormat.toLowerCase() : "png"; const format = formatRaw === "jpg" || formatRaw === "jpeg" ? "jpeg" : "png"; const maxWidth = typeof params.maxWidth === "number" && Number.isFinite(params.maxWidth) ? params.maxWidth : undefined; const quality = typeof params.quality === "number" && Number.isFinite(params.quality) ? params.quality : undefined; const raw = (await invoke("canvas.snapshot", { format, maxWidth, quality, })) as { payload?: unknown }; const payload = parseCanvasSnapshotPayload(raw?.payload); const filePath = canvasSnapshotTempPath({ ext: payload.format === "jpeg" ? "jpg" : payload.format, }); await writeBase64ToFile(filePath, payload.base64); const mimeType = imageMimeFromFormat(payload.format) ?? "image/png"; return await imageResult({ label: "canvas:snapshot", path: filePath, base64: payload.base64, mimeType, details: { format: payload.format }, }); } case "a2ui_push": { const jsonl = typeof params.jsonl === "string" && params.jsonl.trim() ? params.jsonl : typeof params.jsonlPath === "string" && params.jsonlPath.trim() ? await fs.readFile(params.jsonlPath.trim(), "utf8") : ""; if (!jsonl.trim()) throw new Error("jsonl or jsonlPath required"); await invoke("canvas.a2ui.pushJSONL", { jsonl }); return jsonResult({ ok: true }); } case "a2ui_reset": await invoke("canvas.a2ui.reset", undefined); return jsonResult({ ok: true }); default: throw new Error(`Unknown action: ${action}`); } }, }; }