openclaw/src/agents/tools/browser-tool.ts

import { Type } from "@sinclair/typebox";

import {
  browserCloseTab,
  browserFocusTab,
  browserOpenTab,
  browserSnapshot,
  browserStart,
  browserStatus,
  browserStop,
  browserTabs,
} from "../../browser/client.js";
import {
  browserAct,
  browserArmDialog,
  browserArmFileChooser,
  browserConsoleMessages,
  browserNavigate,
  browserPdfSave,
  browserScreenshotAction,
} from "../../browser/client-actions.js";
import { resolveBrowserConfig } from "../../browser/config.js";
import { loadConfig } from "../../config/config.js";
import {
  type AnyAgentTool,
  imageResultFromFile,
  jsonResult,
  readStringParam,
} from "./common.js";

const BrowserActSchema = Type.Union([
  Type.Object({
    kind: Type.Literal("click"),
    ref: Type.String(),
    targetId: Type.Optional(Type.String()),
    doubleClick: Type.Optional(Type.Boolean()),
    button: Type.Optional(Type.String()),
    modifiers: Type.Optional(Type.Array(Type.String())),
  }),
  Type.Object({
    kind: Type.Literal("type"),
    ref: Type.String(),
    text: Type.String(),
    targetId: Type.Optional(Type.String()),
    submit: Type.Optional(Type.Boolean()),
    slowly: Type.Optional(Type.Boolean()),
  }),
  Type.Object({
    kind: Type.Literal("press"),
    key: Type.String(),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("hover"),
    ref: Type.String(),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("drag"),
    startRef: Type.String(),
    endRef: Type.String(),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("select"),
    ref: Type.String(),
    values: Type.Array(Type.String()),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("fill"),
    fields: Type.Array(Type.Record(Type.String(), Type.Unknown())),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("resize"),
    width: Type.Number(),
    height: Type.Number(),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("wait"),
    timeMs: Type.Optional(Type.Number()),
    text: Type.Optional(Type.String()),
    textGone: Type.Optional(Type.String()),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("evaluate"),
    fn: Type.String(),
    ref: Type.Optional(Type.String()),
    targetId: Type.Optional(Type.String()),
  }),
  Type.Object({
    kind: Type.Literal("close"),
    targetId: Type.Optional(Type.String()),
  }),
]);

// IMPORTANT: OpenAI function tool schemas must have a top-level `type: "object"`.
// A root-level `Type.Union([...])` compiles to `{ anyOf: [...] }` (no `type`),
// which OpenAI rejects ("Invalid schema ... type: None"). Keep this schema an object.
const BrowserToolSchema = Type.Object({
  action: Type.Union([
    Type.Literal("status"),
    Type.Literal("start"),
    Type.Literal("stop"),
    Type.Literal("tabs"),
    Type.Literal("open"),
    Type.Literal("focus"),
    Type.Literal("close"),
    Type.Literal("snapshot"),
    Type.Literal("screenshot"),
    Type.Literal("navigate"),
    Type.Literal("console"),
    Type.Literal("pdf"),
    Type.Literal("upload"),
    Type.Literal("dialog"),
    Type.Literal("act"),
  ]),
  controlUrl: Type.Optional(Type.String()),
  targetUrl: Type.Optional(Type.String()),
  targetId: Type.Optional(Type.String()),
  limit: Type.Optional(Type.Number()),
  format: Type.Optional(Type.Union([Type.Literal("aria"), Type.Literal("ai")])),
  fullPage: Type.Optional(Type.Boolean()),
  ref: Type.Optional(Type.String()),
  element: Type.Optional(Type.String()),
  type: Type.Optional(Type.Union([Type.Literal("png"), Type.Literal("jpeg")])),
  level: Type.Optional(Type.String()),
  paths: Type.Optional(Type.Array(Type.String())),
  inputRef: Type.Optional(Type.String()),
  timeoutMs: Type.Optional(Type.Number()),
  accept: Type.Optional(Type.Boolean()),
  promptText: Type.Optional(Type.String()),
  request: Type.Optional(BrowserActSchema),
});

function resolveBrowserBaseUrl(controlUrl?: string) {
  const cfg = loadConfig();
  const resolved = resolveBrowserConfig(cfg.browser);
  if (!resolved.enabled && !controlUrl?.trim()) {
    throw new Error(
      "Browser control is disabled. Set browser.enabled=true in ~/.clawdbot/clawdbot.json.",
    );
  }
  const url = controlUrl?.trim() ? controlUrl.trim() : resolved.controlUrl;
  return url.replace(/\/$/, "");
}

export function createBrowserTool(opts?: {
  defaultControlUrl?: string;
}): AnyAgentTool {
  return {
    label: "Browser",
    name: "browser",
    description:
      "Control clawd's dedicated browser (status/start/stop/tabs/open/snapshot/screenshot/actions). Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
    parameters: BrowserToolSchema,
    execute: async (_toolCallId, args) => {
      const params = args as Record<string, unknown>;
      const action = readStringParam(params, "action", { required: true });
      const controlUrl = readStringParam(params, "controlUrl");
      const baseUrl = resolveBrowserBaseUrl(
        controlUrl ?? opts?.defaultControlUrl,
      );

      switch (action) {
        case "status":
          return jsonResult(await browserStatus(baseUrl));
        case "start":
          await browserStart(baseUrl);
          return jsonResult(await browserStatus(baseUrl));
        case "stop":
          await browserStop(baseUrl);
          return jsonResult(await browserStatus(baseUrl));
        case "tabs":
          return jsonResult({ tabs: await browserTabs(baseUrl) });
        case "open": {
          const targetUrl = readStringParam(params, "targetUrl", {
            required: true,
          });
          return jsonResult(await browserOpenTab(baseUrl, targetUrl));
        }
        case "focus": {
          const targetId = readStringParam(params, "targetId", {
            required: true,
          });
          await browserFocusTab(baseUrl, targetId);
          return jsonResult({ ok: true });
        }
        case "close": {
          const targetId = readStringParam(params, "targetId");
          if (targetId) await browserCloseTab(baseUrl, targetId);
          else await browserAct(baseUrl, { kind: "close" });
          return jsonResult({ ok: true });
        }
        case "snapshot": {
          const format =
            params.format === "ai" || params.format === "aria"
              ? (params.format as "ai" | "aria")
              : "ai";
          const targetId =
            typeof params.targetId === "string"
              ? params.targetId.trim()
              : undefined;
          const limit =
            typeof params.limit === "number" && Number.isFinite(params.limit)
              ? params.limit
              : undefined;
          const snapshot = await browserSnapshot(baseUrl, {
            format,
            targetId,
            limit,
          });
          if (snapshot.format === "ai") {
            return {
              content: [{ type: "text", text: snapshot.snapshot }],
              details: snapshot,
            };
          }
          return jsonResult(snapshot);
        }
        case "screenshot": {
          const targetId = readStringParam(params, "targetId");
          const fullPage = Boolean(params.fullPage);
          const ref = readStringParam(params, "ref");
          const element = readStringParam(params, "element");
          const type = params.type === "jpeg" ? "jpeg" : "png";
          const result = await browserScreenshotAction(baseUrl, {
            targetId,
            fullPage,
            ref,
            element,
            type,
          });
          return await imageResultFromFile({
            label: "browser:screenshot",
            path: result.path,
            details: result,
          });
        }
        case "navigate": {
          const targetUrl = readStringParam(params, "targetUrl", {
            required: true,
          });
          const targetId = readStringParam(params, "targetId");
          return jsonResult(
            await browserNavigate(baseUrl, { url: targetUrl, targetId }),
          );
        }
        case "console": {
          const level =
            typeof params.level === "string" ? params.level.trim() : undefined;
          const targetId =
            typeof params.targetId === "string"
              ? params.targetId.trim()
              : undefined;
          return jsonResult(
            await browserConsoleMessages(baseUrl, { level, targetId }),
          );
        }
        case "pdf": {
          const targetId =
            typeof params.targetId === "string"
              ? params.targetId.trim()
              : undefined;
          const result = await browserPdfSave(baseUrl, { targetId });
          return {
            content: [{ type: "text", text: `FILE:${result.path}` }],
            details: result,
          };
        }
        case "upload": {
          const paths = Array.isArray(params.paths)
            ? params.paths.map((p) => String(p))
            : [];
          if (paths.length === 0) throw new Error("paths required");
          const ref = readStringParam(params, "ref");
          const inputRef = readStringParam(params, "inputRef");
          const element = readStringParam(params, "element");
          const targetId =
            typeof params.targetId === "string"
              ? params.targetId.trim()
              : undefined;
          const timeoutMs =
            typeof params.timeoutMs === "number" &&
            Number.isFinite(params.timeoutMs)
              ? params.timeoutMs
              : undefined;
          return jsonResult(
            await browserArmFileChooser(baseUrl, {
              paths,
              ref,
              inputRef,
              element,
              targetId,
              timeoutMs,
            }),
          );
        }
        case "dialog": {
          const accept = Boolean(params.accept);
          const promptText =
            typeof params.promptText === "string"
              ? params.promptText
              : undefined;
          const targetId =
            typeof params.targetId === "string"
              ? params.targetId.trim()
              : undefined;
          const timeoutMs =
            typeof params.timeoutMs === "number" &&
            Number.isFinite(params.timeoutMs)
              ? params.timeoutMs
              : undefined;
          return jsonResult(
            await browserArmDialog(baseUrl, {
              accept,
              promptText,
              targetId,
              timeoutMs,
            }),
          );
        }
        case "act": {
          const request = params.request as Record<string, unknown> | undefined;
          if (!request || typeof request !== "object") {
            throw new Error("request required");
          }
          const result = await browserAct(
            baseUrl,
            request as Parameters<typeof browserAct>[1],
          );
          return jsonResult(result);
        }
        default:
          throw new Error(`Unknown action: ${action}`);
      }
    },
  };
}