diff --git a/docs/exa.md b/docs/exa.md new file mode 100644 index 000000000..c34627c10 --- /dev/null +++ b/docs/exa.md @@ -0,0 +1,43 @@ +--- +summary: "Exa AI setup for web_search" +read_when: + - You want to use Exa AI for web search + - You need an EXA_API_KEY or setup details +--- + +# Exa AI + +Clawdbot can use Exa AI for the `web_search` tool. Exa is a search API +built for AI agents. + +## Get an API key + +1) Create an account at https://exa.ai/ +2) Generate an API key at https://dashboard.exa.ai/api-keys +3) Store the key in config (recommended) or set `EXA_API_KEY` in the Gateway environment. + +## Config example + +```json5 +{ + tools: { + web: { + search: { + provider: "exa", + exa: { + apiKey: "your-exa-api-key" + } + } + } + } +} +``` + +## Options + +| Option | Description | Default | +|--------|-------------|---------| +| `contents` | Include page text in results; when false, only URLs and titles are returned | `true` | +| `maxChars` | Max characters of page text per result; higher values provide more context but use more tokens | `1500` | + +See [Web tools](/tools/web) for the full web_search configuration. diff --git a/docs/tools/web.md b/docs/tools/web.md index be2a57f9e..47661401b 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -10,7 +10,7 @@ read_when: Moltbot ships two lightweight web tools: -- `web_search` — Search the web via Brave Search API (default) or Perplexity Sonar (direct or via OpenRouter). +- `web_search` — Search the web via Brave Search API (default), Perplexity Sonar, or Exa. - `web_fetch` — HTTP fetch + readable extraction (HTML → markdown/text). These are **not** browser automation. For JS-heavy sites or logins, use the @@ -21,6 +21,7 @@ These are **not** browser automation. For JS-heavy sites or logins, use the - `web_search` calls your configured provider and returns results. - **Brave** (default): returns structured results (title, URL, snippet). - **Perplexity**: returns AI-synthesized answers with citations from real-time web search. + - **Exa**: fast search built for AI, returns results with optional page text. - Results are cached by query for 15 minutes (configurable). - `web_fetch` does a plain HTTP GET and extracts readable content (HTML → markdown/text). It does **not** execute JavaScript. @@ -32,8 +33,9 @@ These are **not** browser automation. For JS-heavy sites or logins, use the |----------|------|------|---------| | **Brave** (default) | Fast, structured results, free tier | Traditional search results | `BRAVE_API_KEY` | | **Perplexity** | AI-synthesized answers, citations, real-time | Requires Perplexity or OpenRouter access | `OPENROUTER_API_KEY` or `PERPLEXITY_API_KEY` | +| **Exa** | Fast search built for AI, results with optional page text | Requires Exa API key | `EXA_API_KEY` | -See [Brave Search setup](/brave-search) and [Perplexity Sonar](/perplexity) for provider-specific details. +See [Brave Search setup](/brave-search), [Perplexity Sonar](/perplexity), and [Exa AI](/exa) for provider-specific details. Set the provider in config: @@ -42,7 +44,7 @@ Set the provider in config: tools: { web: { search: { - provider: "brave" // or "perplexity" + provider: "brave" // or "perplexity" or "exa" } } } @@ -138,6 +140,48 @@ If no base URL is set, Moltbot chooses a default based on the API key source: | `perplexity/sonar-pro` (default) | Multi-step reasoning with web search | Complex questions | | `perplexity/sonar-reasoning-pro` | Chain-of-thought analysis | Deep research | +## Using Exa + +Exa is a search API built for AI agents. + +### Getting an Exa API key + +1) Create an account at https://exa.ai/ +2) Generate an API key at https://dashboard.exa.ai/api-keys +3) Store the key in config or set `EXA_API_KEY` in the Gateway environment + +### Setting up Exa search + +```json5 +{ + tools: { + web: { + search: { + enabled: true, + provider: "exa", + exa: { + // API key (optional if EXA_API_KEY is set) + apiKey: "your-exa-api-key", + // Include page text in results (default: true) + contents: true, + // Max characters of page text per result (default: 1500) + maxChars: 1500 + } + } + } + } +} +``` + +**Environment alternative:** set `EXA_API_KEY` in the Gateway environment. For a gateway install, put it in `~/.clawdbot/.env`. + +### Exa options + +| Option | Description | Default | +|--------|-------------|---------| +| `contents` | Include page text; when false, only URLs/titles returned | `true` | +| `maxChars` | Max characters of page text per result (higher = more tokens) | `1500` | + ## web_search Search the web using your configured provider. @@ -148,6 +192,7 @@ Search the web using your configured provider. - API key for your chosen provider: - **Brave**: `BRAVE_API_KEY` or `tools.web.search.apiKey` - **Perplexity**: `OPENROUTER_API_KEY`, `PERPLEXITY_API_KEY`, or `tools.web.search.perplexity.apiKey` + - **Exa**: `EXA_API_KEY` or `tools.web.search.exa.apiKey` ### Config diff --git a/src/agents/tools/web-search.test.ts b/src/agents/tools/web-search.test.ts index c6d2b6405..39e563af2 100644 --- a/src/agents/tools/web-search.test.ts +++ b/src/agents/tools/web-search.test.ts @@ -2,8 +2,13 @@ import { describe, expect, it } from "vitest"; import { __testing } from "./web-search.js"; -const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness } = - __testing; +const { + inferPerplexityBaseUrlFromApiKey, + resolvePerplexityBaseUrl, + normalizeFreshness, + resolveExaContents, + resolveExaMaxChars, +} = __testing; describe("web_search perplexity baseUrl defaults", () => { it("detects a Perplexity key prefix", () => { @@ -69,3 +74,30 @@ describe("web_search freshness normalization", () => { expect(normalizeFreshness("2024-03-10to2024-03-01")).toBeUndefined(); }); }); + +describe("web_search exa configuration", () => { + it("defaults contents to true", () => { + expect(resolveExaContents(undefined)).toBe(true); + expect(resolveExaContents({})).toBe(true); + }); + + it("respects explicit contents setting", () => { + expect(resolveExaContents({ contents: false })).toBe(false); + expect(resolveExaContents({ contents: true })).toBe(true); + }); + + it("defaults maxChars to 1500", () => { + expect(resolveExaMaxChars(undefined)).toBe(1500); + expect(resolveExaMaxChars({})).toBe(1500); + }); + + it("respects explicit maxChars setting", () => { + expect(resolveExaMaxChars({ maxChars: 3000 })).toBe(3000); + expect(resolveExaMaxChars({ maxChars: 500 })).toBe(500); + }); + + it("ignores invalid maxChars values", () => { + expect(resolveExaMaxChars({ maxChars: 0 })).toBe(1500); + expect(resolveExaMaxChars({ maxChars: -100 })).toBe(1500); + }); +}); diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index 1d87676e8..dc6c07ba4 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -17,7 +17,7 @@ import { writeCache, } from "./web-shared.js"; -const SEARCH_PROVIDERS = ["brave", "perplexity"] as const; +const SEARCH_PROVIDERS = ["brave", "perplexity", "exa"] as const; const DEFAULT_SEARCH_COUNT = 5; const MAX_SEARCH_COUNT = 10; @@ -28,6 +28,10 @@ const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro"; const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; +// Exa configuration +const EXA_SEARCH_ENDPOINT = "https://api.exa.ai/search"; +const DEFAULT_EXA_MAX_CHARS = 1500; + const SEARCH_CACHE = new Map>>(); const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]); const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/; @@ -90,6 +94,12 @@ type PerplexityConfig = { model?: string; }; +type ExaConfig = { + apiKey?: string; + contents?: boolean; + maxChars?: number; +}; + type PerplexityApiKeySource = "config" | "perplexity_env" | "openrouter_env" | "none"; type PerplexitySearchResponse = { @@ -155,6 +165,20 @@ function resolvePerplexityConfig(search?: WebSearchConfig): PerplexityConfig { return perplexity as PerplexityConfig; } +function resolveExaConfig(search?: WebSearchConfig): ExaConfig { + if (!search || typeof search !== "object") return {}; + const exa = "exa" in search ? search.exa : undefined; + if (!exa || typeof exa !== "object") return {}; + return exa as ExaConfig; +} + +function resolveExaApiKey(exa?: ExaConfig): string | undefined { + // Config key takes precedence + if (exa?.apiKey) return exa.apiKey; + // Fall back to environment variable + return process.env.EXA_API_KEY; +} + function resolvePerplexityApiKey(perplexity?: PerplexityConfig): { apiKey?: string; source: PerplexityApiKeySource; @@ -221,6 +245,16 @@ function resolvePerplexityModel(perplexity?: PerplexityConfig): string { return fromConfig || DEFAULT_PERPLEXITY_MODEL; } +function resolveExaContents(exa?: ExaConfig): boolean { + if (typeof exa?.contents === "boolean") return exa.contents; + return true; +} + +function resolveExaMaxChars(exa?: ExaConfig): number { + if (typeof exa?.maxChars === "number" && exa.maxChars > 0) return exa.maxChars; + return DEFAULT_EXA_MAX_CHARS; +} + function resolveSearchCount(value: unknown, fallback: number): number { const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed))); @@ -306,6 +340,71 @@ async function runPerplexitySearch(params: { return { content, citations }; } +type ExaSearchResponse = { + results?: Array<{ + title?: string; + url?: string; + text?: string; + publishedDate?: string; + }>; +}; + +async function runExaSearch(params: { + query: string; + count: number; + apiKey: string; + timeoutSeconds: number; + contents: boolean; + maxChars: number; +}): Promise<{ + results: Array<{ + title: string; + url: string; + description: string; + published?: string; + }>; +}> { + const body: Record = { + query: params.query, + numResults: params.count, + type: "auto", + }; + + // Add contents configuration if enabled + if (params.contents) { + body.contents = { + text: { maxCharacters: params.maxChars }, + }; + } + + const res = await fetch(EXA_SEARCH_ENDPOINT, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": params.apiKey, + }, + body: JSON.stringify(body), + signal: withTimeout(undefined, params.timeoutSeconds * 1000), + }); + + if (!res.ok) { + const detail = await readResponseText(res); + throw new Error(`Exa API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as ExaSearchResponse; + const results = Array.isArray(data.results) ? data.results : []; + + return { + results: results.map((entry) => ({ + title: entry.title ?? "(no title)", + url: entry.url ?? "", + description: entry.text ?? "", + published: entry.publishedDate, + })), + }; +} + async function runWebSearch(params: { query: string; count: number; @@ -319,11 +418,15 @@ async function runWebSearch(params: { freshness?: string; perplexityBaseUrl?: string; perplexityModel?: string; + exaContents?: boolean; + exaMaxChars?: number; }): Promise> { const cacheKey = normalizeCacheKey( params.provider === "brave" ? `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}:${params.freshness || "default"}` - : `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}`, + : params.provider === "exa" + ? `${params.provider}:${params.query}:${params.count}` + : `${params.provider}:${params.query}:${params.count}:${params.country || "default"}:${params.search_lang || "default"}:${params.ui_lang || "default"}`, ); const cached = readCache(SEARCH_CACHE, cacheKey); if (cached) return { ...cached.value, cached: true }; @@ -351,6 +454,27 @@ async function runWebSearch(params: { return payload; } + if (params.provider === "exa") { + const { results } = await runExaSearch({ + query: params.query, + count: params.count, + apiKey: params.apiKey, + timeoutSeconds: params.timeoutSeconds, + contents: params.exaContents ?? true, + maxChars: params.exaMaxChars ?? DEFAULT_EXA_MAX_CHARS, + }); + + const payload = { + query: params.query, + provider: params.provider, + count: results.length, + tookMs: Date.now() - start, + results, + }; + writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); + return payload; + } + if (params.provider !== "brave") { throw new Error("Unsupported web search provider."); } @@ -415,11 +539,14 @@ export function createWebSearchTool(options?: { const provider = resolveSearchProvider(search); const perplexityConfig = resolvePerplexityConfig(search); + const exaConfig = resolveExaConfig(search); const description = provider === "perplexity" ? "Search the web using Perplexity Sonar (direct or via OpenRouter). Returns AI-synthesized answers with citations from real-time web search." - : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research."; + : provider === "exa" + ? "Search the web using Exa AI. Returns URLs with titles and page text (up to 1500 chars per result by default)." + : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research."; return { label: "Web Search", @@ -430,7 +557,11 @@ export function createWebSearchTool(options?: { const perplexityAuth = provider === "perplexity" ? resolvePerplexityApiKey(perplexityConfig) : undefined; const apiKey = - provider === "perplexity" ? perplexityAuth?.apiKey : resolveSearchApiKey(search); + provider === "perplexity" + ? perplexityAuth?.apiKey + : provider === "exa" + ? resolveExaApiKey(exaConfig) + : resolveSearchApiKey(search); if (!apiKey) { return jsonResult(missingSearchKeyPayload(provider)); @@ -476,6 +607,8 @@ export function createWebSearchTool(options?: { perplexityAuth?.apiKey, ), perplexityModel: resolvePerplexityModel(perplexityConfig), + exaContents: resolveExaContents(exaConfig), + exaMaxChars: resolveExaMaxChars(exaConfig), }); return jsonResult(result); }, @@ -486,4 +619,6 @@ export const __testing = { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness, + resolveExaContents, + resolveExaMaxChars, } as const; diff --git a/src/canvas-host/a2ui/.bundle.hash b/src/canvas-host/a2ui/.bundle.hash index 6c9cb0299..dd57697ae 100644 --- a/src/canvas-host/a2ui/.bundle.hash +++ b/src/canvas-host/a2ui/.bundle.hash @@ -1 +1 @@ -b6d3dea7c656c8a480059c32e954c4d39053ff79c4e9c69b38f4c04e3f0280d4 +bd5789522e6bde45274e15fdd45b10c9a41da378b190d6f42cef5ef2a69d72a7 diff --git a/src/config/schema.ts b/src/config/schema.ts index 9b5ad8be6..2d6edf836 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -445,6 +445,11 @@ const FIELD_HELP: Record = { "Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).", "tools.web.search.perplexity.model": 'Perplexity model override (default: "perplexity/sonar-pro").', + "tools.web.search.exa.apiKey": "Exa API key (fallback: EXA_API_KEY env var).", + "tools.web.search.exa.contents": + "Include page text in results; when false, only URLs and titles are returned (default: true).", + "tools.web.search.exa.maxChars": + "Max characters of page text per result; higher values provide more context but use more tokens (default: 1500).", "tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).", "tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).", "tools.web.fetch.timeoutSeconds": "Timeout in seconds for web_fetch requests.", diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index bb1d45bf0..d0ba0e5a8 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -334,8 +334,8 @@ export type ToolsConfig = { search?: { /** Enable web search tool (default: true when API key is present). */ enabled?: boolean; - /** Search provider ("brave" or "perplexity"). */ - provider?: "brave" | "perplexity"; + /** Search provider ("brave", "perplexity", or "exa"). */ + provider?: "brave" | "perplexity" | "exa"; /** Brave Search API key (optional; defaults to BRAVE_API_KEY env var). */ apiKey?: string; /** Default search results count (1-10). */ @@ -353,6 +353,15 @@ export type ToolsConfig = { /** Model to use (defaults to "perplexity/sonar-pro"). */ model?: string; }; + /** Exa-specific configuration (used when provider="exa"). */ + exa?: { + /** API key for Exa (defaults to EXA_API_KEY env var). */ + apiKey?: string; + /** Include page text in results; when false, only URLs and titles are returned (default: true). */ + contents?: boolean; + /** Max characters of page text per result; higher values use more tokens (default: 1500). */ + maxChars?: number; + }; }; fetch?: { /** Enable web fetch tool (default: true). */ diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 7a63e307d..399aafa2c 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -165,7 +165,7 @@ export const ToolPolicySchema = ToolPolicyBaseSchema.superRefine((value, ctx) => export const ToolsWebSearchSchema = z .object({ enabled: z.boolean().optional(), - provider: z.union([z.literal("brave"), z.literal("perplexity")]).optional(), + provider: z.union([z.literal("brave"), z.literal("perplexity"), z.literal("exa")]).optional(), apiKey: z.string().optional(), maxResults: z.number().int().positive().optional(), timeoutSeconds: z.number().int().positive().optional(), @@ -178,6 +178,14 @@ export const ToolsWebSearchSchema = z }) .strict() .optional(), + exa: z + .object({ + apiKey: z.string().optional(), + contents: z.boolean().optional(), + maxChars: z.number().int().positive().optional(), + }) + .strict() + .optional(), }) .strict() .optional();