From 4ad28bb60c773e2bfe8e2472dbc5853ad05c09f9 Mon Sep 17 00:00:00 2001 From: Louis Walsh Date: Mon, 26 Jan 2026 16:15:10 -0800 Subject: [PATCH] feat(web-fetch): add Exa as a content extraction provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Exa AI as a third content extraction provider for the web_fetch tool (alongside Readability and Firecrawl). - Config schema: Added `tools.web.fetch.exa` configuration object - Types: Added TypeScript types for Exa extraction config - Web fetch: Added `fetchExaContent()` using Exa's `/contents` API - Extraction order: Exa is tried first (if enabled), then falls back to Readability and Firecrawl - Documentation: Added comprehensive Exa setup section in web tools docs - Tests: Added config validation tests for Exa and Firecrawl providers Configuration example: ```json5 { tools: { web: { fetch: { exa: { enabled: true, apiKey: "your-exa-api-key", // or set EXA_API_KEY env var contents: true, maxChars: 1500 } } } } } ``` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docs/tools/web.md | 52 +++++- src/agents/tools/web-fetch.ts | 151 +++++++++++++++++- src/config/config.web-fetch-providers.test.ts | 106 ++++++++++++ src/config/schema.ts | 7 + src/config/types.tools.ts | 13 ++ src/config/zod-schema.agent-runtime.ts | 22 +++ 6 files changed, 344 insertions(+), 7 deletions(-) create mode 100644 src/config/config.web-fetch-providers.test.ts diff --git a/docs/tools/web.md b/docs/tools/web.md index a42369242..e957d8a4f 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -1,9 +1,10 @@ --- -summary: "Web search + fetch tools (Brave Search API, Perplexity direct/OpenRouter)" +summary: "Web search + fetch tools (Brave Search API, Perplexity direct/OpenRouter, Exa)" read_when: - You want to enable web_search or web_fetch - You need Brave Search API key setup - You want to use Perplexity Sonar for web search + - You want to use Exa for content extraction --- # Web tools @@ -11,7 +12,7 @@ read_when: Clawdbot ships two lightweight web tools: - `web_search` — Search the web via Brave Search API (default) or Perplexity Sonar (direct or via OpenRouter). -- `web_fetch` — HTTP fetch + readable extraction (HTML → markdown/text). +- `web_fetch` — HTTP fetch + readable extraction (HTML → markdown/text). Supports Readability, Firecrawl, and Exa extract. These are **not** browser automation. For JS-heavy sites or logins, use the [Browser tool](/tools/browser). @@ -138,6 +139,41 @@ If no base URL is set, Clawdbot chooses a default based on the API key source: | `perplexity/sonar-pro` (default) | Multi-step reasoning with web search | Complex questions | | `perplexity/sonar-reasoning-pro` | Chain-of-thought analysis | Deep research | +## Using Exa for content extraction + +Exa provides a `/contents` API endpoint optimized for extracting text content from web pages. You can enable Exa as a content extractor for `web_fetch`, and it will be tried first before falling back to Readability or Firecrawl. + +### Getting an Exa API key + +1. Sign up at https://exa.ai/ +2. Generate an API key in your account settings + +### Setting up Exa content extraction + +```json5 +{ + tools: { + web: { + fetch: { + exa: { + enabled: true, + // API key (optional if EXA_API_KEY is set) + apiKey: "your-exa-api-key", + // Include page text in results (default: true) + contents: true, + // Max characters of page text per result (default: 1500) + maxChars: 1500 + } + } + } + } +} +``` + +**Environment alternative:** set `EXA_API_KEY` in the Gateway environment. For a gateway install, put it in `~/.clawdbot/.env`. + +When enabled, Exa extract is tried first for HTML content. If it fails, Clawdbot falls back to Readability and then Firecrawl (if configured). + ## web_search Search the web using your configured provider. @@ -209,7 +245,8 @@ Fetch a URL and extract readable content. ### Requirements - `tools.web.fetch.enabled` must not be `false` (default: enabled) -- Optional Firecrawl fallback: set `tools.web.fetch.firecrawl.apiKey` or `FIRECRAWL_API_KEY`. +- Optional Exa content extraction: set `tools.web.fetch.exa.enabled` and `tools.web.fetch.exa.apiKey` or `EXA_API_KEY` +- Optional Firecrawl fallback: set `tools.web.fetch.firecrawl.apiKey` or `FIRECRAWL_API_KEY` ### Config @@ -225,6 +262,13 @@ Fetch a URL and extract readable content. maxRedirects: 3, userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36", readability: true, + exa: { + enabled: true, + apiKey: "EXA_API_KEY_HERE", // optional if EXA_API_KEY is set + contents: true, + maxChars: 1500, + timeoutSeconds: 30 + }, firecrawl: { enabled: true, apiKey: "FIRECRAWL_API_KEY_HERE", // optional if FIRECRAWL_API_KEY is set @@ -246,7 +290,7 @@ Fetch a URL and extract readable content. - `maxChars` (truncate long pages) Notes: -- `web_fetch` uses Readability (main-content extraction) first, then Firecrawl (if configured). If both fail, the tool returns an error. +- `web_fetch` tries Exa content extraction first (if enabled), then Readability (main-content extraction), then Firecrawl (if configured). If all fail, the tool returns an error. - Firecrawl requests use bot-circumvention mode and cache results by default. - `web_fetch` sends a Chrome-like User-Agent and `Accept-Language` by default; override `userAgent` if needed. - `web_fetch` blocks private/internal hostnames and re-checks redirects (limit with `maxRedirects`). diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index 9f1e565dd..c57bbee9a 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -40,6 +40,8 @@ const DEFAULT_FETCH_MAX_REDIRECTS = 3; const DEFAULT_ERROR_MAX_CHARS = 4_000; const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev"; const DEFAULT_FIRECRAWL_MAX_AGE_MS = 172_800_000; +const EXA_CONTENTS_ENDPOINT = "https://api.exa.ai/contents"; +const DEFAULT_EXA_MAX_CHARS = 1500; const DEFAULT_FETCH_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; @@ -78,6 +80,25 @@ type FirecrawlFetchConfig = } | undefined; +type ExaExtractConfig = + | { + enabled?: boolean; + apiKey?: string; + contents?: boolean; + maxChars?: number; + timeoutSeconds?: number; + } + | undefined; + +type ExaContentsResponse = { + results: Array<{ + url: string; + title?: string | null; + text?: string | null; + publishedDate?: string | null; + }>; +}; + function resolveFetchConfig(cfg?: ClawdbotConfig): WebFetchConfig { const fetch = cfg?.tools?.web?.fetch; if (!fetch || typeof fetch !== "object") return undefined; @@ -147,6 +168,35 @@ function resolveFirecrawlMaxAgeMsOrDefault(firecrawl?: FirecrawlFetchConfig): nu return DEFAULT_FIRECRAWL_MAX_AGE_MS; } +function resolveExaExtractConfig(fetch?: WebFetchConfig): ExaExtractConfig { + if (!fetch || typeof fetch !== "object") return undefined; + const exa = "exa" in fetch ? fetch.exa : undefined; + if (!exa || typeof exa !== "object") return undefined; + return exa as ExaExtractConfig; +} + +function resolveExaExtractApiKey(exa?: ExaExtractConfig): string | undefined { + const fromConfig = + exa && "apiKey" in exa && typeof exa.apiKey === "string" ? exa.apiKey.trim() : ""; + const fromEnv = (process.env.EXA_API_KEY ?? "").trim(); + return fromConfig || fromEnv || undefined; +} + +function resolveExaExtractEnabled(params: { exa?: ExaExtractConfig; apiKey?: string }): boolean { + if (typeof params.exa?.enabled === "boolean") return params.exa.enabled; + return false; +} + +function resolveExaExtractContents(exa?: ExaExtractConfig): boolean { + if (typeof exa?.contents === "boolean") return exa.contents; + return true; +} + +function resolveExaExtractMaxChars(exa?: ExaExtractConfig): number { + if (typeof exa?.maxChars === "number" && exa.maxChars > 0) return exa.maxChars; + return DEFAULT_EXA_MAX_CHARS; +} + function resolveMaxChars(value: unknown, fallback: number): number { const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; return Math.max(100, Math.floor(parsed)); @@ -329,6 +379,60 @@ export async function fetchFirecrawlContent(params: { }; } +export async function fetchExaContent(params: { + url: string; + extractMode: ExtractMode; + apiKey: string; + contents: boolean; + maxChars: number; + timeoutSeconds: number; +}): Promise<{ + text: string; + title?: string; + finalUrl?: string; + status?: number; +}> { + const body: Record = { + ids: [params.url], + }; + + if (params.contents) { + body.text = { maxCharacters: params.maxChars }; + } + + const res = await fetch(EXA_CONTENTS_ENDPOINT, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": params.apiKey, + }, + body: JSON.stringify(body), + signal: withTimeout(undefined, params.timeoutSeconds * 1000), + }); + + const payload = (await res.json()) as ExaContentsResponse; + + if (!res.ok) { + const detail = await readResponseText(res); + throw new Error(`Exa contents API error (${res.status}): ${detail || res.statusText}`); + } + + const result = payload.results?.[0]; + if (!result) { + throw new Error("Exa contents returned no results"); + } + + const rawText = result.text ?? ""; + const text = params.extractMode === "text" ? markdownToText(rawText) : rawText; + + return { + text, + title: result.title ?? undefined, + finalUrl: result.url, + status: 200, + }; +} + async function runWebFetch(params: { url: string; extractMode: ExtractMode; @@ -346,6 +450,11 @@ async function runWebFetch(params: { firecrawlProxy: "auto" | "basic" | "stealth"; firecrawlStoreInCache: boolean; firecrawlTimeoutSeconds: number; + exaExtractEnabled: boolean; + exaExtractApiKey?: string; + exaExtractContents: boolean; + exaExtractMaxChars: number; + exaExtractTimeoutSeconds: number; }): Promise> { const cacheKey = normalizeCacheKey( `fetch:${params.url}:${params.extractMode}:${params.maxChars}`, @@ -464,7 +573,29 @@ async function runWebFetch(params: { let extractor = "raw"; let text = body; if (contentType.includes("text/html")) { - if (params.readabilityEnabled) { + // Try Exa extract first if enabled + if (params.exaExtractEnabled && params.exaExtractApiKey) { + try { + const exa = await fetchExaContent({ + url: finalUrl, + extractMode: params.extractMode, + apiKey: params.exaExtractApiKey, + contents: params.exaExtractContents, + maxChars: params.exaExtractMaxChars, + timeoutSeconds: params.exaExtractTimeoutSeconds, + }); + if (exa.text) { + text = exa.text; + title = exa.title; + extractor = "exa"; + } + } catch { + // Fall back to Readability/Firecrawl if Exa fails + } + } + + // If Exa didn't succeed, try Readability + if (extractor === "raw" && params.readabilityEnabled) { const readable = await extractReadableContent({ html: body, url: finalUrl, @@ -480,13 +611,13 @@ async function runWebFetch(params: { text = firecrawl.text; title = firecrawl.title; extractor = "firecrawl"; - } else { + } else if (!params.exaExtractEnabled) { throw new Error( "Web fetch extraction failed: Readability and Firecrawl returned no content.", ); } } - } else { + } else if (extractor === "raw" && !params.exaExtractEnabled) { throw new Error( "Web fetch extraction failed: Readability disabled and Firecrawl unavailable.", ); @@ -586,6 +717,15 @@ export function createWebFetchTool(options?: { firecrawl?.timeoutSeconds ?? fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, ); + const exaExtract = resolveExaExtractConfig(fetch); + const exaExtractApiKey = resolveExaExtractApiKey(exaExtract); + const exaExtractEnabled = resolveExaExtractEnabled({ exa: exaExtract, apiKey: exaExtractApiKey }); + const exaExtractContents = resolveExaExtractContents(exaExtract); + const exaExtractMaxChars = resolveExaExtractMaxChars(exaExtract); + const exaExtractTimeoutSeconds = resolveTimeoutSeconds( + exaExtract?.timeoutSeconds ?? fetch?.timeoutSeconds, + DEFAULT_TIMEOUT_SECONDS, + ); const userAgent = (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) || DEFAULT_FETCH_USER_AGENT; @@ -617,6 +757,11 @@ export function createWebFetchTool(options?: { firecrawlProxy: "auto", firecrawlStoreInCache: true, firecrawlTimeoutSeconds, + exaExtractEnabled, + exaExtractApiKey, + exaExtractContents, + exaExtractMaxChars, + exaExtractTimeoutSeconds, }); return jsonResult(result); }, diff --git a/src/config/config.web-fetch-providers.test.ts b/src/config/config.web-fetch-providers.test.ts new file mode 100644 index 000000000..4423c76f8 --- /dev/null +++ b/src/config/config.web-fetch-providers.test.ts @@ -0,0 +1,106 @@ +import { describe, expect, it } from "vitest"; + +import { validateConfigObject } from "./config.js"; + +describe("web fetch providers config", () => { + it("accepts exa content extraction config", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + enabled: true, + exa: { + enabled: true, + apiKey: "test-key", + contents: true, + maxChars: 2000, + timeoutSeconds: 30, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(true); + }); + + it("accepts firecrawl config", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + enabled: true, + firecrawl: { + enabled: true, + apiKey: "test-key", + baseUrl: "https://api.firecrawl.dev", + onlyMainContent: true, + maxAgeMs: 86400000, + timeoutSeconds: 60, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(true); + }); + + it("accepts both exa and firecrawl config", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + enabled: true, + readability: true, + exa: { + enabled: true, + apiKey: "exa-key", + contents: true, + maxChars: 1500, + }, + firecrawl: { + enabled: true, + apiKey: "firecrawl-key", + onlyMainContent: true, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(true); + }); + + it("rejects invalid exa maxChars", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + exa: { + maxChars: -100, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(false); + }); + + it("rejects invalid firecrawl maxAgeMs", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + firecrawl: { + maxAgeMs: -1000, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(false); + }); +}); diff --git a/src/config/schema.ts b/src/config/schema.ts index 9627d64f3..2c4876350 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -461,6 +461,13 @@ const FIELD_HELP: Record = { "tools.web.fetch.firecrawl.maxAgeMs": "Firecrawl maxAge (ms) for cached results when supported by the API.", "tools.web.fetch.firecrawl.timeoutSeconds": "Timeout in seconds for Firecrawl requests.", + "tools.web.fetch.exa.enabled": "Enable Exa content extraction for web_fetch (default: false).", + "tools.web.fetch.exa.apiKey": "Exa API key (fallback: EXA_API_KEY env var).", + "tools.web.fetch.exa.contents": + "Include page text in Exa results; when false, only URLs and titles are returned (default: true).", + "tools.web.fetch.exa.maxChars": + "Max characters of page text per Exa result; higher values provide more context but use more tokens (default: 1500).", + "tools.web.fetch.exa.timeoutSeconds": "Timeout in seconds for Exa requests.", "channels.slack.allowBots": "Allow bot-authored messages to trigger Slack replies (default: false).", "channels.slack.thread.historyScope": diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index d84dd1aa7..791329f91 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -381,6 +381,19 @@ export type ToolsConfig = { /** Timeout in seconds for Firecrawl requests. */ timeoutSeconds?: number; }; + /** Exa-specific configuration for content extraction. */ + exa?: { + /** Enable Exa content extraction (default: false). */ + enabled?: boolean; + /** API key for Exa (defaults to EXA_API_KEY env var). */ + apiKey?: string; + /** Include page text in results (default: true). */ + contents?: boolean; + /** Max characters of page text per result (default: 1500). */ + maxChars?: number; + /** Timeout in seconds for Exa requests. */ + timeoutSeconds?: number; + }; }; }; media?: MediaToolsConfig; diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index b5a03a3ea..f9b122e86 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -192,7 +192,29 @@ export const ToolsWebFetchSchema = z timeoutSeconds: z.number().int().positive().optional(), cacheTtlMinutes: z.number().nonnegative().optional(), maxRedirects: z.number().int().nonnegative().optional(), + readability: z.boolean().optional(), userAgent: z.string().optional(), + firecrawl: z + .object({ + enabled: z.boolean().optional(), + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + onlyMainContent: z.boolean().optional(), + maxAgeMs: z.number().int().positive().optional(), + timeoutSeconds: z.number().int().positive().optional(), + }) + .strict() + .optional(), + exa: z + .object({ + enabled: z.boolean().optional(), + apiKey: z.string().optional(), + contents: z.boolean().optional(), + maxChars: z.number().int().positive().optional(), + timeoutSeconds: z.number().int().positive().optional(), + }) + .strict() + .optional(), }) .strict() .optional();