From 77c59d53c8211970ea43488683ca50406c90eedb Mon Sep 17 00:00:00 2001 From: Matt Harris Date: Mon, 26 Jan 2026 18:29:39 -0500 Subject: [PATCH] Add Parallel as web-search and web-fetch tools --- docs/tools/web.md | 69 +++++++- src/agents/tools/web-fetch.ts | 166 +++++++++++++++++- src/agents/tools/web-search.test.ts | 52 +++++- src/agents/tools/web-search.ts | 142 ++++++++++++++- src/commands/configure.shared.ts | 2 +- src/commands/configure.wizard.ts | 87 +++++++-- src/commands/onboard-non-interactive/local.ts | 2 +- .../onboard-non-interactive/remote.ts | 2 +- src/config/config.web-search-provider.test.ts | 39 ++++ src/config/types.tools.ts | 22 ++- src/config/zod-schema.agent-runtime.ts | 23 ++- src/security/audit-extra.ts | 4 +- src/wizard/onboarding.finalize.ts | 28 ++- src/wizard/onboarding.test.ts | 21 +++ 14 files changed, 618 insertions(+), 41 deletions(-) diff --git a/docs/tools/web.md b/docs/tools/web.md index a42369242..5c57a9946 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -1,17 +1,18 @@ --- -summary: "Web search + fetch tools (Brave Search API, Perplexity direct/OpenRouter)" +summary: "Web search + fetch tools (Brave Search API, Perplexity direct/OpenRouter, Parallel)" read_when: - You want to enable web_search or web_fetch - You need Brave Search API key setup - You want to use Perplexity Sonar for web search + - You want to use Parallel for web search or content extraction --- # Web tools Clawdbot ships two lightweight web tools: -- `web_search` — Search the web via Brave Search API (default) or Perplexity Sonar (direct or via OpenRouter). -- `web_fetch` — HTTP fetch + readable extraction (HTML → markdown/text). +- `web_search` — Search the web via Brave Search API (default), Perplexity Sonar (direct or via OpenRouter), or Parallel Search API. +- `web_fetch` — HTTP fetch + readable extraction (HTML → markdown/text). Supports Readability, Firecrawl, and Parallel extract. These are **not** browser automation. For JS-heavy sites or logins, use the [Browser tool](/tools/browser). @@ -32,6 +33,7 @@ These are **not** browser automation. For JS-heavy sites or logins, use the |----------|------|------|---------| | **Brave** (default) | Fast, structured results, free tier | Traditional search results | `BRAVE_API_KEY` | | **Perplexity** | AI-synthesized answers, citations, real-time | Requires Perplexity or OpenRouter access | `OPENROUTER_API_KEY` or `PERPLEXITY_API_KEY` | +| **Parallel** | Relevant excerpts, real-time, agentic mode | Requires Parallel API access | `PARALLEL_API_KEY` | See [Brave Search setup](/brave-search) and [Perplexity Sonar](/perplexity) for provider-specific details. @@ -42,7 +44,7 @@ Set the provider in config: tools: { web: { search: { - provider: "brave" // or "perplexity" + provider: "brave" // or "perplexity" or "parallel" } } } @@ -138,6 +140,64 @@ If no base URL is set, Clawdbot chooses a default based on the API key source: | `perplexity/sonar-pro` (default) | Multi-step reasoning with web search | Complex questions | | `perplexity/sonar-reasoning-pro` | Chain-of-thought analysis | Deep research | +## Using Parallel + +Parallel provides web search and content extraction APIs optimized for agentic workflows. +Results include relevant excerpts from each page. + +### Getting a Parallel API key + +1) Sign up at https://parallel.ai/ +2) Generate an API key in your account settings + +### Setting up Parallel search + +```json5 +{ + tools: { + web: { + search: { + enabled: true, + provider: "parallel", + parallel: { + // API key (optional if PARALLEL_API_KEY is set) + apiKey: "your-parallel-api-key", + // Base URL (optional, defaults to https://api.parallel.ai) + baseUrl: "https://api.parallel.ai" + } + } + } + } +} +``` + +**Environment alternative:** set `PARALLEL_API_KEY` in the Gateway environment. +For a gateway install, put it in `~/.clawdbot/.env`. + +### Using Parallel extract for web_fetch + +Parallel extract can be used as a content extractor for `web_fetch`, providing +markdown content from web pages. Enable it alongside or instead of Readability/Firecrawl: + +```json5 +{ + tools: { + web: { + fetch: { + parallel: { + enabled: true, + // API key (optional if PARALLEL_API_KEY is set) + apiKey: "your-parallel-api-key" + } + } + } + } +} +``` + +When enabled, Parallel extract is tried first for HTML content. If it fails, +Clawdbot falls back to Readability and then Firecrawl (if configured). + ## web_search Search the web using your configured provider. @@ -148,6 +208,7 @@ Search the web using your configured provider. - API key for your chosen provider: - **Brave**: `BRAVE_API_KEY` or `tools.web.search.apiKey` - **Perplexity**: `OPENROUTER_API_KEY`, `PERPLEXITY_API_KEY`, or `tools.web.search.perplexity.apiKey` + - **Parallel**: `PARALLEL_API_KEY` or `tools.web.search.parallel.apiKey` ### Config diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index 9f1e565dd..9e096cf66 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -40,6 +40,7 @@ const DEFAULT_FETCH_MAX_REDIRECTS = 3; const DEFAULT_ERROR_MAX_CHARS = 4_000; const DEFAULT_FIRECRAWL_BASE_URL = "https://api.firecrawl.dev"; const DEFAULT_FIRECRAWL_MAX_AGE_MS = 172_800_000; +const DEFAULT_PARALLEL_BASE_URL = "https://api.parallel.ai"; const DEFAULT_FETCH_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; @@ -78,6 +79,32 @@ type FirecrawlFetchConfig = } | undefined; +type ParallelExtractConfig = + | { + enabled?: boolean; + apiKey?: string; + baseUrl?: string; + timeoutSeconds?: number; + } + | undefined; + +type ParallelExtractResponse = { + extract_id: string; + results: Array<{ + url: string; + title?: string | null; + full_content?: string | null; + excerpts?: string[] | null; + publish_date?: string | null; + }>; + errors: Array<{ + url: string; + error_type: string; + http_status_code?: number | null; + content?: string | null; + }>; +}; + function resolveFetchConfig(cfg?: ClawdbotConfig): WebFetchConfig { const fetch = cfg?.tools?.web?.fetch; if (!fetch || typeof fetch !== "object") return undefined; @@ -147,6 +174,38 @@ function resolveFirecrawlMaxAgeMsOrDefault(firecrawl?: FirecrawlFetchConfig): nu return DEFAULT_FIRECRAWL_MAX_AGE_MS; } +function resolveParallelExtractConfig(fetch?: WebFetchConfig): ParallelExtractConfig { + if (!fetch || typeof fetch !== "object") return undefined; + const parallel = "parallel" in fetch ? fetch.parallel : undefined; + if (!parallel || typeof parallel !== "object") return undefined; + return parallel as ParallelExtractConfig; +} + +function resolveParallelExtractApiKey(parallel?: ParallelExtractConfig): string | undefined { + const fromConfig = + parallel && "apiKey" in parallel && typeof parallel.apiKey === "string" + ? parallel.apiKey.trim() + : ""; + const fromEnv = (process.env.PARALLEL_API_KEY ?? "").trim(); + return fromConfig || fromEnv || undefined; +} + +function resolveParallelExtractEnabled(params: { + parallel?: ParallelExtractConfig; + apiKey?: string; +}): boolean { + if (typeof params.parallel?.enabled === "boolean") return params.parallel.enabled; + return false; +} + +function resolveParallelExtractBaseUrl(parallel?: ParallelExtractConfig): string { + const raw = + parallel && "baseUrl" in parallel && typeof parallel.baseUrl === "string" + ? parallel.baseUrl.trim() + : ""; + return raw || DEFAULT_PARALLEL_BASE_URL; +} + function resolveMaxChars(value: unknown, fallback: number): number { const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; return Math.max(100, Math.floor(parsed)); @@ -329,6 +388,67 @@ export async function fetchFirecrawlContent(params: { }; } +export async function fetchParallelContent(params: { + url: string; + extractMode: ExtractMode; + apiKey: string; + baseUrl: string; + timeoutSeconds: number; +}): Promise<{ + text: string; + title?: string; + finalUrl?: string; + status?: number; +}> { + const endpoint = `${params.baseUrl.replace(/\/$/, "")}/v1beta/extract`; + + const res = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + "parallel-beta": "search-extract-2025-10-10", + }, + body: JSON.stringify({ + urls: [params.url], + full_content: true, + }), + signal: withTimeout(undefined, params.timeoutSeconds * 1000), + }); + + const payload = (await res.json()) as ParallelExtractResponse; + + if (!res.ok) { + const errorDetail = payload?.errors?.[0]; + const detail = errorDetail + ? `${errorDetail.error_type}: ${errorDetail.content || ""}` + : res.statusText; + throw new Error(`Parallel extract failed (${res.status}): ${detail}`.trim()); + } + + if (payload.errors?.length && !payload.results?.length) { + const errorDetail = payload.errors[0]; + throw new Error( + `Parallel extract failed: ${errorDetail?.error_type || "unknown error"}`.trim(), + ); + } + + const result = payload.results?.[0]; + if (!result) { + throw new Error("Parallel extract returned no results"); + } + + const rawText = result.full_content ?? result.excerpts?.join("\n\n") ?? ""; + const text = params.extractMode === "text" ? markdownToText(rawText) : rawText; + + return { + text, + title: result.title ?? undefined, + finalUrl: result.url, + status: 200, + }; +} + async function runWebFetch(params: { url: string; extractMode: ExtractMode; @@ -346,6 +466,10 @@ async function runWebFetch(params: { firecrawlProxy: "auto" | "basic" | "stealth"; firecrawlStoreInCache: boolean; firecrawlTimeoutSeconds: number; + parallelExtractEnabled: boolean; + parallelExtractApiKey?: string; + parallelExtractBaseUrl: string; + parallelExtractTimeoutSeconds: number; }): Promise> { const cacheKey = normalizeCacheKey( `fetch:${params.url}:${params.extractMode}:${params.maxChars}`, @@ -464,7 +588,28 @@ async function runWebFetch(params: { let extractor = "raw"; let text = body; if (contentType.includes("text/html")) { - if (params.readabilityEnabled) { + // Try Parallel extract first if enabled + if (params.parallelExtractEnabled && params.parallelExtractApiKey) { + try { + const parallel = await fetchParallelContent({ + url: finalUrl, + extractMode: params.extractMode, + apiKey: params.parallelExtractApiKey, + baseUrl: params.parallelExtractBaseUrl, + timeoutSeconds: params.parallelExtractTimeoutSeconds, + }); + if (parallel.text) { + text = parallel.text; + title = parallel.title; + extractor = "parallel"; + } + } catch { + // Fall back to Readability/Firecrawl if Parallel fails + } + } + + // If Parallel didn't succeed, try Readability + if (extractor === "raw" && params.readabilityEnabled) { const readable = await extractReadableContent({ html: body, url: finalUrl, @@ -480,13 +625,13 @@ async function runWebFetch(params: { text = firecrawl.text; title = firecrawl.title; extractor = "firecrawl"; - } else { + } else if (!params.parallelExtractEnabled) { throw new Error( "Web fetch extraction failed: Readability and Firecrawl returned no content.", ); } } - } else { + } else if (extractor === "raw" && !params.parallelExtractEnabled) { throw new Error( "Web fetch extraction failed: Readability disabled and Firecrawl unavailable.", ); @@ -586,6 +731,17 @@ export function createWebFetchTool(options?: { firecrawl?.timeoutSeconds ?? fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, ); + const parallelExtract = resolveParallelExtractConfig(fetch); + const parallelExtractApiKey = resolveParallelExtractApiKey(parallelExtract); + const parallelExtractEnabled = resolveParallelExtractEnabled({ + parallel: parallelExtract, + apiKey: parallelExtractApiKey, + }); + const parallelExtractBaseUrl = resolveParallelExtractBaseUrl(parallelExtract); + const parallelExtractTimeoutSeconds = resolveTimeoutSeconds( + parallelExtract?.timeoutSeconds ?? fetch?.timeoutSeconds, + DEFAULT_TIMEOUT_SECONDS, + ); const userAgent = (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) || DEFAULT_FETCH_USER_AGENT; @@ -617,6 +773,10 @@ export function createWebFetchTool(options?: { firecrawlProxy: "auto", firecrawlStoreInCache: true, firecrawlTimeoutSeconds, + parallelExtractEnabled, + parallelExtractApiKey, + parallelExtractBaseUrl, + parallelExtractTimeoutSeconds, }); return jsonResult(result); }, diff --git a/src/agents/tools/web-search.test.ts b/src/agents/tools/web-search.test.ts index c6d2b6405..8779c0530 100644 --- a/src/agents/tools/web-search.test.ts +++ b/src/agents/tools/web-search.test.ts @@ -1,9 +1,14 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { __testing } from "./web-search.js"; -const { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness } = - __testing; +const { + inferPerplexityBaseUrlFromApiKey, + resolvePerplexityBaseUrl, + normalizeFreshness, + resolveParallelApiKey, + resolveParallelBaseUrl, +} = __testing; describe("web_search perplexity baseUrl defaults", () => { it("detects a Perplexity key prefix", () => { @@ -69,3 +74,44 @@ describe("web_search freshness normalization", () => { expect(normalizeFreshness("2024-03-10to2024-03-01")).toBeUndefined(); }); }); + +describe("web_search parallel config resolution", () => { + const originalEnv = process.env; + + beforeEach(() => { + vi.resetModules(); + process.env = { ...originalEnv }; + delete process.env.PARALLEL_API_KEY; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + it("returns config apiKey when provided", () => { + expect(resolveParallelApiKey({ apiKey: "test-key" })).toBe("test-key"); + }); + + it("returns env apiKey when config is empty", () => { + process.env.PARALLEL_API_KEY = "env-key"; + expect(resolveParallelApiKey({})).toBe("env-key"); + }); + + it("returns undefined when no key is available", () => { + expect(resolveParallelApiKey({})).toBeUndefined(); + }); + + it("returns config baseUrl when provided", () => { + expect(resolveParallelBaseUrl({ baseUrl: "https://custom.api.com" })).toBe( + "https://custom.api.com", + ); + }); + + it("returns default baseUrl when config is empty", () => { + expect(resolveParallelBaseUrl({})).toBe("https://api.parallel.ai"); + }); + + it("returns default baseUrl for undefined config", () => { + expect(resolveParallelBaseUrl(undefined)).toBe("https://api.parallel.ai"); + }); +}); diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index 50d3d19a1..89e29515d 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -17,7 +17,7 @@ import { writeCache, } from "./web-shared.js"; -const SEARCH_PROVIDERS = ["brave", "perplexity"] as const; +const SEARCH_PROVIDERS = ["brave", "perplexity", "parallel"] as const; const DEFAULT_SEARCH_COUNT = 5; const MAX_SEARCH_COUNT = 10; @@ -27,6 +27,7 @@ const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro"; const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; +const DEFAULT_PARALLEL_BASE_URL = "https://api.parallel.ai"; const SEARCH_CACHE = new Map>>(); const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]); @@ -103,6 +104,22 @@ type PerplexitySearchResponse = { type PerplexityBaseUrlHint = "direct" | "openrouter"; +type ParallelConfig = { + apiKey?: string; + baseUrl?: string; +}; + +type ParallelSearchResponse = { + search_id: string; + results: Array<{ + url: string; + title?: string | null; + excerpts?: string[] | null; + publish_date?: string | null; + }>; + warnings?: Array<{ message?: string }> | null; +}; + function resolveSearchConfig(cfg?: ClawdbotConfig): WebSearchConfig { const search = cfg?.tools?.web?.search; if (!search || typeof search !== "object") return undefined; @@ -131,6 +148,14 @@ function missingSearchKeyPayload(provider: (typeof SEARCH_PROVIDERS)[number]) { docs: "https://docs.clawd.bot/tools/web", }; } + if (provider === "parallel") { + return { + error: "missing_parallel_api_key", + message: + "web_search (parallel) needs an API key. Set PARALLEL_API_KEY in the Gateway environment, or configure tools.web.search.parallel.apiKey.", + docs: "https://docs.clawd.bot/tools/web", + }; + } return { error: "missing_brave_api_key", message: `web_search needs a Brave Search API key. Run \`${formatCliCommand("clawdbot configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`, @@ -144,6 +169,7 @@ function resolveSearchProvider(search?: WebSearchConfig): (typeof SEARCH_PROVIDE ? search.provider.trim().toLowerCase() : ""; if (raw === "perplexity") return "perplexity"; + if (raw === "parallel") return "parallel"; if (raw === "brave") return "brave"; return "brave"; } @@ -221,6 +247,28 @@ function resolvePerplexityModel(perplexity?: PerplexityConfig): string { return fromConfig || DEFAULT_PERPLEXITY_MODEL; } +function resolveParallelConfig(search?: WebSearchConfig): ParallelConfig { + if (!search || typeof search !== "object") return {}; + const parallel = "parallel" in search ? search.parallel : undefined; + if (!parallel || typeof parallel !== "object") return {}; + return parallel as ParallelConfig; +} + +function resolveParallelApiKey(parallel?: ParallelConfig): string | undefined { + const fromConfig = normalizeApiKey(parallel?.apiKey); + if (fromConfig) return fromConfig; + const fromEnv = normalizeApiKey(process.env.PARALLEL_API_KEY); + return fromEnv || undefined; +} + +function resolveParallelBaseUrl(parallel?: ParallelConfig): string { + const fromConfig = + parallel && "baseUrl" in parallel && typeof parallel.baseUrl === "string" + ? parallel.baseUrl.trim() + : ""; + return fromConfig || DEFAULT_PARALLEL_BASE_URL; +} + function resolveSearchCount(value: unknown, fallback: number): number { const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback; const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed))); @@ -306,6 +354,60 @@ async function runPerplexitySearch(params: { return { content, citations }; } +async function runParallelSearch(params: { + query: string; + count: number; + apiKey: string; + baseUrl: string; + timeoutSeconds: number; +}): Promise<{ + results: Array<{ + title: string; + url: string; + description: string; + published?: string; + siteName?: string; + }>; + searchId: string; +}> { + const endpoint = `${params.baseUrl.replace(/\/$/, "")}/v1beta/search`; + + const res = await fetch(endpoint, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + "parallel-beta": "search-extract-2025-10-10", + }, + body: JSON.stringify({ + objective: params.query, + search_queries: [params.query], + max_results: params.count, + excerpts: { + max_chars_per_result: 500, + }, + mode: "agentic", + }), + signal: withTimeout(undefined, params.timeoutSeconds * 1000), + }); + + if (!res.ok) { + const detail = await readResponseText(res); + throw new Error(`Parallel Search API error (${res.status}): ${detail || res.statusText}`); + } + + const data = (await res.json()) as ParallelSearchResponse; + const results = (data.results ?? []).map((entry) => ({ + title: entry.title ?? "", + url: entry.url ?? "", + description: entry.excerpts?.join("\n\n") ?? "", + published: entry.publish_date ?? undefined, + siteName: resolveSiteName(entry.url ?? ""), + })); + + return { results, searchId: data.search_id }; +} + async function runWebSearch(params: { query: string; count: number; @@ -319,6 +421,7 @@ async function runWebSearch(params: { freshness?: string; perplexityBaseUrl?: string; perplexityModel?: string; + parallelBaseUrl?: string; }): Promise> { const cacheKey = normalizeCacheKey( params.provider === "brave" @@ -351,6 +454,27 @@ async function runWebSearch(params: { return payload; } + if (params.provider === "parallel") { + const { results, searchId } = await runParallelSearch({ + query: params.query, + count: params.count, + apiKey: params.apiKey, + baseUrl: params.parallelBaseUrl ?? DEFAULT_PARALLEL_BASE_URL, + timeoutSeconds: params.timeoutSeconds, + }); + + const payload = { + query: params.query, + provider: params.provider, + searchId, + count: results.length, + tookMs: Date.now() - start, + results, + }; + writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); + return payload; + } + if (params.provider !== "brave") { throw new Error("Unsupported web search provider."); } @@ -415,11 +539,14 @@ export function createWebSearchTool(options?: { const provider = resolveSearchProvider(search); const perplexityConfig = resolvePerplexityConfig(search); + const parallelConfig = resolveParallelConfig(search); const description = provider === "perplexity" ? "Search the web using Perplexity Sonar (direct or via OpenRouter). Returns AI-synthesized answers with citations from real-time web search." - : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research."; + : provider === "parallel" + ? "Search the web using Parallel Search API. Returns relevant excerpts and titles from web search results." + : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research."; return { label: "Web Search", @@ -429,8 +556,14 @@ export function createWebSearchTool(options?: { execute: async (_toolCallId, args) => { const perplexityAuth = provider === "perplexity" ? resolvePerplexityApiKey(perplexityConfig) : undefined; + const parallelApiKey = + provider === "parallel" ? resolveParallelApiKey(parallelConfig) : undefined; const apiKey = - provider === "perplexity" ? perplexityAuth?.apiKey : resolveSearchApiKey(search); + provider === "perplexity" + ? perplexityAuth?.apiKey + : provider === "parallel" + ? parallelApiKey + : resolveSearchApiKey(search); if (!apiKey) { return jsonResult(missingSearchKeyPayload(provider)); @@ -476,6 +609,7 @@ export function createWebSearchTool(options?: { perplexityAuth?.apiKey, ), perplexityModel: resolvePerplexityModel(perplexityConfig), + parallelBaseUrl: resolveParallelBaseUrl(parallelConfig), }); return jsonResult(result); }, @@ -486,4 +620,6 @@ export const __testing = { inferPerplexityBaseUrlFromApiKey, resolvePerplexityBaseUrl, normalizeFreshness, + resolveParallelApiKey, + resolveParallelBaseUrl, } as const; diff --git a/src/commands/configure.shared.ts b/src/commands/configure.shared.ts index bc89529d8..3aa90f775 100644 --- a/src/commands/configure.shared.ts +++ b/src/commands/configure.shared.ts @@ -35,7 +35,7 @@ export const CONFIGURE_SECTION_OPTIONS: Array<{ }> = [ { value: "workspace", label: "Workspace", hint: "Set workspace + sessions" }, { value: "model", label: "Model", hint: "Pick provider + credentials" }, - { value: "web", label: "Web tools", hint: "Configure Brave search + fetch" }, + { value: "web", label: "Web tools", hint: "Configure web search + fetch" }, { value: "gateway", label: "Gateway", hint: "Port, bind, auth, tailscale" }, { value: "daemon", diff --git a/src/commands/configure.wizard.ts b/src/commands/configure.wizard.ts index 7563643e1..b997daedf 100644 --- a/src/commands/configure.wizard.ts +++ b/src/commands/configure.wizard.ts @@ -88,18 +88,43 @@ async function promptChannelMode(runtime: RuntimeEnv): Promise = { + brave: "BRAVE_API_KEY", + perplexity: "PERPLEXITY_API_KEY or OPENROUTER_API_KEY", + parallel: "PARALLEL_API_KEY", +}; + +const PROVIDER_PLACEHOLDERS: Record = { + brave: "BSA...", + perplexity: "pplx-... or sk-or-...", + parallel: "your-parallel-key", +}; + async function promptWebToolsConfig( nextConfig: ClawdbotConfig, runtime: RuntimeEnv, ): Promise { const existingSearch = nextConfig.tools?.web?.search; const existingFetch = nextConfig.tools?.web?.fetch; - const hasSearchKey = Boolean(existingSearch?.apiKey); + const existingProvider = existingSearch?.provider ?? "brave"; + const hasSearchKey = Boolean( + existingSearch?.apiKey || + existingSearch?.perplexity?.apiKey || + existingSearch?.parallel?.apiKey, + ); note( [ "Web search lets your agent look things up online using the `web_search` tool.", - "It requires a Brave Search API key (you can store it in the config or set BRAVE_API_KEY in the Gateway environment).", + "Choose a provider and configure its API key.", "Docs: https://docs.clawd.bot/tools/web", ].join("\n"), "Web search", @@ -107,35 +132,73 @@ async function promptWebToolsConfig( const enableSearch = guardCancel( await confirm({ - message: "Enable web_search (Brave Search)?", + message: "Enable web_search?", initialValue: existingSearch?.enabled ?? hasSearchKey, }), runtime, ); - let nextSearch = { + let nextSearch: NonNullable["web"]>["search"] = { ...existingSearch, enabled: enableSearch, }; if (enableSearch) { + const provider = guardCancel( + await select({ + message: "Search provider", + options: SEARCH_PROVIDER_OPTIONS.map((opt) => ({ + value: opt.value, + label: opt.label, + hint: opt.hint, + })), + initialValue: existingProvider, + }), + runtime, + ) as SearchProvider; + + nextSearch = { ...nextSearch, provider }; + + const envVar = PROVIDER_ENV_VARS[provider]; + const placeholder = PROVIDER_PLACEHOLDERS[provider]; + const currentKey = + provider === "brave" + ? existingSearch?.apiKey + : provider === "perplexity" + ? existingSearch?.perplexity?.apiKey + : existingSearch?.parallel?.apiKey; + const hasCurrentKey = Boolean(currentKey); + const keyInput = guardCancel( await text({ - message: hasSearchKey - ? "Brave Search API key (leave blank to keep current or use BRAVE_API_KEY)" - : "Brave Search API key (paste it here; leave blank to use BRAVE_API_KEY)", - placeholder: hasSearchKey ? "Leave blank to keep current" : "BSA...", + message: hasCurrentKey + ? `API key (leave blank to keep current or use ${envVar})` + : `API key (paste it here; leave blank to use ${envVar})`, + placeholder: hasCurrentKey ? "Leave blank to keep current" : placeholder, }), runtime, ); const key = String(keyInput ?? "").trim(); - if (key) { - nextSearch = { ...nextSearch, apiKey: key }; - } else if (!hasSearchKey) { + + if (key || hasCurrentKey) { + if (provider === "brave") { + nextSearch = { ...nextSearch, apiKey: key || currentKey }; + } else if (provider === "perplexity") { + nextSearch = { + ...nextSearch, + perplexity: { ...existingSearch?.perplexity, apiKey: key || currentKey }, + }; + } else if (provider === "parallel") { + nextSearch = { + ...nextSearch, + parallel: { ...existingSearch?.parallel, apiKey: key || currentKey }, + }; + } + } else { note( [ "No key stored yet, so web_search will stay unavailable.", - "Store a key here or set BRAVE_API_KEY in the Gateway environment.", + `Store a key here or set ${envVar} in the Gateway environment.`, "Docs: https://docs.clawd.bot/tools/web", ].join("\n"), "Web search", diff --git a/src/commands/onboard-non-interactive/local.ts b/src/commands/onboard-non-interactive/local.ts index 8701bac7a..f700ef1e7 100644 --- a/src/commands/onboard-non-interactive/local.ts +++ b/src/commands/onboard-non-interactive/local.ts @@ -125,7 +125,7 @@ export async function runNonInteractiveOnboardingLocal(params: { if (!opts.json) { runtime.log( - `Tip: run \`${formatCliCommand("clawdbot configure --section web")}\` to store your Brave API key for web_search. Docs: https://docs.clawd.bot/tools/web`, + `Tip: run \`${formatCliCommand("clawdbot configure --section web")}\` to configure web_search (Brave, Perplexity, or Parallel). Docs: https://docs.clawd.bot/tools/web`, ); } } diff --git a/src/commands/onboard-non-interactive/remote.ts b/src/commands/onboard-non-interactive/remote.ts index 1426448fc..036a64014 100644 --- a/src/commands/onboard-non-interactive/remote.ts +++ b/src/commands/onboard-non-interactive/remote.ts @@ -47,7 +47,7 @@ export async function runNonInteractiveOnboardingRemote(params: { runtime.log(`Remote gateway: ${remoteUrl}`); runtime.log(`Auth: ${payload.auth}`); runtime.log( - `Tip: run \`${formatCliCommand("clawdbot configure --section web")}\` to store your Brave API key for web_search. Docs: https://docs.clawd.bot/tools/web`, + `Tip: run \`${formatCliCommand("clawdbot configure --section web")}\` to configure web_search (Brave, Perplexity, or Parallel). Docs: https://docs.clawd.bot/tools/web`, ); } } diff --git a/src/config/config.web-search-provider.test.ts b/src/config/config.web-search-provider.test.ts index a91c0f438..0a0e68583 100644 --- a/src/config/config.web-search-provider.test.ts +++ b/src/config/config.web-search-provider.test.ts @@ -22,4 +22,43 @@ describe("web search provider config", () => { expect(res.ok).toBe(true); }); + + it("accepts parallel provider and config", () => { + const res = validateConfigObject({ + tools: { + web: { + search: { + enabled: true, + provider: "parallel", + parallel: { + apiKey: "test-parallel-key", + baseUrl: "https://api.parallel.ai", + }, + }, + }, + }, + }); + + expect(res.ok).toBe(true); + }); + + it("accepts parallel extract config for fetch", () => { + const res = validateConfigObject({ + tools: { + web: { + fetch: { + enabled: true, + parallel: { + enabled: true, + apiKey: "test-parallel-key", + baseUrl: "https://api.parallel.ai", + timeoutSeconds: 30, + }, + }, + }, + }, + }); + + expect(res.ok).toBe(true); + }); }); diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index d84dd1aa7..86de42491 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -332,8 +332,8 @@ export type ToolsConfig = { search?: { /** Enable web search tool (default: true when API key is present). */ enabled?: boolean; - /** Search provider ("brave" or "perplexity"). */ - provider?: "brave" | "perplexity"; + /** Search provider ("brave", "perplexity", or "parallel"). */ + provider?: "brave" | "perplexity" | "parallel"; /** Brave Search API key (optional; defaults to BRAVE_API_KEY env var). */ apiKey?: string; /** Default search results count (1-10). */ @@ -351,6 +351,13 @@ export type ToolsConfig = { /** Model to use (defaults to "perplexity/sonar-pro"). */ model?: string; }; + /** Parallel-specific configuration (used when provider="parallel"). */ + parallel?: { + /** API key for Parallel (defaults to PARALLEL_API_KEY env var). */ + apiKey?: string; + /** Base URL for API requests (defaults to https://api.parallel.ai). */ + baseUrl?: string; + }; }; fetch?: { /** Enable web fetch tool (default: true). */ @@ -381,6 +388,17 @@ export type ToolsConfig = { /** Timeout in seconds for Firecrawl requests. */ timeoutSeconds?: number; }; + /** Parallel extract configuration (alternative to Readability/Firecrawl). */ + parallel?: { + /** Enable Parallel extract (default: false). */ + enabled?: boolean; + /** Parallel API key (defaults to PARALLEL_API_KEY env var). */ + apiKey?: string; + /** Parallel API base URL (defaults to https://api.parallel.ai). */ + baseUrl?: string; + /** Timeout in seconds for Parallel extract requests. */ + timeoutSeconds?: number; + }; }; }; media?: MediaToolsConfig; diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 99074c55e..da4d015ba 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -159,7 +159,8 @@ export const ToolPolicySchema = ToolPolicyBaseSchema.superRefine((value, ctx) => if (value.allow && value.allow.length > 0 && value.alsoAllow && value.alsoAllow.length > 0) { ctx.addIssue({ code: z.ZodIssueCode.custom, - message: "tools policy cannot set both allow and alsoAllow in the same scope (merge alsoAllow into allow, or remove allow and use profile + alsoAllow)", + message: + "tools policy cannot set both allow and alsoAllow in the same scope (merge alsoAllow into allow, or remove allow and use profile + alsoAllow)", }); } }).optional(); @@ -167,7 +168,9 @@ export const ToolPolicySchema = ToolPolicyBaseSchema.superRefine((value, ctx) => export const ToolsWebSearchSchema = z .object({ enabled: z.boolean().optional(), - provider: z.union([z.literal("brave"), z.literal("perplexity")]).optional(), + provider: z + .union([z.literal("brave"), z.literal("perplexity"), z.literal("parallel")]) + .optional(), apiKey: z.string().optional(), maxResults: z.number().int().positive().optional(), timeoutSeconds: z.number().int().positive().optional(), @@ -180,6 +183,13 @@ export const ToolsWebSearchSchema = z }) .strict() .optional(), + parallel: z + .object({ + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + }) + .strict() + .optional(), }) .strict() .optional(); @@ -192,6 +202,15 @@ export const ToolsWebFetchSchema = z cacheTtlMinutes: z.number().nonnegative().optional(), maxRedirects: z.number().int().nonnegative().optional(), userAgent: z.string().optional(), + parallel: z + .object({ + enabled: z.boolean().optional(), + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + timeoutSeconds: z.number().int().positive().optional(), + }) + .strict() + .optional(), }) .strict() .optional(); diff --git a/src/security/audit-extra.ts b/src/security/audit-extra.ts index 9aabb9721..bb38d2d5c 100644 --- a/src/security/audit-extra.ts +++ b/src/security/audit-extra.ts @@ -436,9 +436,11 @@ function hasWebSearchKey(cfg: ClawdbotConfig, env: NodeJS.ProcessEnv): boolean { return Boolean( search?.apiKey || search?.perplexity?.apiKey || + search?.parallel?.apiKey || env.BRAVE_API_KEY || env.PERPLEXITY_API_KEY || - env.OPENROUTER_API_KEY, + env.OPENROUTER_API_KEY || + env.PARALLEL_API_KEY, ); } diff --git a/src/wizard/onboarding.finalize.ts b/src/wizard/onboarding.finalize.ts index 32ab53dcf..b3de32dbb 100644 --- a/src/wizard/onboarding.finalize.ts +++ b/src/wizard/onboarding.finalize.ts @@ -432,8 +432,20 @@ export async function finalizeOnboardingWizard(options: FinalizeOnboardingOption ); } - const webSearchKey = (nextConfig.tools?.web?.search?.apiKey ?? "").trim(); - const webSearchEnv = (process.env.BRAVE_API_KEY ?? "").trim(); + const searchConfig = nextConfig.tools?.web?.search; + const webSearchKey = ( + searchConfig?.apiKey || + searchConfig?.perplexity?.apiKey || + searchConfig?.parallel?.apiKey || + "" + ).trim(); + const webSearchEnv = ( + process.env.BRAVE_API_KEY || + process.env.PERPLEXITY_API_KEY || + process.env.OPENROUTER_API_KEY || + process.env.PARALLEL_API_KEY || + "" + ).trim(); const hasWebSearchKey = Boolean(webSearchKey || webSearchEnv); await prompter.note( hasWebSearchKey @@ -441,20 +453,20 @@ export async function finalizeOnboardingWizard(options: FinalizeOnboardingOption "Web search is enabled, so your agent can look things up online when needed.", "", webSearchKey - ? "API key: stored in config (tools.web.search.apiKey)." - : "API key: provided via BRAVE_API_KEY env var (Gateway environment).", + ? "API key: stored in config." + : "API key: provided via environment variable.", "Docs: https://docs.clawd.bot/tools/web", ].join("\n") : [ - "If you want your agent to be able to search the web, you’ll need an API key.", + "If you want your agent to be able to search the web, you'll need an API key.", "", - "Clawdbot uses Brave Search for the `web_search` tool. Without a Brave Search API key, web search won’t work.", + "Clawdbot supports Brave Search, Perplexity, and Parallel for the `web_search` tool.", "", "Set it up interactively:", `- Run: ${formatCliCommand("clawdbot configure --section web")}`, - "- Enable web_search and paste your Brave Search API key", + "- Choose a provider and paste your API key", "", - "Alternative: set BRAVE_API_KEY in the Gateway environment (no config changes).", + "Alternative: set BRAVE_API_KEY, PERPLEXITY_API_KEY, or PARALLEL_API_KEY in the Gateway environment.", "Docs: https://docs.clawd.bot/tools/web", ].join("\n"), "Web search (optional)", diff --git a/src/wizard/onboarding.test.ts b/src/wizard/onboarding.test.ts index 23e4a6018..838ab938a 100644 --- a/src/wizard/onboarding.test.ts +++ b/src/wizard/onboarding.test.ts @@ -284,7 +284,13 @@ describe("runOnboardingWizard", () => { it("shows the web search hint at the end of onboarding", async () => { const prevBraveKey = process.env.BRAVE_API_KEY; + const prevPerplexityKey = process.env.PERPLEXITY_API_KEY; + const prevOpenRouterKey = process.env.OPENROUTER_API_KEY; + const prevParallelKey = process.env.PARALLEL_API_KEY; delete process.env.BRAVE_API_KEY; + delete process.env.PERPLEXITY_API_KEY; + delete process.env.OPENROUTER_API_KEY; + delete process.env.PARALLEL_API_KEY; try { const note: WizardPrompter["note"] = vi.fn(async () => {}); @@ -329,6 +335,21 @@ describe("runOnboardingWizard", () => { } else { process.env.BRAVE_API_KEY = prevBraveKey; } + if (prevPerplexityKey === undefined) { + delete process.env.PERPLEXITY_API_KEY; + } else { + process.env.PERPLEXITY_API_KEY = prevPerplexityKey; + } + if (prevOpenRouterKey === undefined) { + delete process.env.OPENROUTER_API_KEY; + } else { + process.env.OPENROUTER_API_KEY = prevOpenRouterKey; + } + if (prevParallelKey === undefined) { + delete process.env.PARALLEL_API_KEY; + } else { + process.env.PARALLEL_API_KEY = prevParallelKey; + } } }); });