feat(tools): add Tavily web search provider
Add Tavily as a third web search provider option alongside Brave and Perplexity. Tavily returns structured search results optimized for AI applications. Configuration: - Set tools.web.search.provider to "tavily" - Provide API key via tools.web.search.tavily.apiKey or TAVILY_API_KEY env
This commit is contained in:
parent
97200984f8
commit
18657bf90c
@ -17,7 +17,7 @@ import {
|
|||||||
writeCache,
|
writeCache,
|
||||||
} from "./web-shared.js";
|
} from "./web-shared.js";
|
||||||
|
|
||||||
const SEARCH_PROVIDERS = ["brave", "perplexity"] as const;
|
const SEARCH_PROVIDERS = ["brave", "perplexity", "tavily"] as const;
|
||||||
const DEFAULT_SEARCH_COUNT = 5;
|
const DEFAULT_SEARCH_COUNT = 5;
|
||||||
const MAX_SEARCH_COUNT = 10;
|
const MAX_SEARCH_COUNT = 10;
|
||||||
|
|
||||||
@ -28,6 +28,8 @@ const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro";
|
|||||||
const PERPLEXITY_KEY_PREFIXES = ["pplx-"];
|
const PERPLEXITY_KEY_PREFIXES = ["pplx-"];
|
||||||
const OPENROUTER_KEY_PREFIXES = ["sk-or-"];
|
const OPENROUTER_KEY_PREFIXES = ["sk-or-"];
|
||||||
|
|
||||||
|
const TAVILY_SEARCH_ENDPOINT = "https://api.tavily.com/search";
|
||||||
|
|
||||||
const SEARCH_CACHE = new Map<string, CacheEntry<Record<string, unknown>>>();
|
const SEARCH_CACHE = new Map<string, CacheEntry<Record<string, unknown>>>();
|
||||||
const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]);
|
const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]);
|
||||||
const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/;
|
const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/;
|
||||||
@ -103,6 +105,21 @@ type PerplexitySearchResponse = {
|
|||||||
|
|
||||||
type PerplexityBaseUrlHint = "direct" | "openrouter";
|
type PerplexityBaseUrlHint = "direct" | "openrouter";
|
||||||
|
|
||||||
|
type TavilyConfig = {
|
||||||
|
apiKey?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type TavilySearchResult = {
|
||||||
|
title?: string;
|
||||||
|
url?: string;
|
||||||
|
content?: string;
|
||||||
|
score?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
type TavilySearchResponse = {
|
||||||
|
results?: TavilySearchResult[];
|
||||||
|
};
|
||||||
|
|
||||||
function resolveSearchConfig(cfg?: ClawdbotConfig): WebSearchConfig {
|
function resolveSearchConfig(cfg?: ClawdbotConfig): WebSearchConfig {
|
||||||
const search = cfg?.tools?.web?.search;
|
const search = cfg?.tools?.web?.search;
|
||||||
if (!search || typeof search !== "object") return undefined;
|
if (!search || typeof search !== "object") return undefined;
|
||||||
@ -131,6 +148,14 @@ function missingSearchKeyPayload(provider: (typeof SEARCH_PROVIDERS)[number]) {
|
|||||||
docs: "https://docs.clawd.bot/tools/web",
|
docs: "https://docs.clawd.bot/tools/web",
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
if (provider === "tavily") {
|
||||||
|
return {
|
||||||
|
error: "missing_tavily_api_key",
|
||||||
|
message:
|
||||||
|
"web_search (tavily) needs an API key. Set TAVILY_API_KEY in the Gateway environment, or configure tools.web.search.tavily.apiKey.",
|
||||||
|
docs: "https://docs.clawd.bot/tools/web",
|
||||||
|
};
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
error: "missing_brave_api_key",
|
error: "missing_brave_api_key",
|
||||||
message: `web_search needs a Brave Search API key. Run \`${formatCliCommand("clawdbot configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`,
|
message: `web_search needs a Brave Search API key. Run \`${formatCliCommand("clawdbot configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`,
|
||||||
@ -144,6 +169,7 @@ function resolveSearchProvider(search?: WebSearchConfig): (typeof SEARCH_PROVIDE
|
|||||||
? search.provider.trim().toLowerCase()
|
? search.provider.trim().toLowerCase()
|
||||||
: "";
|
: "";
|
||||||
if (raw === "perplexity") return "perplexity";
|
if (raw === "perplexity") return "perplexity";
|
||||||
|
if (raw === "tavily") return "tavily";
|
||||||
if (raw === "brave") return "brave";
|
if (raw === "brave") return "brave";
|
||||||
return "brave";
|
return "brave";
|
||||||
}
|
}
|
||||||
@ -221,6 +247,21 @@ function resolvePerplexityModel(perplexity?: PerplexityConfig): string {
|
|||||||
return fromConfig || DEFAULT_PERPLEXITY_MODEL;
|
return fromConfig || DEFAULT_PERPLEXITY_MODEL;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveTavilyConfig(search?: WebSearchConfig): TavilyConfig {
|
||||||
|
if (!search || typeof search !== "object") return {};
|
||||||
|
const tavily = "tavily" in search ? search.tavily : undefined;
|
||||||
|
if (!tavily || typeof tavily !== "object") return {};
|
||||||
|
return tavily as TavilyConfig;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveTavilyApiKey(tavily?: TavilyConfig): string | undefined {
|
||||||
|
const fromConfig = normalizeApiKey(tavily?.apiKey);
|
||||||
|
if (fromConfig) return fromConfig;
|
||||||
|
const fromEnv = normalizeApiKey(process.env.TAVILY_API_KEY);
|
||||||
|
if (fromEnv) return fromEnv;
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
function resolveSearchCount(value: unknown, fallback: number): number {
|
function resolveSearchCount(value: unknown, fallback: number): number {
|
||||||
const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
const parsed = typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
||||||
const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed)));
|
const clamped = Math.max(1, Math.min(MAX_SEARCH_COUNT, Math.floor(parsed)));
|
||||||
@ -306,6 +347,41 @@ async function runPerplexitySearch(params: {
|
|||||||
return { content, citations };
|
return { content, citations };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function runTavilySearch(params: {
|
||||||
|
query: string;
|
||||||
|
count: number;
|
||||||
|
apiKey: string;
|
||||||
|
timeoutSeconds: number;
|
||||||
|
}): Promise<Array<{ title: string; url: string; description: string; siteName?: string }>> {
|
||||||
|
const res = await fetch(TAVILY_SEARCH_ENDPOINT, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
api_key: params.apiKey,
|
||||||
|
query: params.query,
|
||||||
|
max_results: params.count,
|
||||||
|
include_answer: false,
|
||||||
|
}),
|
||||||
|
signal: withTimeout(undefined, params.timeoutSeconds * 1000),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
const detail = await readResponseText(res);
|
||||||
|
throw new Error(`Tavily API error (${res.status}): ${detail || res.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await res.json()) as TavilySearchResponse;
|
||||||
|
const results = Array.isArray(data.results) ? data.results : [];
|
||||||
|
return results.map((entry) => ({
|
||||||
|
title: entry.title ?? "",
|
||||||
|
url: entry.url ?? "",
|
||||||
|
description: entry.content ?? "",
|
||||||
|
siteName: resolveSiteName(entry.url ?? ""),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
async function runWebSearch(params: {
|
async function runWebSearch(params: {
|
||||||
query: string;
|
query: string;
|
||||||
count: number;
|
count: number;
|
||||||
@ -351,6 +427,25 @@ async function runWebSearch(params: {
|
|||||||
return payload;
|
return payload;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (params.provider === "tavily") {
|
||||||
|
const results = await runTavilySearch({
|
||||||
|
query: params.query,
|
||||||
|
count: params.count,
|
||||||
|
apiKey: params.apiKey,
|
||||||
|
timeoutSeconds: params.timeoutSeconds,
|
||||||
|
});
|
||||||
|
|
||||||
|
const payload = {
|
||||||
|
query: params.query,
|
||||||
|
provider: params.provider,
|
||||||
|
count: results.length,
|
||||||
|
tookMs: Date.now() - start,
|
||||||
|
results,
|
||||||
|
};
|
||||||
|
writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs);
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
|
||||||
if (params.provider !== "brave") {
|
if (params.provider !== "brave") {
|
||||||
throw new Error("Unsupported web search provider.");
|
throw new Error("Unsupported web search provider.");
|
||||||
}
|
}
|
||||||
@ -415,11 +510,14 @@ export function createWebSearchTool(options?: {
|
|||||||
|
|
||||||
const provider = resolveSearchProvider(search);
|
const provider = resolveSearchProvider(search);
|
||||||
const perplexityConfig = resolvePerplexityConfig(search);
|
const perplexityConfig = resolvePerplexityConfig(search);
|
||||||
|
const tavilyConfig = resolveTavilyConfig(search);
|
||||||
|
|
||||||
const description =
|
const description =
|
||||||
provider === "perplexity"
|
provider === "perplexity"
|
||||||
? "Search the web using Perplexity Sonar (direct or via OpenRouter). Returns AI-synthesized answers with citations from real-time web search."
|
? "Search the web using Perplexity Sonar (direct or via OpenRouter). Returns AI-synthesized answers with citations from real-time web search."
|
||||||
: "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.";
|
: provider === "tavily"
|
||||||
|
? "Search the web using Tavily Search API. Returns high-quality search results optimized for AI applications."
|
||||||
|
: "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.";
|
||||||
|
|
||||||
return {
|
return {
|
||||||
label: "Web Search",
|
label: "Web Search",
|
||||||
@ -430,7 +528,11 @@ export function createWebSearchTool(options?: {
|
|||||||
const perplexityAuth =
|
const perplexityAuth =
|
||||||
provider === "perplexity" ? resolvePerplexityApiKey(perplexityConfig) : undefined;
|
provider === "perplexity" ? resolvePerplexityApiKey(perplexityConfig) : undefined;
|
||||||
const apiKey =
|
const apiKey =
|
||||||
provider === "perplexity" ? perplexityAuth?.apiKey : resolveSearchApiKey(search);
|
provider === "perplexity"
|
||||||
|
? perplexityAuth?.apiKey
|
||||||
|
: provider === "tavily"
|
||||||
|
? resolveTavilyApiKey(tavilyConfig)
|
||||||
|
: resolveSearchApiKey(search);
|
||||||
|
|
||||||
if (!apiKey) {
|
if (!apiKey) {
|
||||||
return jsonResult(missingSearchKeyPayload(provider));
|
return jsonResult(missingSearchKeyPayload(provider));
|
||||||
|
|||||||
@ -22,4 +22,22 @@ describe("web search provider config", () => {
|
|||||||
|
|
||||||
expect(res.ok).toBe(true);
|
expect(res.ok).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("accepts tavily provider and config", () => {
|
||||||
|
const res = validateConfigObject({
|
||||||
|
tools: {
|
||||||
|
web: {
|
||||||
|
search: {
|
||||||
|
enabled: true,
|
||||||
|
provider: "tavily",
|
||||||
|
tavily: {
|
||||||
|
apiKey: "tvly-test-key",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(res.ok).toBe(true);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@ -191,6 +191,7 @@ const FIELD_LABELS: Record<string, string> = {
|
|||||||
"tools.web.search.maxResults": "Web Search Max Results",
|
"tools.web.search.maxResults": "Web Search Max Results",
|
||||||
"tools.web.search.timeoutSeconds": "Web Search Timeout (sec)",
|
"tools.web.search.timeoutSeconds": "Web Search Timeout (sec)",
|
||||||
"tools.web.search.cacheTtlMinutes": "Web Search Cache TTL (min)",
|
"tools.web.search.cacheTtlMinutes": "Web Search Cache TTL (min)",
|
||||||
|
"tools.web.search.tavily.apiKey": "Tavily API Key",
|
||||||
"tools.web.fetch.enabled": "Enable Web Fetch Tool",
|
"tools.web.fetch.enabled": "Enable Web Fetch Tool",
|
||||||
"tools.web.fetch.maxChars": "Web Fetch Max Chars",
|
"tools.web.fetch.maxChars": "Web Fetch Max Chars",
|
||||||
"tools.web.fetch.timeoutSeconds": "Web Fetch Timeout (sec)",
|
"tools.web.fetch.timeoutSeconds": "Web Fetch Timeout (sec)",
|
||||||
@ -426,7 +427,7 @@ const FIELD_HELP: Record<string, string> = {
|
|||||||
'Text suffix for cross-context markers (supports "{channel}").',
|
'Text suffix for cross-context markers (supports "{channel}").',
|
||||||
"tools.message.broadcast.enabled": "Enable broadcast action (default: true).",
|
"tools.message.broadcast.enabled": "Enable broadcast action (default: true).",
|
||||||
"tools.web.search.enabled": "Enable the web_search tool (requires a provider API key).",
|
"tools.web.search.enabled": "Enable the web_search tool (requires a provider API key).",
|
||||||
"tools.web.search.provider": 'Search provider ("brave" or "perplexity").',
|
"tools.web.search.provider": 'Search provider ("brave", "perplexity", or "tavily").',
|
||||||
"tools.web.search.apiKey": "Brave Search API key (fallback: BRAVE_API_KEY env var).",
|
"tools.web.search.apiKey": "Brave Search API key (fallback: BRAVE_API_KEY env var).",
|
||||||
"tools.web.search.maxResults": "Default number of results to return (1-10).",
|
"tools.web.search.maxResults": "Default number of results to return (1-10).",
|
||||||
"tools.web.search.timeoutSeconds": "Timeout in seconds for web_search requests.",
|
"tools.web.search.timeoutSeconds": "Timeout in seconds for web_search requests.",
|
||||||
@ -437,6 +438,7 @@ const FIELD_HELP: Record<string, string> = {
|
|||||||
"Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).",
|
"Perplexity base URL override (default: https://openrouter.ai/api/v1 or https://api.perplexity.ai).",
|
||||||
"tools.web.search.perplexity.model":
|
"tools.web.search.perplexity.model":
|
||||||
'Perplexity model override (default: "perplexity/sonar-pro").',
|
'Perplexity model override (default: "perplexity/sonar-pro").',
|
||||||
|
"tools.web.search.tavily.apiKey": "Tavily API key (fallback: TAVILY_API_KEY env var).",
|
||||||
"tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).",
|
"tools.web.fetch.enabled": "Enable the web_fetch tool (lightweight HTTP fetch).",
|
||||||
"tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).",
|
"tools.web.fetch.maxChars": "Max characters returned by web_fetch (truncated).",
|
||||||
"tools.web.fetch.timeoutSeconds": "Timeout in seconds for web_fetch requests.",
|
"tools.web.fetch.timeoutSeconds": "Timeout in seconds for web_fetch requests.",
|
||||||
|
|||||||
@ -319,8 +319,8 @@ export type ToolsConfig = {
|
|||||||
search?: {
|
search?: {
|
||||||
/** Enable web search tool (default: true when API key is present). */
|
/** Enable web search tool (default: true when API key is present). */
|
||||||
enabled?: boolean;
|
enabled?: boolean;
|
||||||
/** Search provider ("brave" or "perplexity"). */
|
/** Search provider ("brave", "perplexity", or "tavily"). */
|
||||||
provider?: "brave" | "perplexity";
|
provider?: "brave" | "perplexity" | "tavily";
|
||||||
/** Brave Search API key (optional; defaults to BRAVE_API_KEY env var). */
|
/** Brave Search API key (optional; defaults to BRAVE_API_KEY env var). */
|
||||||
apiKey?: string;
|
apiKey?: string;
|
||||||
/** Default search results count (1-10). */
|
/** Default search results count (1-10). */
|
||||||
@ -338,6 +338,11 @@ export type ToolsConfig = {
|
|||||||
/** Model to use (defaults to "perplexity/sonar-pro"). */
|
/** Model to use (defaults to "perplexity/sonar-pro"). */
|
||||||
model?: string;
|
model?: string;
|
||||||
};
|
};
|
||||||
|
/** Tavily-specific configuration (used when provider="tavily"). */
|
||||||
|
tavily?: {
|
||||||
|
/** API key for Tavily (defaults to TAVILY_API_KEY env var). */
|
||||||
|
apiKey?: string;
|
||||||
|
};
|
||||||
};
|
};
|
||||||
fetch?: {
|
fetch?: {
|
||||||
/** Enable web fetch tool (default: true). */
|
/** Enable web fetch tool (default: true). */
|
||||||
|
|||||||
@ -158,7 +158,9 @@ export const ToolPolicySchema = z
|
|||||||
export const ToolsWebSearchSchema = z
|
export const ToolsWebSearchSchema = z
|
||||||
.object({
|
.object({
|
||||||
enabled: z.boolean().optional(),
|
enabled: z.boolean().optional(),
|
||||||
provider: z.union([z.literal("brave"), z.literal("perplexity")]).optional(),
|
provider: z
|
||||||
|
.union([z.literal("brave"), z.literal("perplexity"), z.literal("tavily")])
|
||||||
|
.optional(),
|
||||||
apiKey: z.string().optional(),
|
apiKey: z.string().optional(),
|
||||||
maxResults: z.number().int().positive().optional(),
|
maxResults: z.number().int().positive().optional(),
|
||||||
timeoutSeconds: z.number().int().positive().optional(),
|
timeoutSeconds: z.number().int().positive().optional(),
|
||||||
@ -171,6 +173,12 @@ export const ToolsWebSearchSchema = z
|
|||||||
})
|
})
|
||||||
.strict()
|
.strict()
|
||||||
.optional(),
|
.optional(),
|
||||||
|
tavily: z
|
||||||
|
.object({
|
||||||
|
apiKey: z.string().optional(),
|
||||||
|
})
|
||||||
|
.strict()
|
||||||
|
.optional(),
|
||||||
})
|
})
|
||||||
.strict()
|
.strict()
|
||||||
.optional();
|
.optional();
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user