From 93e700a35348acb849665f287cca2db1eb84a371 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 27 Jan 2026 19:27:12 +0100 Subject: [PATCH] feat(models): add Eden AI model scanning Add scanEdenAiModels() function to scan Eden AI's model registry with support for provider filtering, free model detection, and the {object, data} response format. - Add EDENAI_API_KEY env variable support - Add --source edenai option to `clawdbot models scan` - Add edenai to provider list in models status - Add unit tests for Eden AI scanning Co-Authored-By: Claude Opus 4.5 --- src/agents/model-auth.ts | 1 + src/agents/model-scan.test.ts | 148 +++++++++++++- src/agents/model-scan.ts | 220 ++++++++++++++++++++- src/cli/models-cli.ts | 3 +- src/commands/models/list.status-command.ts | 1 + src/commands/models/scan.ts | 47 +++-- 6 files changed, 405 insertions(+), 15 deletions(-) diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index 1445b53f7..a73f1e172 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -277,6 +277,7 @@ export function resolveEnvApiKey(provider: string): EnvApiKeyResult | null { cerebras: "CEREBRAS_API_KEY", xai: "XAI_API_KEY", openrouter: "OPENROUTER_API_KEY", + edenai: "EDENAI_API_KEY", "vercel-ai-gateway": "AI_GATEWAY_API_KEY", moonshot: "MOONSHOT_API_KEY", "kimi-code": "KIMICODE_API_KEY", diff --git a/src/agents/model-scan.test.ts b/src/agents/model-scan.test.ts index d69445324..5381dbe79 100644 --- a/src/agents/model-scan.test.ts +++ b/src/agents/model-scan.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; -import { scanOpenRouterModels } from "./model-scan.js"; +import { scanEdenAiModels, scanOpenRouterModels } from "./model-scan.js"; function createFetchFixture(payload: unknown): typeof fetch { return async () => @@ -86,3 +86,149 @@ describe("scanOpenRouterModels", () => { } }); }); + +describe("scanEdenAiModels", () => { + it("lists models from {object, data} response format", async () => { + const fetchImpl = createFetchFixture({ + object: "list", + data: [ + { + id: "anthropic/claude-3-haiku", + model_name: "Claude 3 Haiku", + owned_by: "anthropic", + context_length: 200000, + created: 1700000000, + capabilities: { + supports_function_calling: true, + supports_vision: false, + supports_tool_choice: true, + input_modalities: ["text"], + output_modalities: ["text"], + }, + pricing: { input_cost_per_token: 0.00025, output_cost_per_token: 0.00125 }, + }, + ], + }); + + const results = await scanEdenAiModels({ + fetchImpl, + apiKey: "test-key", + probe: false, + }); + + expect(results).toHaveLength(1); + expect(results[0]?.id).toBe("anthropic/claude-3-haiku"); + expect(results[0]?.modelRef).toBe("edenai/anthropic/claude-3-haiku"); + expect(results[0]?.provider).toBe("edenai"); + expect(results[0]?.name).toBe("Claude 3 Haiku"); + expect(results[0]?.supportsToolsMeta).toBe(true); + }); + + it("requires an API key", async () => { + const fetchImpl = createFetchFixture({ object: "list", data: [] }); + const previousKey = process.env.EDENAI_API_KEY; + try { + delete process.env.EDENAI_API_KEY; + await expect(scanEdenAiModels({ fetchImpl, probe: false, apiKey: "" })).rejects.toThrow( + /Missing Eden AI API key/, + ); + } finally { + if (previousKey === undefined) { + delete process.env.EDENAI_API_KEY; + } else { + process.env.EDENAI_API_KEY = previousKey; + } + } + }); + + it("filters by provider", async () => { + const fetchImpl = createFetchFixture({ + object: "list", + data: [ + { + id: "anthropic/claude-3", + model_name: "Claude", + owned_by: "anthropic", + context_length: 200000, + created: 1700000000, + capabilities: { + supports_function_calling: true, + supports_vision: false, + supports_tool_choice: true, + input_modalities: ["text"], + output_modalities: ["text"], + }, + pricing: { input_cost_per_token: 0.001, output_cost_per_token: 0.002 }, + }, + { + id: "openai/gpt-4", + model_name: "GPT-4", + owned_by: "openai", + context_length: 128000, + created: 1700000000, + capabilities: { + supports_function_calling: true, + supports_vision: true, + supports_tool_choice: true, + input_modalities: ["text", "image"], + output_modalities: ["text"], + }, + pricing: { input_cost_per_token: 0.003, output_cost_per_token: 0.006 }, + }, + ], + }); + + const results = await scanEdenAiModels({ + fetchImpl, + apiKey: "test-key", + probe: false, + providerFilter: "anthropic", + }); + + expect(results).toHaveLength(1); + expect(results[0]?.id).toBe("anthropic/claude-3"); + }); + + it("detects free models", async () => { + const fetchImpl = createFetchFixture({ + object: "list", + data: [ + { + id: "free/model", + model_name: "Free Model", + owned_by: "free", + context_length: 4096, + created: 1700000000, + capabilities: { + supports_function_calling: false, + supports_vision: false, + supports_tool_choice: false, + input_modalities: ["text"], + output_modalities: ["text"], + }, + pricing: { input_cost_per_token: 0, output_cost_per_token: 0 }, + }, + { + id: "paid/model", + model_name: "Paid Model", + owned_by: "paid", + context_length: 8192, + created: 1700000000, + capabilities: { + supports_function_calling: true, + supports_vision: false, + supports_tool_choice: true, + input_modalities: ["text"], + output_modalities: ["text"], + }, + pricing: { input_cost_per_token: 0.001, output_cost_per_token: 0.002 }, + }, + ], + }); + + const results = await scanEdenAiModels({ fetchImpl, apiKey: "test-key", probe: false }); + + expect(results.find((r) => r.id === "free/model")?.isFree).toBe(true); + expect(results.find((r) => r.id === "paid/model")?.isFree).toBe(false); + }); +}); diff --git a/src/agents/model-scan.ts b/src/agents/model-scan.ts index ba4775372..f77beefc7 100644 --- a/src/agents/model-scan.ts +++ b/src/agents/model-scan.ts @@ -10,6 +10,7 @@ import { import { Type } from "@sinclair/typebox"; const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"; +const EDENAI_MODELS_URL = "https://api.edenai.run/v3/llm/models"; const DEFAULT_TIMEOUT_MS = 12_000; const DEFAULT_CONCURRENCY = 3; @@ -463,5 +464,220 @@ export async function scanOpenRouterModels( ); } -export { OPENROUTER_MODELS_URL }; -export type { OpenRouterModelMeta, OpenRouterModelPricing }; +// Eden AI model scanning + +type EdenAiModelMeta = { + id: string; + model_name: string; + owned_by: string; + context_length: number | null; + created: number; + capabilities: { + supports_function_calling: boolean; + supports_vision: boolean; + supports_tool_choice: boolean; + input_modalities: string[]; + output_modalities: string[]; + }; + pricing: { + input_cost_per_token: number; + output_cost_per_token: number; + }; +}; + +export type EdenAiScanOptions = { + apiKey?: string; + fetchImpl?: typeof fetch; + timeoutMs?: number; + concurrency?: number; + minParamB?: number; + maxAgeDays?: number; + providerFilter?: string; + probe?: boolean; + onProgress?: (update: { phase: "catalog" | "probe"; completed: number; total: number }) => void; +}; + +function parseEdenAiModality(meta: EdenAiModelMeta): Array<"text" | "image"> { + const modalities = meta.capabilities?.input_modalities ?? []; + const hasImage = modalities.some((m) => m.toLowerCase() === "image"); + return hasImage ? ["text", "image"] : ["text"]; +} + +async function fetchEdenAiModels( + fetchImpl: typeof fetch, + apiKey: string, +): Promise { + const res = await fetchImpl(EDENAI_MODELS_URL, { + headers: { + Accept: "application/json", + Authorization: `Bearer ${apiKey}`, + }, + }); + if (!res.ok) { + throw new Error(`Eden AI /llm/models failed: HTTP ${res.status}`); + } + const payload = (await res.json()) as unknown; + // Eden AI returns {object: "list", data: [...]} format + const entries = Array.isArray(payload) + ? payload + : payload && + typeof payload === "object" && + "data" in payload && + Array.isArray((payload as { data: unknown }).data) + ? (payload as { data: unknown[] }).data + : []; + + return entries + .map((entry) => { + if (!entry || typeof entry !== "object") return null; + const obj = entry as Record; + const id = typeof obj.id === "string" ? obj.id.trim() : ""; + if (!id) return null; + const model_name = + typeof obj.model_name === "string" && obj.model_name.trim() ? obj.model_name.trim() : id; + const owned_by = typeof obj.owned_by === "string" ? obj.owned_by.trim() : ""; + + const context_length = + typeof obj.context_length === "number" && Number.isFinite(obj.context_length) + ? obj.context_length + : null; + + const created = + typeof obj.created === "number" && Number.isFinite(obj.created) ? obj.created : 0; + + const caps = (obj.capabilities ?? {}) as Record; + const capabilities = { + supports_function_calling: caps.supports_function_calling === true, + supports_vision: caps.supports_vision === true, + supports_tool_choice: caps.supports_tool_choice === true, + input_modalities: Array.isArray(caps.input_modalities) + ? caps.input_modalities.filter((m): m is string => typeof m === "string") + : [], + output_modalities: Array.isArray(caps.output_modalities) + ? caps.output_modalities.filter((m): m is string => typeof m === "string") + : [], + }; + + const pricingRaw = (obj.pricing ?? {}) as Record; + const pricing = { + input_cost_per_token: + typeof pricingRaw.input_cost_per_token === "number" ? pricingRaw.input_cost_per_token : 0, + output_cost_per_token: + typeof pricingRaw.output_cost_per_token === "number" + ? pricingRaw.output_cost_per_token + : 0, + }; + + return { + id, + model_name, + owned_by, + context_length, + created, + capabilities, + pricing, + } satisfies EdenAiModelMeta; + }) + .filter((entry): entry is EdenAiModelMeta => Boolean(entry)); +} + +function isFreeEdenAiModel(entry: EdenAiModelMeta): boolean { + return entry.pricing.input_cost_per_token === 0 && entry.pricing.output_cost_per_token === 0; +} + +export async function scanEdenAiModels( + options: EdenAiScanOptions = {}, +): Promise { + const fetchImpl = options.fetchImpl ?? fetch; + const probe = options.probe ?? true; + const apiKey = options.apiKey?.trim() || getEnvApiKey("edenai") || ""; + if (!apiKey) { + throw new Error("Missing Eden AI API key. Set EDENAI_API_KEY to run models scan."); + } + + // timeoutMs reserved for future probing support + const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY)); + const minParamB = Math.max(0, Math.floor(options.minParamB ?? 0)); + const maxAgeDays = Math.max(0, Math.floor(options.maxAgeDays ?? 0)); + const providerFilter = options.providerFilter?.trim().toLowerCase() ?? ""; + + const catalog = await fetchEdenAiModels(fetchImpl, apiKey); + const now = Date.now(); + + const filtered = catalog.filter((entry) => { + if (providerFilter) { + const prefix = entry.id.split("/")[0]?.toLowerCase() ?? ""; + if (prefix !== providerFilter) return false; + } + if (minParamB > 0) { + const params = inferParamBFromIdOrName(`${entry.id} ${entry.model_name}`); + if (!params || params < minParamB) return false; + } + if (maxAgeDays > 0 && entry.created > 0) { + const createdMs = entry.created * 1000; + const ageMs = now - createdMs; + const ageDays = ageMs / (24 * 60 * 60 * 1000); + if (ageDays > maxAgeDays) return false; + } + return true; + }); + + options.onProgress?.({ + phase: "probe", + completed: 0, + total: filtered.length, + }); + + return mapWithConcurrency( + filtered, + concurrency, + async (entry) => { + const isFree = isFreeEdenAiModel(entry); + const inferredParamB = inferParamBFromIdOrName(`${entry.id} ${entry.model_name}`); + const modalities = parseEdenAiModality(entry); + const modalityString = modalities.includes("image") ? "text+image" : "text"; + + const baseResult = { + id: entry.id, + name: entry.model_name, + provider: "edenai", + modelRef: `edenai/${entry.id}`, + contextLength: entry.context_length, + maxCompletionTokens: null, + supportedParametersCount: 0, + supportsToolsMeta: entry.capabilities.supports_function_calling, + modality: modalityString, + inferredParamB, + createdAtMs: entry.created > 0 ? entry.created * 1000 : null, + pricing: { + prompt: entry.pricing.input_cost_per_token, + completion: entry.pricing.output_cost_per_token, + request: 0, + image: 0, + webSearch: 0, + internalReasoning: 0, + }, + isFree, + }; + + // Eden AI probing is not yet supported - their API structure differs from OpenAI + // Use --no-probe for catalog listing, or implement Eden AI-specific probing later + return { + ...baseResult, + tool: { ok: false, latencyMs: null, skipped: !probe }, + image: { ok: false, latencyMs: null, skipped: !probe }, + } satisfies ModelScanResult; + }, + { + onProgress: (completed, total) => + options.onProgress?.({ + phase: "probe", + completed, + total, + }), + }, + ); +} + +export { OPENROUTER_MODELS_URL, EDENAI_MODELS_URL }; +export type { OpenRouterModelMeta, OpenRouterModelPricing, EdenAiModelMeta }; diff --git a/src/cli/models-cli.ts b/src/cli/models-cli.ts index e58c23078..b5f89fd44 100644 --- a/src/cli/models-cli.ts +++ b/src/cli/models-cli.ts @@ -246,7 +246,8 @@ export function registerModelsCli(program: Command) { models .command("scan") - .description("Scan OpenRouter free models for tools + images") + .description("Scan model registries for tools + images") + .option("--source ", "Model registry (openrouter, edenai)", "openrouter") .option("--min-params ", "Minimum parameter size (billions)") .option("--max-age-days ", "Skip models older than N days") .option("--provider ", "Filter by provider prefix") diff --git a/src/commands/models/list.status-command.ts b/src/commands/models/list.status-command.ts index ce1d05489..8dcdff089 100644 --- a/src/commands/models/list.status-command.ts +++ b/src/commands/models/list.status-command.ts @@ -131,6 +131,7 @@ export async function modelsStatusCommand( "cerebras", "xai", "openrouter", + "edenai", "zai", "mistral", "synthetic", diff --git a/src/commands/models/scan.ts b/src/commands/models/scan.ts index abde4877c..e71503171 100644 --- a/src/commands/models/scan.ts +++ b/src/commands/models/scan.ts @@ -1,6 +1,10 @@ import { cancel, multiselect as clackMultiselect, isCancel } from "@clack/prompts"; import { resolveApiKeyForProvider } from "../../agents/model-auth.js"; -import { type ModelScanResult, scanOpenRouterModels } from "../../agents/model-scan.js"; +import { + type ModelScanResult, + scanEdenAiModels, + scanOpenRouterModels, +} from "../../agents/model-scan.js"; import { withProgressTotals } from "../../cli/progress.js"; import { loadConfig } from "../../config/config.js"; import { logConfigUpdated } from "../../config/logging.js"; @@ -120,6 +124,8 @@ function printScanTable(results: ModelScanResult[], runtime: RuntimeEnv) { } } +export type ModelScanSource = "openrouter" | "edenai"; + export async function modelsScanCommand( opts: { minParams?: string; @@ -134,6 +140,7 @@ export async function modelsScanCommand( setImage?: boolean; json?: boolean; probe?: boolean; + source?: ModelScanSource; }, runtime: RuntimeEnv, ) { @@ -160,11 +167,15 @@ export async function modelsScanCommand( const cfg = loadConfig(); const probe = opts.probe ?? true; + const source: ModelScanSource = opts.source ?? "openrouter"; + const providerForKey = source === "edenai" ? "edenai" : "openrouter"; + const sourceLabel = source === "edenai" ? "Eden AI" : "OpenRouter"; + let storedKey: string | undefined; - if (probe) { + if (probe || source === "edenai") { try { const resolved = await resolveApiKeyForProvider({ - provider: "openrouter", + provider: providerForKey, cfg, }); storedKey = resolved.apiKey; @@ -172,14 +183,15 @@ export async function modelsScanCommand( storedKey = undefined; } } + const results = await withProgressTotals( { - label: "Scanning OpenRouter models...", + label: `Scanning ${sourceLabel} models...`, indeterminate: false, enabled: opts.json !== true, }, - async (update) => - await scanOpenRouterModels({ + async (progressUpdate) => { + const scanOpts = { apiKey: storedKey ?? undefined, minParamB: minParams, maxAgeDays, @@ -187,22 +199,35 @@ export async function modelsScanCommand( timeoutMs: timeout, concurrency, probe, - onProgress: ({ phase, completed, total }) => { + onProgress: ({ + phase, + completed, + total, + }: { + phase: string; + completed: number; + total: number; + }) => { if (phase !== "probe") return; const labelBase = probe ? "Probing models" : "Scanning models"; - update({ + progressUpdate({ completed, total, label: `${labelBase} (${completed}/${total})`, }); }, - }), + }; + if (source === "edenai") { + return await scanEdenAiModels(scanOpts); + } + return await scanOpenRouterModels(scanOpts); + }, ); if (!probe) { if (!opts.json) { runtime.log( - `Found ${results.length} OpenRouter free models (metadata only; pass --probe to test tools/images).`, + `Found ${results.length} ${sourceLabel} models (metadata only; pass --probe to test tools/images).`, ); printScanTable(sortScanResults(results), runtime); } else { @@ -213,7 +238,7 @@ export async function modelsScanCommand( const toolOk = results.filter((entry) => entry.tool.ok); if (toolOk.length === 0) { - throw new Error("No tool-capable OpenRouter free models found."); + throw new Error(`No tool-capable ${sourceLabel} models found.`); } const sorted = sortScanResults(results);