feat(models): add Eden AI model scanning
Add scanEdenAiModels() function to scan Eden AI's model registry with
support for provider filtering, free model detection, and the
{object, data} response format.
- Add EDENAI_API_KEY env variable support
- Add --source edenai option to `clawdbot models scan`
- Add edenai to provider list in models status
- Add unit tests for Eden AI scanning
Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
sam
parent
6af205a13a
commit
93e700a353
@ -277,6 +277,7 @@ export function resolveEnvApiKey(provider: string): EnvApiKeyResult | null {
|
||||
cerebras: "CEREBRAS_API_KEY",
|
||||
xai: "XAI_API_KEY",
|
||||
openrouter: "OPENROUTER_API_KEY",
|
||||
edenai: "EDENAI_API_KEY",
|
||||
"vercel-ai-gateway": "AI_GATEWAY_API_KEY",
|
||||
moonshot: "MOONSHOT_API_KEY",
|
||||
"kimi-code": "KIMICODE_API_KEY",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { scanOpenRouterModels } from "./model-scan.js";
|
||||
import { scanEdenAiModels, scanOpenRouterModels } from "./model-scan.js";
|
||||
|
||||
function createFetchFixture(payload: unknown): typeof fetch {
|
||||
return async () =>
|
||||
@ -86,3 +86,149 @@ describe("scanOpenRouterModels", () => {
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("scanEdenAiModels", () => {
|
||||
it("lists models from {object, data} response format", async () => {
|
||||
const fetchImpl = createFetchFixture({
|
||||
object: "list",
|
||||
data: [
|
||||
{
|
||||
id: "anthropic/claude-3-haiku",
|
||||
model_name: "Claude 3 Haiku",
|
||||
owned_by: "anthropic",
|
||||
context_length: 200000,
|
||||
created: 1700000000,
|
||||
capabilities: {
|
||||
supports_function_calling: true,
|
||||
supports_vision: false,
|
||||
supports_tool_choice: true,
|
||||
input_modalities: ["text"],
|
||||
output_modalities: ["text"],
|
||||
},
|
||||
pricing: { input_cost_per_token: 0.00025, output_cost_per_token: 0.00125 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const results = await scanEdenAiModels({
|
||||
fetchImpl,
|
||||
apiKey: "test-key",
|
||||
probe: false,
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.id).toBe("anthropic/claude-3-haiku");
|
||||
expect(results[0]?.modelRef).toBe("edenai/anthropic/claude-3-haiku");
|
||||
expect(results[0]?.provider).toBe("edenai");
|
||||
expect(results[0]?.name).toBe("Claude 3 Haiku");
|
||||
expect(results[0]?.supportsToolsMeta).toBe(true);
|
||||
});
|
||||
|
||||
it("requires an API key", async () => {
|
||||
const fetchImpl = createFetchFixture({ object: "list", data: [] });
|
||||
const previousKey = process.env.EDENAI_API_KEY;
|
||||
try {
|
||||
delete process.env.EDENAI_API_KEY;
|
||||
await expect(scanEdenAiModels({ fetchImpl, probe: false, apiKey: "" })).rejects.toThrow(
|
||||
/Missing Eden AI API key/,
|
||||
);
|
||||
} finally {
|
||||
if (previousKey === undefined) {
|
||||
delete process.env.EDENAI_API_KEY;
|
||||
} else {
|
||||
process.env.EDENAI_API_KEY = previousKey;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("filters by provider", async () => {
|
||||
const fetchImpl = createFetchFixture({
|
||||
object: "list",
|
||||
data: [
|
||||
{
|
||||
id: "anthropic/claude-3",
|
||||
model_name: "Claude",
|
||||
owned_by: "anthropic",
|
||||
context_length: 200000,
|
||||
created: 1700000000,
|
||||
capabilities: {
|
||||
supports_function_calling: true,
|
||||
supports_vision: false,
|
||||
supports_tool_choice: true,
|
||||
input_modalities: ["text"],
|
||||
output_modalities: ["text"],
|
||||
},
|
||||
pricing: { input_cost_per_token: 0.001, output_cost_per_token: 0.002 },
|
||||
},
|
||||
{
|
||||
id: "openai/gpt-4",
|
||||
model_name: "GPT-4",
|
||||
owned_by: "openai",
|
||||
context_length: 128000,
|
||||
created: 1700000000,
|
||||
capabilities: {
|
||||
supports_function_calling: true,
|
||||
supports_vision: true,
|
||||
supports_tool_choice: true,
|
||||
input_modalities: ["text", "image"],
|
||||
output_modalities: ["text"],
|
||||
},
|
||||
pricing: { input_cost_per_token: 0.003, output_cost_per_token: 0.006 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const results = await scanEdenAiModels({
|
||||
fetchImpl,
|
||||
apiKey: "test-key",
|
||||
probe: false,
|
||||
providerFilter: "anthropic",
|
||||
});
|
||||
|
||||
expect(results).toHaveLength(1);
|
||||
expect(results[0]?.id).toBe("anthropic/claude-3");
|
||||
});
|
||||
|
||||
it("detects free models", async () => {
|
||||
const fetchImpl = createFetchFixture({
|
||||
object: "list",
|
||||
data: [
|
||||
{
|
||||
id: "free/model",
|
||||
model_name: "Free Model",
|
||||
owned_by: "free",
|
||||
context_length: 4096,
|
||||
created: 1700000000,
|
||||
capabilities: {
|
||||
supports_function_calling: false,
|
||||
supports_vision: false,
|
||||
supports_tool_choice: false,
|
||||
input_modalities: ["text"],
|
||||
output_modalities: ["text"],
|
||||
},
|
||||
pricing: { input_cost_per_token: 0, output_cost_per_token: 0 },
|
||||
},
|
||||
{
|
||||
id: "paid/model",
|
||||
model_name: "Paid Model",
|
||||
owned_by: "paid",
|
||||
context_length: 8192,
|
||||
created: 1700000000,
|
||||
capabilities: {
|
||||
supports_function_calling: true,
|
||||
supports_vision: false,
|
||||
supports_tool_choice: true,
|
||||
input_modalities: ["text"],
|
||||
output_modalities: ["text"],
|
||||
},
|
||||
pricing: { input_cost_per_token: 0.001, output_cost_per_token: 0.002 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const results = await scanEdenAiModels({ fetchImpl, apiKey: "test-key", probe: false });
|
||||
|
||||
expect(results.find((r) => r.id === "free/model")?.isFree).toBe(true);
|
||||
expect(results.find((r) => r.id === "paid/model")?.isFree).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@ -10,6 +10,7 @@ import {
|
||||
import { Type } from "@sinclair/typebox";
|
||||
|
||||
const OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models";
|
||||
const EDENAI_MODELS_URL = "https://api.edenai.run/v3/llm/models";
|
||||
const DEFAULT_TIMEOUT_MS = 12_000;
|
||||
const DEFAULT_CONCURRENCY = 3;
|
||||
|
||||
@ -463,5 +464,220 @@ export async function scanOpenRouterModels(
|
||||
);
|
||||
}
|
||||
|
||||
export { OPENROUTER_MODELS_URL };
|
||||
export type { OpenRouterModelMeta, OpenRouterModelPricing };
|
||||
// Eden AI model scanning
|
||||
|
||||
type EdenAiModelMeta = {
|
||||
id: string;
|
||||
model_name: string;
|
||||
owned_by: string;
|
||||
context_length: number | null;
|
||||
created: number;
|
||||
capabilities: {
|
||||
supports_function_calling: boolean;
|
||||
supports_vision: boolean;
|
||||
supports_tool_choice: boolean;
|
||||
input_modalities: string[];
|
||||
output_modalities: string[];
|
||||
};
|
||||
pricing: {
|
||||
input_cost_per_token: number;
|
||||
output_cost_per_token: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type EdenAiScanOptions = {
|
||||
apiKey?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
timeoutMs?: number;
|
||||
concurrency?: number;
|
||||
minParamB?: number;
|
||||
maxAgeDays?: number;
|
||||
providerFilter?: string;
|
||||
probe?: boolean;
|
||||
onProgress?: (update: { phase: "catalog" | "probe"; completed: number; total: number }) => void;
|
||||
};
|
||||
|
||||
function parseEdenAiModality(meta: EdenAiModelMeta): Array<"text" | "image"> {
|
||||
const modalities = meta.capabilities?.input_modalities ?? [];
|
||||
const hasImage = modalities.some((m) => m.toLowerCase() === "image");
|
||||
return hasImage ? ["text", "image"] : ["text"];
|
||||
}
|
||||
|
||||
async function fetchEdenAiModels(
|
||||
fetchImpl: typeof fetch,
|
||||
apiKey: string,
|
||||
): Promise<EdenAiModelMeta[]> {
|
||||
const res = await fetchImpl(EDENAI_MODELS_URL, {
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
Authorization: `Bearer ${apiKey}`,
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
throw new Error(`Eden AI /llm/models failed: HTTP ${res.status}`);
|
||||
}
|
||||
const payload = (await res.json()) as unknown;
|
||||
// Eden AI returns {object: "list", data: [...]} format
|
||||
const entries = Array.isArray(payload)
|
||||
? payload
|
||||
: payload &&
|
||||
typeof payload === "object" &&
|
||||
"data" in payload &&
|
||||
Array.isArray((payload as { data: unknown }).data)
|
||||
? (payload as { data: unknown[] }).data
|
||||
: [];
|
||||
|
||||
return entries
|
||||
.map((entry) => {
|
||||
if (!entry || typeof entry !== "object") return null;
|
||||
const obj = entry as Record<string, unknown>;
|
||||
const id = typeof obj.id === "string" ? obj.id.trim() : "";
|
||||
if (!id) return null;
|
||||
const model_name =
|
||||
typeof obj.model_name === "string" && obj.model_name.trim() ? obj.model_name.trim() : id;
|
||||
const owned_by = typeof obj.owned_by === "string" ? obj.owned_by.trim() : "";
|
||||
|
||||
const context_length =
|
||||
typeof obj.context_length === "number" && Number.isFinite(obj.context_length)
|
||||
? obj.context_length
|
||||
: null;
|
||||
|
||||
const created =
|
||||
typeof obj.created === "number" && Number.isFinite(obj.created) ? obj.created : 0;
|
||||
|
||||
const caps = (obj.capabilities ?? {}) as Record<string, unknown>;
|
||||
const capabilities = {
|
||||
supports_function_calling: caps.supports_function_calling === true,
|
||||
supports_vision: caps.supports_vision === true,
|
||||
supports_tool_choice: caps.supports_tool_choice === true,
|
||||
input_modalities: Array.isArray(caps.input_modalities)
|
||||
? caps.input_modalities.filter((m): m is string => typeof m === "string")
|
||||
: [],
|
||||
output_modalities: Array.isArray(caps.output_modalities)
|
||||
? caps.output_modalities.filter((m): m is string => typeof m === "string")
|
||||
: [],
|
||||
};
|
||||
|
||||
const pricingRaw = (obj.pricing ?? {}) as Record<string, unknown>;
|
||||
const pricing = {
|
||||
input_cost_per_token:
|
||||
typeof pricingRaw.input_cost_per_token === "number" ? pricingRaw.input_cost_per_token : 0,
|
||||
output_cost_per_token:
|
||||
typeof pricingRaw.output_cost_per_token === "number"
|
||||
? pricingRaw.output_cost_per_token
|
||||
: 0,
|
||||
};
|
||||
|
||||
return {
|
||||
id,
|
||||
model_name,
|
||||
owned_by,
|
||||
context_length,
|
||||
created,
|
||||
capabilities,
|
||||
pricing,
|
||||
} satisfies EdenAiModelMeta;
|
||||
})
|
||||
.filter((entry): entry is EdenAiModelMeta => Boolean(entry));
|
||||
}
|
||||
|
||||
function isFreeEdenAiModel(entry: EdenAiModelMeta): boolean {
|
||||
return entry.pricing.input_cost_per_token === 0 && entry.pricing.output_cost_per_token === 0;
|
||||
}
|
||||
|
||||
export async function scanEdenAiModels(
|
||||
options: EdenAiScanOptions = {},
|
||||
): Promise<ModelScanResult[]> {
|
||||
const fetchImpl = options.fetchImpl ?? fetch;
|
||||
const probe = options.probe ?? true;
|
||||
const apiKey = options.apiKey?.trim() || getEnvApiKey("edenai") || "";
|
||||
if (!apiKey) {
|
||||
throw new Error("Missing Eden AI API key. Set EDENAI_API_KEY to run models scan.");
|
||||
}
|
||||
|
||||
// timeoutMs reserved for future probing support
|
||||
const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY));
|
||||
const minParamB = Math.max(0, Math.floor(options.minParamB ?? 0));
|
||||
const maxAgeDays = Math.max(0, Math.floor(options.maxAgeDays ?? 0));
|
||||
const providerFilter = options.providerFilter?.trim().toLowerCase() ?? "";
|
||||
|
||||
const catalog = await fetchEdenAiModels(fetchImpl, apiKey);
|
||||
const now = Date.now();
|
||||
|
||||
const filtered = catalog.filter((entry) => {
|
||||
if (providerFilter) {
|
||||
const prefix = entry.id.split("/")[0]?.toLowerCase() ?? "";
|
||||
if (prefix !== providerFilter) return false;
|
||||
}
|
||||
if (minParamB > 0) {
|
||||
const params = inferParamBFromIdOrName(`${entry.id} ${entry.model_name}`);
|
||||
if (!params || params < minParamB) return false;
|
||||
}
|
||||
if (maxAgeDays > 0 && entry.created > 0) {
|
||||
const createdMs = entry.created * 1000;
|
||||
const ageMs = now - createdMs;
|
||||
const ageDays = ageMs / (24 * 60 * 60 * 1000);
|
||||
if (ageDays > maxAgeDays) return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
options.onProgress?.({
|
||||
phase: "probe",
|
||||
completed: 0,
|
||||
total: filtered.length,
|
||||
});
|
||||
|
||||
return mapWithConcurrency(
|
||||
filtered,
|
||||
concurrency,
|
||||
async (entry) => {
|
||||
const isFree = isFreeEdenAiModel(entry);
|
||||
const inferredParamB = inferParamBFromIdOrName(`${entry.id} ${entry.model_name}`);
|
||||
const modalities = parseEdenAiModality(entry);
|
||||
const modalityString = modalities.includes("image") ? "text+image" : "text";
|
||||
|
||||
const baseResult = {
|
||||
id: entry.id,
|
||||
name: entry.model_name,
|
||||
provider: "edenai",
|
||||
modelRef: `edenai/${entry.id}`,
|
||||
contextLength: entry.context_length,
|
||||
maxCompletionTokens: null,
|
||||
supportedParametersCount: 0,
|
||||
supportsToolsMeta: entry.capabilities.supports_function_calling,
|
||||
modality: modalityString,
|
||||
inferredParamB,
|
||||
createdAtMs: entry.created > 0 ? entry.created * 1000 : null,
|
||||
pricing: {
|
||||
prompt: entry.pricing.input_cost_per_token,
|
||||
completion: entry.pricing.output_cost_per_token,
|
||||
request: 0,
|
||||
image: 0,
|
||||
webSearch: 0,
|
||||
internalReasoning: 0,
|
||||
},
|
||||
isFree,
|
||||
};
|
||||
|
||||
// Eden AI probing is not yet supported - their API structure differs from OpenAI
|
||||
// Use --no-probe for catalog listing, or implement Eden AI-specific probing later
|
||||
return {
|
||||
...baseResult,
|
||||
tool: { ok: false, latencyMs: null, skipped: !probe },
|
||||
image: { ok: false, latencyMs: null, skipped: !probe },
|
||||
} satisfies ModelScanResult;
|
||||
},
|
||||
{
|
||||
onProgress: (completed, total) =>
|
||||
options.onProgress?.({
|
||||
phase: "probe",
|
||||
completed,
|
||||
total,
|
||||
}),
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
export { OPENROUTER_MODELS_URL, EDENAI_MODELS_URL };
|
||||
export type { OpenRouterModelMeta, OpenRouterModelPricing, EdenAiModelMeta };
|
||||
|
||||
@ -246,7 +246,8 @@ export function registerModelsCli(program: Command) {
|
||||
|
||||
models
|
||||
.command("scan")
|
||||
.description("Scan OpenRouter free models for tools + images")
|
||||
.description("Scan model registries for tools + images")
|
||||
.option("--source <registry>", "Model registry (openrouter, edenai)", "openrouter")
|
||||
.option("--min-params <b>", "Minimum parameter size (billions)")
|
||||
.option("--max-age-days <days>", "Skip models older than N days")
|
||||
.option("--provider <name>", "Filter by provider prefix")
|
||||
|
||||
@ -131,6 +131,7 @@ export async function modelsStatusCommand(
|
||||
"cerebras",
|
||||
"xai",
|
||||
"openrouter",
|
||||
"edenai",
|
||||
"zai",
|
||||
"mistral",
|
||||
"synthetic",
|
||||
|
||||
@ -1,6 +1,10 @@
|
||||
import { cancel, multiselect as clackMultiselect, isCancel } from "@clack/prompts";
|
||||
import { resolveApiKeyForProvider } from "../../agents/model-auth.js";
|
||||
import { type ModelScanResult, scanOpenRouterModels } from "../../agents/model-scan.js";
|
||||
import {
|
||||
type ModelScanResult,
|
||||
scanEdenAiModels,
|
||||
scanOpenRouterModels,
|
||||
} from "../../agents/model-scan.js";
|
||||
import { withProgressTotals } from "../../cli/progress.js";
|
||||
import { loadConfig } from "../../config/config.js";
|
||||
import { logConfigUpdated } from "../../config/logging.js";
|
||||
@ -120,6 +124,8 @@ function printScanTable(results: ModelScanResult[], runtime: RuntimeEnv) {
|
||||
}
|
||||
}
|
||||
|
||||
export type ModelScanSource = "openrouter" | "edenai";
|
||||
|
||||
export async function modelsScanCommand(
|
||||
opts: {
|
||||
minParams?: string;
|
||||
@ -134,6 +140,7 @@ export async function modelsScanCommand(
|
||||
setImage?: boolean;
|
||||
json?: boolean;
|
||||
probe?: boolean;
|
||||
source?: ModelScanSource;
|
||||
},
|
||||
runtime: RuntimeEnv,
|
||||
) {
|
||||
@ -160,11 +167,15 @@ export async function modelsScanCommand(
|
||||
|
||||
const cfg = loadConfig();
|
||||
const probe = opts.probe ?? true;
|
||||
const source: ModelScanSource = opts.source ?? "openrouter";
|
||||
const providerForKey = source === "edenai" ? "edenai" : "openrouter";
|
||||
const sourceLabel = source === "edenai" ? "Eden AI" : "OpenRouter";
|
||||
|
||||
let storedKey: string | undefined;
|
||||
if (probe) {
|
||||
if (probe || source === "edenai") {
|
||||
try {
|
||||
const resolved = await resolveApiKeyForProvider({
|
||||
provider: "openrouter",
|
||||
provider: providerForKey,
|
||||
cfg,
|
||||
});
|
||||
storedKey = resolved.apiKey;
|
||||
@ -172,14 +183,15 @@ export async function modelsScanCommand(
|
||||
storedKey = undefined;
|
||||
}
|
||||
}
|
||||
|
||||
const results = await withProgressTotals(
|
||||
{
|
||||
label: "Scanning OpenRouter models...",
|
||||
label: `Scanning ${sourceLabel} models...`,
|
||||
indeterminate: false,
|
||||
enabled: opts.json !== true,
|
||||
},
|
||||
async (update) =>
|
||||
await scanOpenRouterModels({
|
||||
async (progressUpdate) => {
|
||||
const scanOpts = {
|
||||
apiKey: storedKey ?? undefined,
|
||||
minParamB: minParams,
|
||||
maxAgeDays,
|
||||
@ -187,22 +199,35 @@ export async function modelsScanCommand(
|
||||
timeoutMs: timeout,
|
||||
concurrency,
|
||||
probe,
|
||||
onProgress: ({ phase, completed, total }) => {
|
||||
onProgress: ({
|
||||
phase,
|
||||
completed,
|
||||
total,
|
||||
}: {
|
||||
phase: string;
|
||||
completed: number;
|
||||
total: number;
|
||||
}) => {
|
||||
if (phase !== "probe") return;
|
||||
const labelBase = probe ? "Probing models" : "Scanning models";
|
||||
update({
|
||||
progressUpdate({
|
||||
completed,
|
||||
total,
|
||||
label: `${labelBase} (${completed}/${total})`,
|
||||
});
|
||||
},
|
||||
}),
|
||||
};
|
||||
if (source === "edenai") {
|
||||
return await scanEdenAiModels(scanOpts);
|
||||
}
|
||||
return await scanOpenRouterModels(scanOpts);
|
||||
},
|
||||
);
|
||||
|
||||
if (!probe) {
|
||||
if (!opts.json) {
|
||||
runtime.log(
|
||||
`Found ${results.length} OpenRouter free models (metadata only; pass --probe to test tools/images).`,
|
||||
`Found ${results.length} ${sourceLabel} models (metadata only; pass --probe to test tools/images).`,
|
||||
);
|
||||
printScanTable(sortScanResults(results), runtime);
|
||||
} else {
|
||||
@ -213,7 +238,7 @@ export async function modelsScanCommand(
|
||||
|
||||
const toolOk = results.filter((entry) => entry.tool.ok);
|
||||
if (toolOk.length === 0) {
|
||||
throw new Error("No tool-capable OpenRouter free models found.");
|
||||
throw new Error(`No tool-capable ${sourceLabel} models found.`);
|
||||
}
|
||||
|
||||
const sorted = sortScanResults(results);
|
||||
|
||||
Loading…
Reference in New Issue
Block a user