Merge 8755080620 into da71eaebd2
This commit is contained in:
commit
8e85672fc8
@ -108,9 +108,10 @@ export function registerModelsCli(program: Command) {
|
|||||||
.command("set")
|
.command("set")
|
||||||
.description("Set the default model")
|
.description("Set the default model")
|
||||||
.argument("<model>", "Model id or alias")
|
.argument("<model>", "Model id or alias")
|
||||||
.action(async (model: string) => {
|
.option("--force", "Skip model validation", false)
|
||||||
|
.action(async (model: string, opts) => {
|
||||||
await runModelsCommand(async () => {
|
await runModelsCommand(async () => {
|
||||||
await modelsSetCommand(model, defaultRuntime);
|
await modelsSetCommand(model, defaultRuntime, { force: Boolean(opts.force) });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -118,9 +119,10 @@ export function registerModelsCli(program: Command) {
|
|||||||
.command("set-image")
|
.command("set-image")
|
||||||
.description("Set the image model")
|
.description("Set the image model")
|
||||||
.argument("<model>", "Model id or alias")
|
.argument("<model>", "Model id or alias")
|
||||||
.action(async (model: string) => {
|
.option("--force", "Skip model validation", false)
|
||||||
|
.action(async (model: string, opts) => {
|
||||||
await runModelsCommand(async () => {
|
await runModelsCommand(async () => {
|
||||||
await modelsSetImageCommand(model, defaultRuntime);
|
await modelsSetImageCommand(model, defaultRuntime, { force: Boolean(opts.force) });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -175,9 +177,10 @@ export function registerModelsCli(program: Command) {
|
|||||||
.command("add")
|
.command("add")
|
||||||
.description("Add a fallback model")
|
.description("Add a fallback model")
|
||||||
.argument("<model>", "Model id or alias")
|
.argument("<model>", "Model id or alias")
|
||||||
.action(async (model: string) => {
|
.option("--force", "Skip model validation", false)
|
||||||
|
.action(async (model: string, opts) => {
|
||||||
await runModelsCommand(async () => {
|
await runModelsCommand(async () => {
|
||||||
await modelsFallbacksAddCommand(model, defaultRuntime);
|
await modelsFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
@ -219,9 +222,10 @@ export function registerModelsCli(program: Command) {
|
|||||||
.command("add")
|
.command("add")
|
||||||
.description("Add an image fallback model")
|
.description("Add an image fallback model")
|
||||||
.argument("<model>", "Model id or alias")
|
.argument("<model>", "Model id or alias")
|
||||||
.action(async (model: string) => {
|
.option("--force", "Skip model validation", false)
|
||||||
|
.action(async (model: string, opts) => {
|
||||||
await runModelsCommand(async () => {
|
await runModelsCommand(async () => {
|
||||||
await modelsImageFallbacksAddCommand(model, defaultRuntime);
|
await modelsImageFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) });
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,13 @@ vi.mock("../config/config.js", () => ({
|
|||||||
loadConfig,
|
loadConfig,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
// Mock model catalog to return empty (graceful degradation - all models are valid)
|
||||||
|
vi.mock("../agents/model-catalog.js", () => ({
|
||||||
|
loadModelCatalog: vi.fn().mockResolvedValue([]),
|
||||||
|
findModelInCatalog: vi.fn().mockReturnValue(undefined),
|
||||||
|
modelSupportsVision: vi.fn().mockReturnValue(false),
|
||||||
|
}));
|
||||||
|
|
||||||
describe("models set + fallbacks", () => {
|
describe("models set + fallbacks", () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
readConfigFileSnapshot.mockReset();
|
readConfigFileSnapshot.mockReset();
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import {
|
|||||||
modelKey,
|
modelKey,
|
||||||
resolveModelTarget,
|
resolveModelTarget,
|
||||||
updateConfig,
|
updateConfig,
|
||||||
|
validateModelInCatalog,
|
||||||
} from "./shared.js";
|
} from "./shared.js";
|
||||||
|
|
||||||
export async function modelsFallbacksListCommand(
|
export async function modelsFallbacksListCommand(
|
||||||
@ -35,17 +36,39 @@ export async function modelsFallbacksListCommand(
|
|||||||
for (const entry of fallbacks) runtime.log(`- ${entry}`);
|
for (const entry of fallbacks) runtime.log(`- ${entry}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function modelsFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) {
|
export async function modelsFallbacksAddCommand(
|
||||||
const updated = await updateConfig((cfg) => {
|
modelRaw: string,
|
||||||
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
runtime: RuntimeEnv,
|
||||||
const targetKey = modelKey(resolved.provider, resolved.model);
|
opts?: { force?: boolean },
|
||||||
const nextModels = { ...cfg.agents?.defaults?.models };
|
) {
|
||||||
|
const cfg = loadConfig();
|
||||||
|
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
||||||
|
const targetKey = modelKey(resolved.provider, resolved.model);
|
||||||
|
|
||||||
|
// Validate model exists in catalog unless --force is used
|
||||||
|
const validation = await validateModelInCatalog(resolved.provider, resolved.model);
|
||||||
|
if (!validation.valid) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`);
|
||||||
|
} else {
|
||||||
|
const suggestionText =
|
||||||
|
validation.suggestions && validation.suggestions.length > 0
|
||||||
|
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
|
||||||
|
: "";
|
||||||
|
throw new Error(
|
||||||
|
`Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await updateConfig((cfgSnapshot) => {
|
||||||
|
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
|
||||||
if (!nextModels[targetKey]) nextModels[targetKey] = {};
|
if (!nextModels[targetKey]) nextModels[targetKey] = {};
|
||||||
const aliasIndex = buildModelAliasIndex({
|
const aliasIndex = buildModelAliasIndex({
|
||||||
cfg,
|
cfg: cfgSnapshot,
|
||||||
defaultProvider: DEFAULT_PROVIDER,
|
defaultProvider: DEFAULT_PROVIDER,
|
||||||
});
|
});
|
||||||
const existing = cfg.agents?.defaults?.model?.fallbacks ?? [];
|
const existing = cfgSnapshot.agents?.defaults?.model?.fallbacks ?? [];
|
||||||
const existingKeys = existing
|
const existingKeys = existing
|
||||||
.map((entry) =>
|
.map((entry) =>
|
||||||
resolveModelRefFromString({
|
resolveModelRefFromString({
|
||||||
@ -57,18 +80,18 @@ export async function modelsFallbacksAddCommand(modelRaw: string, runtime: Runti
|
|||||||
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
|
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
|
||||||
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
|
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
|
||||||
|
|
||||||
if (existingKeys.includes(targetKey)) return cfg;
|
if (existingKeys.includes(targetKey)) return cfgSnapshot;
|
||||||
|
|
||||||
const existingModel = cfg.agents?.defaults?.model as
|
const existingModel = cfgSnapshot.agents?.defaults?.model as
|
||||||
| { primary?: string; fallbacks?: string[] }
|
| { primary?: string; fallbacks?: string[] }
|
||||||
| undefined;
|
| undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...cfg,
|
...cfgSnapshot,
|
||||||
agents: {
|
agents: {
|
||||||
...cfg.agents,
|
...cfgSnapshot.agents,
|
||||||
defaults: {
|
defaults: {
|
||||||
...cfg.agents?.defaults,
|
...cfgSnapshot.agents?.defaults,
|
||||||
model: {
|
model: {
|
||||||
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
|
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
|
||||||
fallbacks: [...existing, targetKey],
|
fallbacks: [...existing, targetKey],
|
||||||
|
|||||||
@ -8,6 +8,7 @@ import {
|
|||||||
modelKey,
|
modelKey,
|
||||||
resolveModelTarget,
|
resolveModelTarget,
|
||||||
updateConfig,
|
updateConfig,
|
||||||
|
validateImageModel,
|
||||||
} from "./shared.js";
|
} from "./shared.js";
|
||||||
|
|
||||||
export async function modelsImageFallbacksListCommand(
|
export async function modelsImageFallbacksListCommand(
|
||||||
@ -35,17 +36,49 @@ export async function modelsImageFallbacksListCommand(
|
|||||||
for (const entry of fallbacks) runtime.log(`- ${entry}`);
|
for (const entry of fallbacks) runtime.log(`- ${entry}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) {
|
export async function modelsImageFallbacksAddCommand(
|
||||||
const updated = await updateConfig((cfg) => {
|
modelRaw: string,
|
||||||
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
runtime: RuntimeEnv,
|
||||||
const targetKey = modelKey(resolved.provider, resolved.model);
|
opts?: { force?: boolean },
|
||||||
const nextModels = { ...cfg.agents?.defaults?.models };
|
) {
|
||||||
|
const cfg = loadConfig();
|
||||||
|
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
||||||
|
const targetKey = modelKey(resolved.provider, resolved.model);
|
||||||
|
|
||||||
|
// Validate model exists in catalog and supports vision unless --force is used
|
||||||
|
const validation = await validateImageModel(resolved.provider, resolved.model);
|
||||||
|
if (!validation.valid) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`);
|
||||||
|
} else {
|
||||||
|
const suggestionText =
|
||||||
|
validation.suggestions && validation.suggestions.length > 0
|
||||||
|
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
|
||||||
|
: "";
|
||||||
|
throw new Error(
|
||||||
|
`Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
} else if (validation.entry && validation.supportsVision === false) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(
|
||||||
|
`⚠️ Model ${targetKey} may not support image input. Proceeding anyway (--force).`,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
runtime.log(
|
||||||
|
`⚠️ Model ${targetKey} does not appear to support image input. Use --force to set anyway.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await updateConfig((cfgSnapshot) => {
|
||||||
|
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
|
||||||
if (!nextModels[targetKey]) nextModels[targetKey] = {};
|
if (!nextModels[targetKey]) nextModels[targetKey] = {};
|
||||||
const aliasIndex = buildModelAliasIndex({
|
const aliasIndex = buildModelAliasIndex({
|
||||||
cfg,
|
cfg: cfgSnapshot,
|
||||||
defaultProvider: DEFAULT_PROVIDER,
|
defaultProvider: DEFAULT_PROVIDER,
|
||||||
});
|
});
|
||||||
const existing = cfg.agents?.defaults?.imageModel?.fallbacks ?? [];
|
const existing = cfgSnapshot.agents?.defaults?.imageModel?.fallbacks ?? [];
|
||||||
const existingKeys = existing
|
const existingKeys = existing
|
||||||
.map((entry) =>
|
.map((entry) =>
|
||||||
resolveModelRefFromString({
|
resolveModelRefFromString({
|
||||||
@ -57,18 +90,18 @@ export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime:
|
|||||||
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
|
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
|
||||||
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
|
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
|
||||||
|
|
||||||
if (existingKeys.includes(targetKey)) return cfg;
|
if (existingKeys.includes(targetKey)) return cfgSnapshot;
|
||||||
|
|
||||||
const existingModel = cfg.agents?.defaults?.imageModel as
|
const existingModel = cfgSnapshot.agents?.defaults?.imageModel as
|
||||||
| { primary?: string; fallbacks?: string[] }
|
| { primary?: string; fallbacks?: string[] }
|
||||||
| undefined;
|
| undefined;
|
||||||
|
|
||||||
return {
|
return {
|
||||||
...cfg,
|
...cfgSnapshot,
|
||||||
agents: {
|
agents: {
|
||||||
...cfg.agents,
|
...cfgSnapshot.agents,
|
||||||
defaults: {
|
defaults: {
|
||||||
...cfg.agents?.defaults,
|
...cfgSnapshot.agents?.defaults,
|
||||||
imageModel: {
|
imageModel: {
|
||||||
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
|
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
|
||||||
fallbacks: [...existing, targetKey],
|
fallbacks: [...existing, targetKey],
|
||||||
|
|||||||
@ -1,22 +1,50 @@
|
|||||||
import { logConfigUpdated } from "../../config/logging.js";
|
import { logConfigUpdated } from "../../config/logging.js";
|
||||||
import type { RuntimeEnv } from "../../runtime.js";
|
import type { RuntimeEnv } from "../../runtime.js";
|
||||||
import { resolveModelTarget, updateConfig } from "./shared.js";
|
import { resolveModelTarget, updateConfig, validateImageModel } from "./shared.js";
|
||||||
|
|
||||||
export async function modelsSetImageCommand(modelRaw: string, runtime: RuntimeEnv) {
|
export async function modelsSetImageCommand(
|
||||||
const updated = await updateConfig((cfg) => {
|
modelRaw: string,
|
||||||
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
runtime: RuntimeEnv,
|
||||||
const key = `${resolved.provider}/${resolved.model}`;
|
opts?: { force?: boolean },
|
||||||
const nextModels = { ...cfg.agents?.defaults?.models };
|
) {
|
||||||
|
const cfg = (await import("../../config/config.js")).loadConfig();
|
||||||
|
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
||||||
|
const key = `${resolved.provider}/${resolved.model}`;
|
||||||
|
|
||||||
|
// Validate model exists in catalog and supports vision unless --force is used
|
||||||
|
const validation = await validateImageModel(resolved.provider, resolved.model);
|
||||||
|
if (!validation.valid) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`);
|
||||||
|
} else {
|
||||||
|
const suggestionText =
|
||||||
|
validation.suggestions && validation.suggestions.length > 0
|
||||||
|
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
|
||||||
|
: "";
|
||||||
|
throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`);
|
||||||
|
}
|
||||||
|
} else if (validation.entry && validation.supportsVision === false) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(`⚠️ Model ${key} may not support image input. Proceeding anyway (--force).`);
|
||||||
|
} else {
|
||||||
|
runtime.log(
|
||||||
|
`⚠️ Model ${key} does not appear to support image input. Use --force to set anyway.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await updateConfig((cfgSnapshot) => {
|
||||||
|
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
|
||||||
if (!nextModels[key]) nextModels[key] = {};
|
if (!nextModels[key]) nextModels[key] = {};
|
||||||
const existingModel = cfg.agents?.defaults?.imageModel as
|
const existingModel = cfgSnapshot.agents?.defaults?.imageModel as
|
||||||
| { primary?: string; fallbacks?: string[] }
|
| { primary?: string; fallbacks?: string[] }
|
||||||
| undefined;
|
| undefined;
|
||||||
return {
|
return {
|
||||||
...cfg,
|
...cfgSnapshot,
|
||||||
agents: {
|
agents: {
|
||||||
...cfg.agents,
|
...cfgSnapshot.agents,
|
||||||
defaults: {
|
defaults: {
|
||||||
...cfg.agents?.defaults,
|
...cfgSnapshot.agents?.defaults,
|
||||||
imageModel: {
|
imageModel: {
|
||||||
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
|
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
|
||||||
primary: key,
|
primary: key,
|
||||||
|
|||||||
@ -1,22 +1,42 @@
|
|||||||
import { logConfigUpdated } from "../../config/logging.js";
|
import { logConfigUpdated } from "../../config/logging.js";
|
||||||
import type { RuntimeEnv } from "../../runtime.js";
|
import type { RuntimeEnv } from "../../runtime.js";
|
||||||
import { resolveModelTarget, updateConfig } from "./shared.js";
|
import { resolveModelTarget, updateConfig, validateModelInCatalog } from "./shared.js";
|
||||||
|
|
||||||
export async function modelsSetCommand(modelRaw: string, runtime: RuntimeEnv) {
|
export async function modelsSetCommand(
|
||||||
const updated = await updateConfig((cfg) => {
|
modelRaw: string,
|
||||||
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
runtime: RuntimeEnv,
|
||||||
const key = `${resolved.provider}/${resolved.model}`;
|
opts?: { force?: boolean },
|
||||||
const nextModels = { ...cfg.agents?.defaults?.models };
|
) {
|
||||||
|
const cfg = (await import("../../config/config.js")).loadConfig();
|
||||||
|
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
|
||||||
|
const key = `${resolved.provider}/${resolved.model}`;
|
||||||
|
|
||||||
|
// Validate model exists in catalog unless --force is used
|
||||||
|
const validation = await validateModelInCatalog(resolved.provider, resolved.model);
|
||||||
|
if (!validation.valid) {
|
||||||
|
if (opts?.force) {
|
||||||
|
runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`);
|
||||||
|
} else {
|
||||||
|
const suggestionText =
|
||||||
|
validation.suggestions && validation.suggestions.length > 0
|
||||||
|
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
|
||||||
|
: "";
|
||||||
|
throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const updated = await updateConfig((cfgSnapshot) => {
|
||||||
|
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
|
||||||
if (!nextModels[key]) nextModels[key] = {};
|
if (!nextModels[key]) nextModels[key] = {};
|
||||||
const existingModel = cfg.agents?.defaults?.model as
|
const existingModel = cfgSnapshot.agents?.defaults?.model as
|
||||||
| { primary?: string; fallbacks?: string[] }
|
| { primary?: string; fallbacks?: string[] }
|
||||||
| undefined;
|
| undefined;
|
||||||
return {
|
return {
|
||||||
...cfg,
|
...cfgSnapshot,
|
||||||
agents: {
|
agents: {
|
||||||
...cfg.agents,
|
...cfgSnapshot.agents,
|
||||||
defaults: {
|
defaults: {
|
||||||
...cfg.agents?.defaults,
|
...cfgSnapshot.agents?.defaults,
|
||||||
model: {
|
model: {
|
||||||
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
|
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
|
||||||
primary: key,
|
primary: key,
|
||||||
|
|||||||
154
src/commands/models/shared.test.ts
Normal file
154
src/commands/models/shared.test.ts
Normal file
@ -0,0 +1,154 @@
|
|||||||
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
|
const mockLoadModelCatalog = vi.fn();
|
||||||
|
const mockFindModelInCatalog = vi.fn();
|
||||||
|
const mockModelSupportsVision = vi.fn();
|
||||||
|
|
||||||
|
vi.mock("../../agents/model-catalog.js", () => ({
|
||||||
|
loadModelCatalog: () => mockLoadModelCatalog(),
|
||||||
|
findModelInCatalog: (catalog: unknown[], provider: string, modelId: string) =>
|
||||||
|
mockFindModelInCatalog(catalog, provider, modelId),
|
||||||
|
modelSupportsVision: (entry: unknown) => mockModelSupportsVision(entry),
|
||||||
|
}));
|
||||||
|
|
||||||
|
describe("validateModelInCatalog", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
mockLoadModelCatalog.mockReset();
|
||||||
|
mockFindModelInCatalog.mockReset();
|
||||||
|
mockModelSupportsVision.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns valid: true when model is found in catalog", async () => {
|
||||||
|
const mockEntry = { id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" };
|
||||||
|
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(mockEntry);
|
||||||
|
|
||||||
|
const { validateModelInCatalog } = await import("./shared.js");
|
||||||
|
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(true);
|
||||||
|
expect(result.entry).toEqual(mockEntry);
|
||||||
|
expect(result.suggestions).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns valid: false with suggestions when model not found", async () => {
|
||||||
|
const catalog = [
|
||||||
|
{ id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" },
|
||||||
|
{ id: "claude-sonnet-4-5", name: "Claude Sonnet 4.5", provider: "anthropic" },
|
||||||
|
];
|
||||||
|
mockLoadModelCatalog.mockResolvedValue(catalog);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(undefined);
|
||||||
|
|
||||||
|
const { validateModelInCatalog } = await import("./shared.js");
|
||||||
|
const result = await validateModelInCatalog("anthropic", "claude-sonnet-4");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(false);
|
||||||
|
expect(result.suggestions).toBeDefined();
|
||||||
|
expect(result.suggestions?.length).toBeGreaterThan(0);
|
||||||
|
// claude-sonnet-4-5 should be suggested as it's similar
|
||||||
|
expect(result.suggestions).toContain("anthropic/claude-sonnet-4-5");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns valid: true when catalog is empty (graceful degradation)", async () => {
|
||||||
|
mockLoadModelCatalog.mockResolvedValue([]);
|
||||||
|
|
||||||
|
const { validateModelInCatalog } = await import("./shared.js");
|
||||||
|
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(true);
|
||||||
|
expect(result.entry).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns valid: true when catalog fails to load (graceful degradation)", async () => {
|
||||||
|
mockLoadModelCatalog.mockRejectedValue(new Error("Failed to load catalog"));
|
||||||
|
|
||||||
|
const { validateModelInCatalog } = await import("./shared.js");
|
||||||
|
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(true);
|
||||||
|
expect(result.entry).toBeUndefined();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fuzzy matches similar model names within same provider", async () => {
|
||||||
|
const catalog = [
|
||||||
|
{ id: "gpt-4o", name: "GPT-4o", provider: "openai" },
|
||||||
|
{ id: "gpt-4o-mini", name: "GPT-4o Mini", provider: "openai" },
|
||||||
|
{ id: "gpt-4-turbo", name: "GPT-4 Turbo", provider: "openai" },
|
||||||
|
];
|
||||||
|
mockLoadModelCatalog.mockResolvedValue(catalog);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(undefined);
|
||||||
|
|
||||||
|
const { validateModelInCatalog } = await import("./shared.js");
|
||||||
|
const result = await validateModelInCatalog("openai", "gpt-4");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(false);
|
||||||
|
expect(result.suggestions).toBeDefined();
|
||||||
|
// gpt-4o and gpt-4-turbo should be suggested
|
||||||
|
expect(result.suggestions?.some((s) => s.includes("gpt-4"))).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("validateImageModel", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
vi.resetModules();
|
||||||
|
mockLoadModelCatalog.mockReset();
|
||||||
|
mockFindModelInCatalog.mockReset();
|
||||||
|
mockModelSupportsVision.mockReset();
|
||||||
|
});
|
||||||
|
|
||||||
|
afterEach(() => {
|
||||||
|
vi.restoreAllMocks();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns supportsVision: true when model supports image input", async () => {
|
||||||
|
const mockEntry = {
|
||||||
|
id: "gpt-4o",
|
||||||
|
name: "GPT-4o",
|
||||||
|
provider: "openai",
|
||||||
|
input: ["text", "image"],
|
||||||
|
};
|
||||||
|
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(mockEntry);
|
||||||
|
mockModelSupportsVision.mockReturnValue(true);
|
||||||
|
|
||||||
|
const { validateImageModel } = await import("./shared.js");
|
||||||
|
const result = await validateImageModel("openai", "gpt-4o");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(true);
|
||||||
|
expect(result.supportsVision).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns supportsVision: false when model does not support image input", async () => {
|
||||||
|
const mockEntry = {
|
||||||
|
id: "gpt-3.5-turbo",
|
||||||
|
name: "GPT-3.5 Turbo",
|
||||||
|
provider: "openai",
|
||||||
|
input: ["text"],
|
||||||
|
};
|
||||||
|
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(mockEntry);
|
||||||
|
mockModelSupportsVision.mockReturnValue(false);
|
||||||
|
|
||||||
|
const { validateImageModel } = await import("./shared.js");
|
||||||
|
const result = await validateImageModel("openai", "gpt-3.5-turbo");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(true);
|
||||||
|
expect(result.supportsVision).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns valid: false when model not found", async () => {
|
||||||
|
mockLoadModelCatalog.mockResolvedValue([{ id: "gpt-4o", name: "GPT-4o", provider: "openai" }]);
|
||||||
|
mockFindModelInCatalog.mockReturnValue(undefined);
|
||||||
|
|
||||||
|
const { validateImageModel } = await import("./shared.js");
|
||||||
|
const result = await validateImageModel("openai", "nonexistent-model");
|
||||||
|
|
||||||
|
expect(result.valid).toBe(false);
|
||||||
|
expect(result.supportsVision).toBeUndefined();
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -94,3 +94,131 @@ export { DEFAULT_MODEL, DEFAULT_PROVIDER };
|
|||||||
* For providers with hierarchical model IDs (e.g., OpenRouter), the model ID may include
|
* For providers with hierarchical model IDs (e.g., OpenRouter), the model ID may include
|
||||||
* sub-providers (e.g., "moonshotai/kimi-k2"), resulting in a key like "openrouter/moonshotai/kimi-k2".
|
* sub-providers (e.g., "moonshotai/kimi-k2"), resulting in a key like "openrouter/moonshotai/kimi-k2".
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import {
|
||||||
|
findModelInCatalog,
|
||||||
|
loadModelCatalog,
|
||||||
|
type ModelCatalogEntry,
|
||||||
|
modelSupportsVision,
|
||||||
|
} from "../../agents/model-catalog.js";
|
||||||
|
|
||||||
|
export type ModelValidationResult = {
|
||||||
|
valid: boolean;
|
||||||
|
entry?: ModelCatalogEntry;
|
||||||
|
suggestions?: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate that a model exists in the catalog.
|
||||||
|
* Returns suggestions for similar models if not found.
|
||||||
|
* Gracefully degrades if catalog fails to load.
|
||||||
|
*/
|
||||||
|
export async function validateModelInCatalog(
|
||||||
|
provider: string,
|
||||||
|
modelId: string,
|
||||||
|
): Promise<ModelValidationResult> {
|
||||||
|
try {
|
||||||
|
const catalog = await loadModelCatalog();
|
||||||
|
if (catalog.length === 0) {
|
||||||
|
// Catalog unavailable - gracefully degrade
|
||||||
|
return { valid: true };
|
||||||
|
}
|
||||||
|
|
||||||
|
const entry = findModelInCatalog(catalog, provider, modelId);
|
||||||
|
if (entry) {
|
||||||
|
return { valid: true, entry };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Find suggestions via fuzzy matching
|
||||||
|
const suggestions = findSimilarModels(catalog, provider, modelId);
|
||||||
|
return { valid: false, suggestions };
|
||||||
|
} catch {
|
||||||
|
// Catalog load failed - gracefully degrade
|
||||||
|
return { valid: true };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Validate that a model exists and supports vision input.
|
||||||
|
*/
|
||||||
|
export async function validateImageModel(
|
||||||
|
provider: string,
|
||||||
|
modelId: string,
|
||||||
|
): Promise<ModelValidationResult & { supportsVision?: boolean }> {
|
||||||
|
const result = await validateModelInCatalog(provider, modelId);
|
||||||
|
if (!result.valid || !result.entry) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...result,
|
||||||
|
supportsVision: modelSupportsVision(result.entry),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function findSimilarModels(
|
||||||
|
catalog: ModelCatalogEntry[],
|
||||||
|
provider: string,
|
||||||
|
modelId: string,
|
||||||
|
): string[] {
|
||||||
|
const normalizedProvider = provider.toLowerCase();
|
||||||
|
const normalizedModelId = modelId.toLowerCase();
|
||||||
|
|
||||||
|
// Score models by similarity
|
||||||
|
const scored = catalog
|
||||||
|
.map((entry) => {
|
||||||
|
const entryProvider = entry.provider.toLowerCase();
|
||||||
|
const entryModelId = entry.id.toLowerCase();
|
||||||
|
|
||||||
|
let score = 0;
|
||||||
|
|
||||||
|
// Boost same-provider matches
|
||||||
|
if (entryProvider === normalizedProvider) {
|
||||||
|
score += 50;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate string similarity for model ID
|
||||||
|
score += stringSimilarity(normalizedModelId, entryModelId);
|
||||||
|
|
||||||
|
return { key: `${entry.provider}/${entry.id}`, score };
|
||||||
|
})
|
||||||
|
.filter((item) => item.score > 30) // Minimum threshold
|
||||||
|
.sort((a, b) => b.score - a.score)
|
||||||
|
.slice(0, 3);
|
||||||
|
|
||||||
|
return scored.map((item) => item.key);
|
||||||
|
}
|
||||||
|
|
||||||
|
function stringSimilarity(a: string, b: string): number {
|
||||||
|
// Simple Levenshtein-based similarity score
|
||||||
|
const maxLen = Math.max(a.length, b.length);
|
||||||
|
if (maxLen === 0) return 100;
|
||||||
|
const distance = levenshteinDistance(a, b);
|
||||||
|
return Math.round((1 - distance / maxLen) * 100);
|
||||||
|
}
|
||||||
|
|
||||||
|
function levenshteinDistance(a: string, b: string): number {
|
||||||
|
const matrix: number[][] = [];
|
||||||
|
|
||||||
|
for (let i = 0; i <= b.length; i++) {
|
||||||
|
matrix[i] = [i];
|
||||||
|
}
|
||||||
|
for (let j = 0; j <= a.length; j++) {
|
||||||
|
matrix[0][j] = j;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 1; i <= b.length; i++) {
|
||||||
|
for (let j = 1; j <= a.length; j++) {
|
||||||
|
if (b[i - 1] === a[j - 1]) {
|
||||||
|
matrix[i][j] = matrix[i - 1][j - 1];
|
||||||
|
} else {
|
||||||
|
matrix[i][j] = Math.min(
|
||||||
|
matrix[i - 1][j - 1] + 1,
|
||||||
|
matrix[i][j - 1] + 1,
|
||||||
|
matrix[i - 1][j] + 1,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return matrix[b.length][a.length];
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user