diff --git a/src/cli/models-cli.ts b/src/cli/models-cli.ts index f68631d18..e29eb85fc 100644 --- a/src/cli/models-cli.ts +++ b/src/cli/models-cli.ts @@ -108,9 +108,10 @@ export function registerModelsCli(program: Command) { .command("set") .description("Set the default model") .argument("", "Model id or alias") - .action(async (model: string) => { + .option("--force", "Skip model validation", false) + .action(async (model: string, opts) => { await runModelsCommand(async () => { - await modelsSetCommand(model, defaultRuntime); + await modelsSetCommand(model, defaultRuntime, { force: Boolean(opts.force) }); }); }); @@ -118,9 +119,10 @@ export function registerModelsCli(program: Command) { .command("set-image") .description("Set the image model") .argument("", "Model id or alias") - .action(async (model: string) => { + .option("--force", "Skip model validation", false) + .action(async (model: string, opts) => { await runModelsCommand(async () => { - await modelsSetImageCommand(model, defaultRuntime); + await modelsSetImageCommand(model, defaultRuntime, { force: Boolean(opts.force) }); }); }); @@ -175,9 +177,10 @@ export function registerModelsCli(program: Command) { .command("add") .description("Add a fallback model") .argument("", "Model id or alias") - .action(async (model: string) => { + .option("--force", "Skip model validation", false) + .action(async (model: string, opts) => { await runModelsCommand(async () => { - await modelsFallbacksAddCommand(model, defaultRuntime); + await modelsFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) }); }); }); @@ -219,9 +222,10 @@ export function registerModelsCli(program: Command) { .command("add") .description("Add an image fallback model") .argument("", "Model id or alias") - .action(async (model: string) => { + .option("--force", "Skip model validation", false) + .action(async (model: string, opts) => { await runModelsCommand(async () => { - await modelsImageFallbacksAddCommand(model, defaultRuntime); + await modelsImageFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) }); }); }); diff --git a/src/commands/models.set.test.ts b/src/commands/models.set.test.ts index 9957bc5e5..bf1d396d8 100644 --- a/src/commands/models.set.test.ts +++ b/src/commands/models.set.test.ts @@ -11,6 +11,13 @@ vi.mock("../config/config.js", () => ({ loadConfig, })); +// Mock model catalog to return empty (graceful degradation - all models are valid) +vi.mock("../agents/model-catalog.js", () => ({ + loadModelCatalog: vi.fn().mockResolvedValue([]), + findModelInCatalog: vi.fn().mockReturnValue(undefined), + modelSupportsVision: vi.fn().mockReturnValue(false), +})); + describe("models set + fallbacks", () => { beforeEach(() => { readConfigFileSnapshot.mockReset(); diff --git a/src/commands/models/fallbacks.ts b/src/commands/models/fallbacks.ts index b34defa09..70059a083 100644 --- a/src/commands/models/fallbacks.ts +++ b/src/commands/models/fallbacks.ts @@ -8,6 +8,7 @@ import { modelKey, resolveModelTarget, updateConfig, + validateModelInCatalog, } from "./shared.js"; export async function modelsFallbacksListCommand( @@ -35,17 +36,39 @@ export async function modelsFallbacksListCommand( for (const entry of fallbacks) runtime.log(`- ${entry}`); } -export async function modelsFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) { - const updated = await updateConfig((cfg) => { - const resolved = resolveModelTarget({ raw: modelRaw, cfg }); - const targetKey = modelKey(resolved.provider, resolved.model); - const nextModels = { ...cfg.agents?.defaults?.models }; +export async function modelsFallbacksAddCommand( + modelRaw: string, + runtime: RuntimeEnv, + opts?: { force?: boolean }, +) { + const cfg = loadConfig(); + const resolved = resolveModelTarget({ raw: modelRaw, cfg }); + const targetKey = modelKey(resolved.provider, resolved.model); + + // Validate model exists in catalog unless --force is used + const validation = await validateModelInCatalog(resolved.provider, resolved.model); + if (!validation.valid) { + if (opts?.force) { + runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`); + } else { + const suggestionText = + validation.suggestions && validation.suggestions.length > 0 + ? `\nDid you mean: ${validation.suggestions.join(", ")}?` + : ""; + throw new Error( + `Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`, + ); + } + } + + const updated = await updateConfig((cfgSnapshot) => { + const nextModels = { ...cfgSnapshot.agents?.defaults?.models }; if (!nextModels[targetKey]) nextModels[targetKey] = {}; const aliasIndex = buildModelAliasIndex({ - cfg, + cfg: cfgSnapshot, defaultProvider: DEFAULT_PROVIDER, }); - const existing = cfg.agents?.defaults?.model?.fallbacks ?? []; + const existing = cfgSnapshot.agents?.defaults?.model?.fallbacks ?? []; const existingKeys = existing .map((entry) => resolveModelRefFromString({ @@ -57,18 +80,18 @@ export async function modelsFallbacksAddCommand(modelRaw: string, runtime: Runti .filter((entry): entry is NonNullable => Boolean(entry)) .map((entry) => modelKey(entry.ref.provider, entry.ref.model)); - if (existingKeys.includes(targetKey)) return cfg; + if (existingKeys.includes(targetKey)) return cfgSnapshot; - const existingModel = cfg.agents?.defaults?.model as + const existingModel = cfgSnapshot.agents?.defaults?.model as | { primary?: string; fallbacks?: string[] } | undefined; return { - ...cfg, + ...cfgSnapshot, agents: { - ...cfg.agents, + ...cfgSnapshot.agents, defaults: { - ...cfg.agents?.defaults, + ...cfgSnapshot.agents?.defaults, model: { ...(existingModel?.primary ? { primary: existingModel.primary } : undefined), fallbacks: [...existing, targetKey], diff --git a/src/commands/models/image-fallbacks.ts b/src/commands/models/image-fallbacks.ts index bffcfb1d8..7c4e4121a 100644 --- a/src/commands/models/image-fallbacks.ts +++ b/src/commands/models/image-fallbacks.ts @@ -8,6 +8,7 @@ import { modelKey, resolveModelTarget, updateConfig, + validateImageModel, } from "./shared.js"; export async function modelsImageFallbacksListCommand( @@ -35,17 +36,49 @@ export async function modelsImageFallbacksListCommand( for (const entry of fallbacks) runtime.log(`- ${entry}`); } -export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) { - const updated = await updateConfig((cfg) => { - const resolved = resolveModelTarget({ raw: modelRaw, cfg }); - const targetKey = modelKey(resolved.provider, resolved.model); - const nextModels = { ...cfg.agents?.defaults?.models }; +export async function modelsImageFallbacksAddCommand( + modelRaw: string, + runtime: RuntimeEnv, + opts?: { force?: boolean }, +) { + const cfg = loadConfig(); + const resolved = resolveModelTarget({ raw: modelRaw, cfg }); + const targetKey = modelKey(resolved.provider, resolved.model); + + // Validate model exists in catalog and supports vision unless --force is used + const validation = await validateImageModel(resolved.provider, resolved.model); + if (!validation.valid) { + if (opts?.force) { + runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`); + } else { + const suggestionText = + validation.suggestions && validation.suggestions.length > 0 + ? `\nDid you mean: ${validation.suggestions.join(", ")}?` + : ""; + throw new Error( + `Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`, + ); + } + } else if (validation.entry && validation.supportsVision === false) { + if (opts?.force) { + runtime.log( + `⚠️ Model ${targetKey} may not support image input. Proceeding anyway (--force).`, + ); + } else { + runtime.log( + `⚠️ Model ${targetKey} does not appear to support image input. Use --force to set anyway.`, + ); + } + } + + const updated = await updateConfig((cfgSnapshot) => { + const nextModels = { ...cfgSnapshot.agents?.defaults?.models }; if (!nextModels[targetKey]) nextModels[targetKey] = {}; const aliasIndex = buildModelAliasIndex({ - cfg, + cfg: cfgSnapshot, defaultProvider: DEFAULT_PROVIDER, }); - const existing = cfg.agents?.defaults?.imageModel?.fallbacks ?? []; + const existing = cfgSnapshot.agents?.defaults?.imageModel?.fallbacks ?? []; const existingKeys = existing .map((entry) => resolveModelRefFromString({ @@ -57,18 +90,18 @@ export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime: .filter((entry): entry is NonNullable => Boolean(entry)) .map((entry) => modelKey(entry.ref.provider, entry.ref.model)); - if (existingKeys.includes(targetKey)) return cfg; + if (existingKeys.includes(targetKey)) return cfgSnapshot; - const existingModel = cfg.agents?.defaults?.imageModel as + const existingModel = cfgSnapshot.agents?.defaults?.imageModel as | { primary?: string; fallbacks?: string[] } | undefined; return { - ...cfg, + ...cfgSnapshot, agents: { - ...cfg.agents, + ...cfgSnapshot.agents, defaults: { - ...cfg.agents?.defaults, + ...cfgSnapshot.agents?.defaults, imageModel: { ...(existingModel?.primary ? { primary: existingModel.primary } : undefined), fallbacks: [...existing, targetKey], diff --git a/src/commands/models/set-image.ts b/src/commands/models/set-image.ts index b0e6e3ecd..f2a0d0c88 100644 --- a/src/commands/models/set-image.ts +++ b/src/commands/models/set-image.ts @@ -1,22 +1,50 @@ import { logConfigUpdated } from "../../config/logging.js"; import type { RuntimeEnv } from "../../runtime.js"; -import { resolveModelTarget, updateConfig } from "./shared.js"; +import { resolveModelTarget, updateConfig, validateImageModel } from "./shared.js"; -export async function modelsSetImageCommand(modelRaw: string, runtime: RuntimeEnv) { - const updated = await updateConfig((cfg) => { - const resolved = resolveModelTarget({ raw: modelRaw, cfg }); - const key = `${resolved.provider}/${resolved.model}`; - const nextModels = { ...cfg.agents?.defaults?.models }; +export async function modelsSetImageCommand( + modelRaw: string, + runtime: RuntimeEnv, + opts?: { force?: boolean }, +) { + const cfg = (await import("../../config/config.js")).loadConfig(); + const resolved = resolveModelTarget({ raw: modelRaw, cfg }); + const key = `${resolved.provider}/${resolved.model}`; + + // Validate model exists in catalog and supports vision unless --force is used + const validation = await validateImageModel(resolved.provider, resolved.model); + if (!validation.valid) { + if (opts?.force) { + runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`); + } else { + const suggestionText = + validation.suggestions && validation.suggestions.length > 0 + ? `\nDid you mean: ${validation.suggestions.join(", ")}?` + : ""; + throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`); + } + } else if (validation.entry && validation.supportsVision === false) { + if (opts?.force) { + runtime.log(`⚠️ Model ${key} may not support image input. Proceeding anyway (--force).`); + } else { + runtime.log( + `⚠️ Model ${key} does not appear to support image input. Use --force to set anyway.`, + ); + } + } + + const updated = await updateConfig((cfgSnapshot) => { + const nextModels = { ...cfgSnapshot.agents?.defaults?.models }; if (!nextModels[key]) nextModels[key] = {}; - const existingModel = cfg.agents?.defaults?.imageModel as + const existingModel = cfgSnapshot.agents?.defaults?.imageModel as | { primary?: string; fallbacks?: string[] } | undefined; return { - ...cfg, + ...cfgSnapshot, agents: { - ...cfg.agents, + ...cfgSnapshot.agents, defaults: { - ...cfg.agents?.defaults, + ...cfgSnapshot.agents?.defaults, imageModel: { ...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined), primary: key, diff --git a/src/commands/models/set.ts b/src/commands/models/set.ts index b6d74711e..7c0bfb41b 100644 --- a/src/commands/models/set.ts +++ b/src/commands/models/set.ts @@ -1,22 +1,42 @@ import { logConfigUpdated } from "../../config/logging.js"; import type { RuntimeEnv } from "../../runtime.js"; -import { resolveModelTarget, updateConfig } from "./shared.js"; +import { resolveModelTarget, updateConfig, validateModelInCatalog } from "./shared.js"; -export async function modelsSetCommand(modelRaw: string, runtime: RuntimeEnv) { - const updated = await updateConfig((cfg) => { - const resolved = resolveModelTarget({ raw: modelRaw, cfg }); - const key = `${resolved.provider}/${resolved.model}`; - const nextModels = { ...cfg.agents?.defaults?.models }; +export async function modelsSetCommand( + modelRaw: string, + runtime: RuntimeEnv, + opts?: { force?: boolean }, +) { + const cfg = (await import("../../config/config.js")).loadConfig(); + const resolved = resolveModelTarget({ raw: modelRaw, cfg }); + const key = `${resolved.provider}/${resolved.model}`; + + // Validate model exists in catalog unless --force is used + const validation = await validateModelInCatalog(resolved.provider, resolved.model); + if (!validation.valid) { + if (opts?.force) { + runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`); + } else { + const suggestionText = + validation.suggestions && validation.suggestions.length > 0 + ? `\nDid you mean: ${validation.suggestions.join(", ")}?` + : ""; + throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`); + } + } + + const updated = await updateConfig((cfgSnapshot) => { + const nextModels = { ...cfgSnapshot.agents?.defaults?.models }; if (!nextModels[key]) nextModels[key] = {}; - const existingModel = cfg.agents?.defaults?.model as + const existingModel = cfgSnapshot.agents?.defaults?.model as | { primary?: string; fallbacks?: string[] } | undefined; return { - ...cfg, + ...cfgSnapshot, agents: { - ...cfg.agents, + ...cfgSnapshot.agents, defaults: { - ...cfg.agents?.defaults, + ...cfgSnapshot.agents?.defaults, model: { ...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined), primary: key, diff --git a/src/commands/models/shared.test.ts b/src/commands/models/shared.test.ts new file mode 100644 index 000000000..3bf8f5e84 --- /dev/null +++ b/src/commands/models/shared.test.ts @@ -0,0 +1,154 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mockLoadModelCatalog = vi.fn(); +const mockFindModelInCatalog = vi.fn(); +const mockModelSupportsVision = vi.fn(); + +vi.mock("../../agents/model-catalog.js", () => ({ + loadModelCatalog: () => mockLoadModelCatalog(), + findModelInCatalog: (catalog: unknown[], provider: string, modelId: string) => + mockFindModelInCatalog(catalog, provider, modelId), + modelSupportsVision: (entry: unknown) => mockModelSupportsVision(entry), +})); + +describe("validateModelInCatalog", () => { + beforeEach(() => { + vi.resetModules(); + mockLoadModelCatalog.mockReset(); + mockFindModelInCatalog.mockReset(); + mockModelSupportsVision.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns valid: true when model is found in catalog", async () => { + const mockEntry = { id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" }; + mockLoadModelCatalog.mockResolvedValue([mockEntry]); + mockFindModelInCatalog.mockReturnValue(mockEntry); + + const { validateModelInCatalog } = await import("./shared.js"); + const result = await validateModelInCatalog("anthropic", "claude-opus-4-5"); + + expect(result.valid).toBe(true); + expect(result.entry).toEqual(mockEntry); + expect(result.suggestions).toBeUndefined(); + }); + + it("returns valid: false with suggestions when model not found", async () => { + const catalog = [ + { id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" }, + { id: "claude-sonnet-4-5", name: "Claude Sonnet 4.5", provider: "anthropic" }, + ]; + mockLoadModelCatalog.mockResolvedValue(catalog); + mockFindModelInCatalog.mockReturnValue(undefined); + + const { validateModelInCatalog } = await import("./shared.js"); + const result = await validateModelInCatalog("anthropic", "claude-sonnet-4"); + + expect(result.valid).toBe(false); + expect(result.suggestions).toBeDefined(); + expect(result.suggestions?.length).toBeGreaterThan(0); + // claude-sonnet-4-5 should be suggested as it's similar + expect(result.suggestions).toContain("anthropic/claude-sonnet-4-5"); + }); + + it("returns valid: true when catalog is empty (graceful degradation)", async () => { + mockLoadModelCatalog.mockResolvedValue([]); + + const { validateModelInCatalog } = await import("./shared.js"); + const result = await validateModelInCatalog("anthropic", "claude-opus-4-5"); + + expect(result.valid).toBe(true); + expect(result.entry).toBeUndefined(); + }); + + it("returns valid: true when catalog fails to load (graceful degradation)", async () => { + mockLoadModelCatalog.mockRejectedValue(new Error("Failed to load catalog")); + + const { validateModelInCatalog } = await import("./shared.js"); + const result = await validateModelInCatalog("anthropic", "claude-opus-4-5"); + + expect(result.valid).toBe(true); + expect(result.entry).toBeUndefined(); + }); + + it("fuzzy matches similar model names within same provider", async () => { + const catalog = [ + { id: "gpt-4o", name: "GPT-4o", provider: "openai" }, + { id: "gpt-4o-mini", name: "GPT-4o Mini", provider: "openai" }, + { id: "gpt-4-turbo", name: "GPT-4 Turbo", provider: "openai" }, + ]; + mockLoadModelCatalog.mockResolvedValue(catalog); + mockFindModelInCatalog.mockReturnValue(undefined); + + const { validateModelInCatalog } = await import("./shared.js"); + const result = await validateModelInCatalog("openai", "gpt-4"); + + expect(result.valid).toBe(false); + expect(result.suggestions).toBeDefined(); + // gpt-4o and gpt-4-turbo should be suggested + expect(result.suggestions?.some((s) => s.includes("gpt-4"))).toBe(true); + }); +}); + +describe("validateImageModel", () => { + beforeEach(() => { + vi.resetModules(); + mockLoadModelCatalog.mockReset(); + mockFindModelInCatalog.mockReset(); + mockModelSupportsVision.mockReset(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("returns supportsVision: true when model supports image input", async () => { + const mockEntry = { + id: "gpt-4o", + name: "GPT-4o", + provider: "openai", + input: ["text", "image"], + }; + mockLoadModelCatalog.mockResolvedValue([mockEntry]); + mockFindModelInCatalog.mockReturnValue(mockEntry); + mockModelSupportsVision.mockReturnValue(true); + + const { validateImageModel } = await import("./shared.js"); + const result = await validateImageModel("openai", "gpt-4o"); + + expect(result.valid).toBe(true); + expect(result.supportsVision).toBe(true); + }); + + it("returns supportsVision: false when model does not support image input", async () => { + const mockEntry = { + id: "gpt-3.5-turbo", + name: "GPT-3.5 Turbo", + provider: "openai", + input: ["text"], + }; + mockLoadModelCatalog.mockResolvedValue([mockEntry]); + mockFindModelInCatalog.mockReturnValue(mockEntry); + mockModelSupportsVision.mockReturnValue(false); + + const { validateImageModel } = await import("./shared.js"); + const result = await validateImageModel("openai", "gpt-3.5-turbo"); + + expect(result.valid).toBe(true); + expect(result.supportsVision).toBe(false); + }); + + it("returns valid: false when model not found", async () => { + mockLoadModelCatalog.mockResolvedValue([{ id: "gpt-4o", name: "GPT-4o", provider: "openai" }]); + mockFindModelInCatalog.mockReturnValue(undefined); + + const { validateImageModel } = await import("./shared.js"); + const result = await validateImageModel("openai", "nonexistent-model"); + + expect(result.valid).toBe(false); + expect(result.supportsVision).toBeUndefined(); + }); +}); diff --git a/src/commands/models/shared.ts b/src/commands/models/shared.ts index 620b45df3..70a1abd18 100644 --- a/src/commands/models/shared.ts +++ b/src/commands/models/shared.ts @@ -94,3 +94,131 @@ export { DEFAULT_MODEL, DEFAULT_PROVIDER }; * For providers with hierarchical model IDs (e.g., OpenRouter), the model ID may include * sub-providers (e.g., "moonshotai/kimi-k2"), resulting in a key like "openrouter/moonshotai/kimi-k2". */ + +import { + findModelInCatalog, + loadModelCatalog, + type ModelCatalogEntry, + modelSupportsVision, +} from "../../agents/model-catalog.js"; + +export type ModelValidationResult = { + valid: boolean; + entry?: ModelCatalogEntry; + suggestions?: string[]; +}; + +/** + * Validate that a model exists in the catalog. + * Returns suggestions for similar models if not found. + * Gracefully degrades if catalog fails to load. + */ +export async function validateModelInCatalog( + provider: string, + modelId: string, +): Promise { + try { + const catalog = await loadModelCatalog(); + if (catalog.length === 0) { + // Catalog unavailable - gracefully degrade + return { valid: true }; + } + + const entry = findModelInCatalog(catalog, provider, modelId); + if (entry) { + return { valid: true, entry }; + } + + // Find suggestions via fuzzy matching + const suggestions = findSimilarModels(catalog, provider, modelId); + return { valid: false, suggestions }; + } catch { + // Catalog load failed - gracefully degrade + return { valid: true }; + } +} + +/** + * Validate that a model exists and supports vision input. + */ +export async function validateImageModel( + provider: string, + modelId: string, +): Promise { + const result = await validateModelInCatalog(provider, modelId); + if (!result.valid || !result.entry) { + return result; + } + return { + ...result, + supportsVision: modelSupportsVision(result.entry), + }; +} + +function findSimilarModels( + catalog: ModelCatalogEntry[], + provider: string, + modelId: string, +): string[] { + const normalizedProvider = provider.toLowerCase(); + const normalizedModelId = modelId.toLowerCase(); + + // Score models by similarity + const scored = catalog + .map((entry) => { + const entryProvider = entry.provider.toLowerCase(); + const entryModelId = entry.id.toLowerCase(); + + let score = 0; + + // Boost same-provider matches + if (entryProvider === normalizedProvider) { + score += 50; + } + + // Calculate string similarity for model ID + score += stringSimilarity(normalizedModelId, entryModelId); + + return { key: `${entry.provider}/${entry.id}`, score }; + }) + .filter((item) => item.score > 30) // Minimum threshold + .sort((a, b) => b.score - a.score) + .slice(0, 3); + + return scored.map((item) => item.key); +} + +function stringSimilarity(a: string, b: string): number { + // Simple Levenshtein-based similarity score + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 100; + const distance = levenshteinDistance(a, b); + return Math.round((1 - distance / maxLen) * 100); +} + +function levenshteinDistance(a: string, b: string): number { + const matrix: number[][] = []; + + for (let i = 0; i <= b.length; i++) { + matrix[i] = [i]; + } + for (let j = 0; j <= a.length; j++) { + matrix[0][j] = j; + } + + for (let i = 1; i <= b.length; i++) { + for (let j = 1; j <= a.length; j++) { + if (b[i - 1] === a[j - 1]) { + matrix[i][j] = matrix[i - 1][j - 1]; + } else { + matrix[i][j] = Math.min( + matrix[i - 1][j - 1] + 1, + matrix[i][j - 1] + 1, + matrix[i - 1][j] + 1, + ); + } + } + } + + return matrix[b.length][a.length]; +}