romtuck/openclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: validate model names in models set/fallbacks commands (#4179)

Browse Source 
Add configuration-time validation for model names using the model catalog.
Invalid models now show fuzzy suggestions. Add --force flag to skip validation.

- Add validateModelInCatalog() and validateImageModel() helpers
- Update models set, set-image, fallbacks add, image-fallbacks add
- Graceful degradation when catalog unavailable
- Add tests for validation functions

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Julian Gardner 2026-01-29 22:59:32 +02:00
parent 4583f88626
commit 8755080620
8 changed files with 449 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/cli/models-cli.ts
View File

@ -108,9 +108,10 @@ export function registerModelsCli(program: Command) {
.command("set")
.description("Set the default model")
.argument("<model>", "Model id or alias")
.action(async (model: string) => {
.option("--force", "Skip model validation", false)
.action(async (model: string, opts) => {
await runModelsCommand(async () => {
await modelsSetCommand(model, defaultRuntime);
await modelsSetCommand(model, defaultRuntime, { force: Boolean(opts.force) });
});
});
@ -118,9 +119,10 @@ export function registerModelsCli(program: Command) {
.command("set-image")
.description("Set the image model")
.argument("<model>", "Model id or alias")
.action(async (model: string) => {
.option("--force", "Skip model validation", false)
.action(async (model: string, opts) => {
await runModelsCommand(async () => {
await modelsSetImageCommand(model, defaultRuntime);
await modelsSetImageCommand(model, defaultRuntime, { force: Boolean(opts.force) });
});
});
@ -175,9 +177,10 @@ export function registerModelsCli(program: Command) {
.command("add")
.description("Add a fallback model")
.argument("<model>", "Model id or alias")
.action(async (model: string) => {
.option("--force", "Skip model validation", false)
.action(async (model: string, opts) => {
await runModelsCommand(async () => {
await modelsFallbacksAddCommand(model, defaultRuntime);
await modelsFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) });
});
});
@ -219,9 +222,10 @@ export function registerModelsCli(program: Command) {
.command("add")
.description("Add an image fallback model")
.argument("<model>", "Model id or alias")
.action(async (model: string) => {
.option("--force", "Skip model validation", false)
.action(async (model: string, opts) => {
await runModelsCommand(async () => {
await modelsImageFallbacksAddCommand(model, defaultRuntime);
await modelsImageFallbacksAddCommand(model, defaultRuntime, { force: Boolean(opts.force) });
});
});

7
src/commands/models.set.test.ts
View File

@ -11,6 +11,13 @@ vi.mock("../config/config.js", () => ({
loadConfig,
}));
// Mock model catalog to return empty (graceful degradation - all models are valid)
vi.mock("../agents/model-catalog.js", () => ({
loadModelCatalog: vi.fn().mockResolvedValue([]),
findModelInCatalog: vi.fn().mockReturnValue(undefined),
modelSupportsVision: vi.fn().mockReturnValue(false),
}));
describe("models set + fallbacks", () => {
beforeEach(() => {
readConfigFileSnapshot.mockReset();

47
src/commands/models/fallbacks.ts
View File

@ -8,6 +8,7 @@ import {
modelKey,
resolveModelTarget,
updateConfig,
validateModelInCatalog,
} from "./shared.js";
export async function modelsFallbacksListCommand(
@ -35,17 +36,39 @@ export async function modelsFallbacksListCommand(
for (const entry of fallbacks) runtime.log(`- ${entry}`);
}
export async function modelsFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
const nextModels = { ...cfg.agents?.defaults?.models };
export async function modelsFallbacksAddCommand(
modelRaw: string,
runtime: RuntimeEnv,
opts?: { force?: boolean },
) {
const cfg = loadConfig();
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
// Validate model exists in catalog unless --force is used
const validation = await validateModelInCatalog(resolved.provider, resolved.model);
if (!validation.valid) {
if (opts?.force) {
runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`);
} else {
const suggestionText =
validation.suggestions && validation.suggestions.length > 0
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
: "";
throw new Error(
`Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`,
);
}
}
const updated = await updateConfig((cfgSnapshot) => {
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
if (!nextModels[targetKey]) nextModels[targetKey] = {};
const aliasIndex = buildModelAliasIndex({
cfg,
cfg: cfgSnapshot,
defaultProvider: DEFAULT_PROVIDER,
});
const existing = cfg.agents?.defaults?.model?.fallbacks ?? [];
const existing = cfgSnapshot.agents?.defaults?.model?.fallbacks ?? [];
const existingKeys = existing
.map((entry) =>
resolveModelRefFromString({
@ -57,18 +80,18 @@ export async function modelsFallbacksAddCommand(modelRaw: string, runtime: Runti
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
if (existingKeys.includes(targetKey)) return cfg;
if (existingKeys.includes(targetKey)) return cfgSnapshot;
const existingModel = cfg.agents?.defaults?.model as
const existingModel = cfgSnapshot.agents?.defaults?.model as
| { primary?: string; fallbacks?: string[] }
| undefined;
return {
...cfg,
...cfgSnapshot,
agents: {
...cfg.agents,
...cfgSnapshot.agents,
defaults: {
...cfg.agents?.defaults,
...cfgSnapshot.agents?.defaults,
model: {
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
fallbacks: [...existing, targetKey],

57
src/commands/models/image-fallbacks.ts
View File

@ -8,6 +8,7 @@ import {
modelKey,
resolveModelTarget,
updateConfig,
validateImageModel,
} from "./shared.js";
export async function modelsImageFallbacksListCommand(
@ -35,17 +36,49 @@ export async function modelsImageFallbacksListCommand(
for (const entry of fallbacks) runtime.log(`- ${entry}`);
}
export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime: RuntimeEnv) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
const nextModels = { ...cfg.agents?.defaults?.models };
export async function modelsImageFallbacksAddCommand(
modelRaw: string,
runtime: RuntimeEnv,
opts?: { force?: boolean },
) {
const cfg = loadConfig();
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const targetKey = modelKey(resolved.provider, resolved.model);
// Validate model exists in catalog and supports vision unless --force is used
const validation = await validateImageModel(resolved.provider, resolved.model);
if (!validation.valid) {
if (opts?.force) {
runtime.log(`⚠️ Model not found in catalog: ${targetKey}. Proceeding anyway (--force).`);
} else {
const suggestionText =
validation.suggestions && validation.suggestions.length > 0
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
: "";
throw new Error(
`Unknown model: ${targetKey}${suggestionText}\nUse --force to skip validation.`,
);
}
} else if (validation.entry && validation.supportsVision === false) {
if (opts?.force) {
runtime.log(
`⚠️ Model ${targetKey} may not support image input. Proceeding anyway (--force).`,
);
} else {
runtime.log(
`⚠️ Model ${targetKey} does not appear to support image input. Use --force to set anyway.`,
);
}
}
const updated = await updateConfig((cfgSnapshot) => {
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
if (!nextModels[targetKey]) nextModels[targetKey] = {};
const aliasIndex = buildModelAliasIndex({
cfg,
cfg: cfgSnapshot,
defaultProvider: DEFAULT_PROVIDER,
});
const existing = cfg.agents?.defaults?.imageModel?.fallbacks ?? [];
const existing = cfgSnapshot.agents?.defaults?.imageModel?.fallbacks ?? [];
const existingKeys = existing
.map((entry) =>
resolveModelRefFromString({
@ -57,18 +90,18 @@ export async function modelsImageFallbacksAddCommand(modelRaw: string, runtime:
.filter((entry): entry is NonNullable<typeof entry> => Boolean(entry))
.map((entry) => modelKey(entry.ref.provider, entry.ref.model));
if (existingKeys.includes(targetKey)) return cfg;
if (existingKeys.includes(targetKey)) return cfgSnapshot;
const existingModel = cfg.agents?.defaults?.imageModel as
const existingModel = cfgSnapshot.agents?.defaults?.imageModel as
| { primary?: string; fallbacks?: string[] }
| undefined;
return {
...cfg,
...cfgSnapshot,
agents: {
...cfg.agents,
...cfgSnapshot.agents,
defaults: {
...cfg.agents?.defaults,
...cfgSnapshot.agents?.defaults,
imageModel: {
...(existingModel?.primary ? { primary: existingModel.primary } : undefined),
fallbacks: [...existing, targetKey],

48
src/commands/models/set-image.ts
View File

@ -1,22 +1,50 @@
import { logConfigUpdated } from "../../config/logging.js";
import type { RuntimeEnv } from "../../runtime.js";
import { resolveModelTarget, updateConfig } from "./shared.js";
import { resolveModelTarget, updateConfig, validateImageModel } from "./shared.js";
export async function modelsSetImageCommand(modelRaw: string, runtime: RuntimeEnv) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const key = `${resolved.provider}/${resolved.model}`;
const nextModels = { ...cfg.agents?.defaults?.models };
export async function modelsSetImageCommand(
modelRaw: string,
runtime: RuntimeEnv,
opts?: { force?: boolean },
) {
const cfg = (await import("../../config/config.js")).loadConfig();
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const key = `${resolved.provider}/${resolved.model}`;
// Validate model exists in catalog and supports vision unless --force is used
const validation = await validateImageModel(resolved.provider, resolved.model);
if (!validation.valid) {
if (opts?.force) {
runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`);
} else {
const suggestionText =
validation.suggestions && validation.suggestions.length > 0
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
: "";
throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`);
}
} else if (validation.entry && validation.supportsVision === false) {
if (opts?.force) {
runtime.log(`⚠️ Model ${key} may not support image input. Proceeding anyway (--force).`);
} else {
runtime.log(
`⚠️ Model ${key} does not appear to support image input. Use --force to set anyway.`,
);
}
}
const updated = await updateConfig((cfgSnapshot) => {
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
if (!nextModels[key]) nextModels[key] = {};
const existingModel = cfg.agents?.defaults?.imageModel as
const existingModel = cfgSnapshot.agents?.defaults?.imageModel as
| { primary?: string; fallbacks?: string[] }
| undefined;
return {
...cfg,
...cfgSnapshot,
agents: {
...cfg.agents,
...cfgSnapshot.agents,
defaults: {
...cfg.agents?.defaults,
...cfgSnapshot.agents?.defaults,
imageModel: {
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
primary: key,

40
src/commands/models/set.ts
View File

@ -1,22 +1,42 @@
import { logConfigUpdated } from "../../config/logging.js";
import type { RuntimeEnv } from "../../runtime.js";
import { resolveModelTarget, updateConfig } from "./shared.js";
import { resolveModelTarget, updateConfig, validateModelInCatalog } from "./shared.js";
export async function modelsSetCommand(modelRaw: string, runtime: RuntimeEnv) {
const updated = await updateConfig((cfg) => {
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const key = `${resolved.provider}/${resolved.model}`;
const nextModels = { ...cfg.agents?.defaults?.models };
export async function modelsSetCommand(
modelRaw: string,
runtime: RuntimeEnv,
opts?: { force?: boolean },
) {
const cfg = (await import("../../config/config.js")).loadConfig();
const resolved = resolveModelTarget({ raw: modelRaw, cfg });
const key = `${resolved.provider}/${resolved.model}`;
// Validate model exists in catalog unless --force is used
const validation = await validateModelInCatalog(resolved.provider, resolved.model);
if (!validation.valid) {
if (opts?.force) {
runtime.log(`⚠️ Model not found in catalog: ${key}. Proceeding anyway (--force).`);
} else {
const suggestionText =
validation.suggestions && validation.suggestions.length > 0
? `\nDid you mean: ${validation.suggestions.join(", ")}?`
: "";
throw new Error(`Unknown model: ${key}${suggestionText}\nUse --force to skip validation.`);
}
}
const updated = await updateConfig((cfgSnapshot) => {
const nextModels = { ...cfgSnapshot.agents?.defaults?.models };
if (!nextModels[key]) nextModels[key] = {};
const existingModel = cfg.agents?.defaults?.model as
const existingModel = cfgSnapshot.agents?.defaults?.model as
| { primary?: string; fallbacks?: string[] }
| undefined;
return {
...cfg,
...cfgSnapshot,
agents: {
...cfg.agents,
...cfgSnapshot.agents,
defaults: {
...cfg.agents?.defaults,
...cfgSnapshot.agents?.defaults,
model: {
...(existingModel?.fallbacks ? { fallbacks: existingModel.fallbacks } : undefined),
primary: key,

154
src/commands/models/shared.test.ts Normal file
View File

@ -0,0 +1,154 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mockLoadModelCatalog = vi.fn();
const mockFindModelInCatalog = vi.fn();
const mockModelSupportsVision = vi.fn();
vi.mock("../../agents/model-catalog.js", () => ({
loadModelCatalog: () => mockLoadModelCatalog(),
findModelInCatalog: (catalog: unknown[], provider: string, modelId: string) =>
mockFindModelInCatalog(catalog, provider, modelId),
modelSupportsVision: (entry: unknown) => mockModelSupportsVision(entry),
}));
describe("validateModelInCatalog", () => {
beforeEach(() => {
vi.resetModules();
mockLoadModelCatalog.mockReset();
mockFindModelInCatalog.mockReset();
mockModelSupportsVision.mockReset();
});
afterEach(() => {
vi.restoreAllMocks();
});
it("returns valid: true when model is found in catalog", async () => {
const mockEntry = { id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" };
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
mockFindModelInCatalog.mockReturnValue(mockEntry);
const { validateModelInCatalog } = await import("./shared.js");
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
expect(result.valid).toBe(true);
expect(result.entry).toEqual(mockEntry);
expect(result.suggestions).toBeUndefined();
});
it("returns valid: false with suggestions when model not found", async () => {
const catalog = [
{ id: "claude-opus-4-5", name: "Claude Opus 4.5", provider: "anthropic" },
{ id: "claude-sonnet-4-5", name: "Claude Sonnet 4.5", provider: "anthropic" },
];
mockLoadModelCatalog.mockResolvedValue(catalog);
mockFindModelInCatalog.mockReturnValue(undefined);
const { validateModelInCatalog } = await import("./shared.js");
const result = await validateModelInCatalog("anthropic", "claude-sonnet-4");
expect(result.valid).toBe(false);
expect(result.suggestions).toBeDefined();
expect(result.suggestions?.length).toBeGreaterThan(0);
// claude-sonnet-4-5 should be suggested as it's similar
expect(result.suggestions).toContain("anthropic/claude-sonnet-4-5");
});
it("returns valid: true when catalog is empty (graceful degradation)", async () => {
mockLoadModelCatalog.mockResolvedValue([]);
const { validateModelInCatalog } = await import("./shared.js");
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
expect(result.valid).toBe(true);
expect(result.entry).toBeUndefined();
});
it("returns valid: true when catalog fails to load (graceful degradation)", async () => {
mockLoadModelCatalog.mockRejectedValue(new Error("Failed to load catalog"));
const { validateModelInCatalog } = await import("./shared.js");
const result = await validateModelInCatalog("anthropic", "claude-opus-4-5");
expect(result.valid).toBe(true);
expect(result.entry).toBeUndefined();
});
it("fuzzy matches similar model names within same provider", async () => {
const catalog = [
{ id: "gpt-4o", name: "GPT-4o", provider: "openai" },
{ id: "gpt-4o-mini", name: "GPT-4o Mini", provider: "openai" },
{ id: "gpt-4-turbo", name: "GPT-4 Turbo", provider: "openai" },
];
mockLoadModelCatalog.mockResolvedValue(catalog);
mockFindModelInCatalog.mockReturnValue(undefined);
const { validateModelInCatalog } = await import("./shared.js");
const result = await validateModelInCatalog("openai", "gpt-4");
expect(result.valid).toBe(false);
expect(result.suggestions).toBeDefined();
// gpt-4o and gpt-4-turbo should be suggested
expect(result.suggestions?.some((s) => s.includes("gpt-4"))).toBe(true);
});
});
describe("validateImageModel", () => {
beforeEach(() => {
vi.resetModules();
mockLoadModelCatalog.mockReset();
mockFindModelInCatalog.mockReset();
mockModelSupportsVision.mockReset();
});
afterEach(() => {
vi.restoreAllMocks();
});
it("returns supportsVision: true when model supports image input", async () => {
const mockEntry = {
id: "gpt-4o",
name: "GPT-4o",
provider: "openai",
input: ["text", "image"],
};
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
mockFindModelInCatalog.mockReturnValue(mockEntry);
mockModelSupportsVision.mockReturnValue(true);
const { validateImageModel } = await import("./shared.js");
const result = await validateImageModel("openai", "gpt-4o");
expect(result.valid).toBe(true);
expect(result.supportsVision).toBe(true);
});
it("returns supportsVision: false when model does not support image input", async () => {
const mockEntry = {
id: "gpt-3.5-turbo",
name: "GPT-3.5 Turbo",
provider: "openai",
input: ["text"],
};
mockLoadModelCatalog.mockResolvedValue([mockEntry]);
mockFindModelInCatalog.mockReturnValue(mockEntry);
mockModelSupportsVision.mockReturnValue(false);
const { validateImageModel } = await import("./shared.js");
const result = await validateImageModel("openai", "gpt-3.5-turbo");
expect(result.valid).toBe(true);
expect(result.supportsVision).toBe(false);
});
it("returns valid: false when model not found", async () => {
mockLoadModelCatalog.mockResolvedValue([{ id: "gpt-4o", name: "GPT-4o", provider: "openai" }]);
mockFindModelInCatalog.mockReturnValue(undefined);
const { validateImageModel } = await import("./shared.js");
const result = await validateImageModel("openai", "nonexistent-model");
expect(result.valid).toBe(false);
expect(result.supportsVision).toBeUndefined();
});
});

128
src/commands/models/shared.ts
View File

@ -94,3 +94,131 @@ export { DEFAULT_MODEL, DEFAULT_PROVIDER };
* For providers with hierarchical model IDs (e.g., OpenRouter), the model ID may include
* sub-providers (e.g., "moonshotai/kimi-k2"), resulting in a key like "openrouter/moonshotai/kimi-k2".
*/
import {
findModelInCatalog,
loadModelCatalog,
type ModelCatalogEntry,
modelSupportsVision,
} from "../../agents/model-catalog.js";
export type ModelValidationResult = {
valid: boolean;
entry?: ModelCatalogEntry;
suggestions?: string[];
};
/**
* Validate that a model exists in the catalog.
* Returns suggestions for similar models if not found.
* Gracefully degrades if catalog fails to load.
*/
export async function validateModelInCatalog(
provider: string,
modelId: string,
): Promise<ModelValidationResult> {
try {
const catalog = await loadModelCatalog();
if (catalog.length === 0) {
// Catalog unavailable - gracefully degrade
return { valid: true };
}
const entry = findModelInCatalog(catalog, provider, modelId);
if (entry) {
return { valid: true, entry };
}
// Find suggestions via fuzzy matching
const suggestions = findSimilarModels(catalog, provider, modelId);
return { valid: false, suggestions };
} catch {
// Catalog load failed - gracefully degrade
return { valid: true };
}
}
/**
* Validate that a model exists and supports vision input.
*/
export async function validateImageModel(
provider: string,
modelId: string,
): Promise<ModelValidationResult & { supportsVision?: boolean }> {
const result = await validateModelInCatalog(provider, modelId);
if (!result.valid || !result.entry) {
return result;
}
return {
...result,
supportsVision: modelSupportsVision(result.entry),
};
}
function findSimilarModels(
catalog: ModelCatalogEntry[],
provider: string,
modelId: string,
): string[] {
const normalizedProvider = provider.toLowerCase();
const normalizedModelId = modelId.toLowerCase();
// Score models by similarity
const scored = catalog
.map((entry) => {
const entryProvider = entry.provider.toLowerCase();
const entryModelId = entry.id.toLowerCase();
let score = 0;
// Boost same-provider matches
if (entryProvider === normalizedProvider) {
score += 50;
}
// Calculate string similarity for model ID
score += stringSimilarity(normalizedModelId, entryModelId);
return { key: `${entry.provider}/${entry.id}`, score };
})
.filter((item) => item.score > 30) // Minimum threshold
.sort((a, b) => b.score - a.score)
.slice(0, 3);
return scored.map((item) => item.key);
}
function stringSimilarity(a: string, b: string): number {
// Simple Levenshtein-based similarity score
const maxLen = Math.max(a.length, b.length);
if (maxLen === 0) return 100;
const distance = levenshteinDistance(a, b);
return Math.round((1 - distance / maxLen) * 100);
}
function levenshteinDistance(a: string, b: string): number {
const matrix: number[][] = [];
for (let i = 0; i <= b.length; i++) {
matrix[i] = [i];
}
for (let j = 0; j <= a.length; j++) {
matrix[0][j] = j;
}
for (let i = 1; i <= b.length; i++) {
for (let j = 1; j <= a.length; j++) {
if (b[i - 1] === a[j - 1]) {
matrix[i][j] = matrix[i - 1][j - 1];
} else {
matrix[i][j] = Math.min(
matrix[i - 1][j - 1] + 1,
matrix[i][j - 1] + 1,
matrix[i - 1][j] + 1,
);
}
}
}
return matrix[b.length][a.length];
}