Add LLM router skeleton (intent + routing.yaml)
This commit is contained in:
FortressFormations
parent
b717724275
commit
8a7c2c8d9a
247
src/routing/llm-router/index.ts
Normal file
247
src/routing/llm-router/index.ts
Normal file
@ -0,0 +1,247 @@
|
|||||||
|
import fs from "node:fs/promises";
|
||||||
|
import path from "node:path";
|
||||||
|
import YAML from "yaml";
|
||||||
|
import type { ModelRef } from "../../agents/model-selection.js";
|
||||||
|
import { parseModelRef } from "../../agents/model-selection.js";
|
||||||
|
|
||||||
|
export type Intent = "chat" | "strategy" | "code" | "summarize" | "tool" | "continuity";
|
||||||
|
|
||||||
|
export type RouteDecision = {
|
||||||
|
intent: Intent;
|
||||||
|
provider: string;
|
||||||
|
model: string;
|
||||||
|
reason: string;
|
||||||
|
isDefault: boolean;
|
||||||
|
fallbacks?: ModelRef[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type IntentRoutingSpec = {
|
||||||
|
primary?: string;
|
||||||
|
fallbacks?: string[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type RouterRoutingConfig = {
|
||||||
|
intents?: Partial<Record<Intent, IntentRoutingSpec>>;
|
||||||
|
};
|
||||||
|
|
||||||
|
type RouterPolicyConfig = {
|
||||||
|
complexity?: {
|
||||||
|
contextTokensGe?: number;
|
||||||
|
target?: string;
|
||||||
|
};
|
||||||
|
guardrails?: {
|
||||||
|
highStakes?: boolean;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export type RouterConfig = {
|
||||||
|
routing?: RouterRoutingConfig;
|
||||||
|
policy?: RouterPolicyConfig;
|
||||||
|
limitsRaw?: string;
|
||||||
|
pricingRaw?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
const INTENTS: Intent[] = ["chat", "strategy", "code", "summarize", "tool", "continuity"];
|
||||||
|
|
||||||
|
const DEFAULT_INTENT_ROUTES: Partial<
|
||||||
|
Record<Intent, { primary: ModelRef; fallbacks?: ModelRef[] }>
|
||||||
|
> = {
|
||||||
|
chat: {
|
||||||
|
primary: { provider: "anthropic", model: "haiku" },
|
||||||
|
fallbacks: [{ provider: "anthropic", model: "sonnet" }],
|
||||||
|
},
|
||||||
|
strategy: {
|
||||||
|
primary: { provider: "anthropic", model: "sonnet" },
|
||||||
|
fallbacks: [{ provider: "anthropic", model: "haiku" }],
|
||||||
|
},
|
||||||
|
code: {
|
||||||
|
primary: { provider: "openai-codex", model: "codex" },
|
||||||
|
},
|
||||||
|
summarize: {
|
||||||
|
primary: { provider: "anthropic", model: "haiku" },
|
||||||
|
},
|
||||||
|
continuity: {
|
||||||
|
primary: { provider: "local", model: "local_small" },
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||||
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
function toOptionalString(value: unknown): string | undefined {
|
||||||
|
if (typeof value !== "string") return undefined;
|
||||||
|
const trimmed = value.trim();
|
||||||
|
return trimmed ? trimmed : undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function toStringArray(value: unknown): string[] | undefined {
|
||||||
|
if (!Array.isArray(value)) return undefined;
|
||||||
|
return value
|
||||||
|
.map((entry) => (typeof entry === "string" ? entry.trim() : ""))
|
||||||
|
.filter((entry) => entry.length > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readOptionalFile(filePath: string): Promise<string | null> {
|
||||||
|
try {
|
||||||
|
return await fs.readFile(filePath, "utf-8");
|
||||||
|
} catch (error) {
|
||||||
|
const err = error as NodeJS.ErrnoException;
|
||||||
|
if (err.code === "ENOENT") return null;
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseRoutingConfig(raw: string): RouterRoutingConfig {
|
||||||
|
const parsed = YAML.parse(raw) as unknown;
|
||||||
|
if (!isRecord(parsed)) return {};
|
||||||
|
const intentsRaw = isRecord(parsed.intents) ? parsed.intents : undefined;
|
||||||
|
if (!intentsRaw) return {};
|
||||||
|
|
||||||
|
const intents: Partial<Record<Intent, IntentRoutingSpec>> = {};
|
||||||
|
for (const intent of INTENTS) {
|
||||||
|
const entry = intentsRaw[intent];
|
||||||
|
if (!isRecord(entry)) continue;
|
||||||
|
const primary = toOptionalString(entry.primary);
|
||||||
|
const fallbacks = toStringArray(entry.fallbacks);
|
||||||
|
if (!primary && !fallbacks) continue;
|
||||||
|
intents[intent] = {
|
||||||
|
...(primary ? { primary } : {}),
|
||||||
|
...(fallbacks ? { fallbacks } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return { intents };
|
||||||
|
}
|
||||||
|
|
||||||
|
function parsePolicyConfig(raw: string): RouterPolicyConfig {
|
||||||
|
const parsed = YAML.parse(raw) as unknown;
|
||||||
|
if (!isRecord(parsed)) return {};
|
||||||
|
const complexityRaw = isRecord(parsed.complexity) ? parsed.complexity : undefined;
|
||||||
|
const guardrailsRaw = isRecord(parsed.guardrails) ? parsed.guardrails : undefined;
|
||||||
|
|
||||||
|
const contextTokensGe = (() => {
|
||||||
|
const value =
|
||||||
|
complexityRaw?.context_tokens_ge ??
|
||||||
|
complexityRaw?.contextTokensGe ??
|
||||||
|
complexityRaw?.threshold;
|
||||||
|
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const target = toOptionalString(complexityRaw?.target);
|
||||||
|
const highStakes = (() => {
|
||||||
|
const value = guardrailsRaw?.high_stakes ?? guardrailsRaw?.highStakes;
|
||||||
|
return typeof value === "boolean" ? value : undefined;
|
||||||
|
})();
|
||||||
|
|
||||||
|
const hasComplexity = contextTokensGe !== undefined || target !== undefined;
|
||||||
|
return {
|
||||||
|
...(hasComplexity
|
||||||
|
? {
|
||||||
|
complexity: {
|
||||||
|
...(contextTokensGe !== undefined ? { contextTokensGe } : {}),
|
||||||
|
...(target ? { target } : {}),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: {}),
|
||||||
|
...(highStakes !== undefined ? { guardrails: { highStakes } } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseModelRefOrNull(raw: string | undefined, defaultProvider: string): ModelRef | null {
|
||||||
|
if (!raw) return null;
|
||||||
|
return parseModelRef(raw, defaultProvider);
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveFallbackRefs(
|
||||||
|
fallbacks: string[] | undefined,
|
||||||
|
defaultProvider: string,
|
||||||
|
): ModelRef[] | undefined {
|
||||||
|
if (!fallbacks) return undefined;
|
||||||
|
const parsed = fallbacks
|
||||||
|
.map((entry) => parseModelRef(entry, defaultProvider))
|
||||||
|
.filter((entry): entry is ModelRef => !!entry);
|
||||||
|
return parsed.length > 0 ? parsed : [];
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function loadRouterConfig(dir: string): Promise<RouterConfig | null> {
|
||||||
|
const [routingRaw, limitsRaw, pricingRaw, policyRaw] = await Promise.all([
|
||||||
|
readOptionalFile(path.join(dir, "routing.yaml")),
|
||||||
|
readOptionalFile(path.join(dir, "limits.yaml")),
|
||||||
|
readOptionalFile(path.join(dir, "pricing.yaml")),
|
||||||
|
readOptionalFile(path.join(dir, "policy.yaml")),
|
||||||
|
]);
|
||||||
|
|
||||||
|
if (!routingRaw && !limitsRaw && !pricingRaw && !policyRaw) return null;
|
||||||
|
|
||||||
|
return {
|
||||||
|
...(routingRaw ? { routing: parseRoutingConfig(routingRaw) } : {}),
|
||||||
|
...(policyRaw ? { policy: parsePolicyConfig(policyRaw) } : {}),
|
||||||
|
...(limitsRaw ? { limitsRaw } : {}),
|
||||||
|
...(pricingRaw ? { pricingRaw } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function resolveRouteDecision(params: {
|
||||||
|
cfg: RouterConfig | null;
|
||||||
|
agentDir?: string;
|
||||||
|
intent: Intent;
|
||||||
|
defaultModelRef: ModelRef;
|
||||||
|
contextTokens?: number;
|
||||||
|
highStakes?: boolean;
|
||||||
|
}): RouteDecision {
|
||||||
|
const { cfg, intent, defaultModelRef, contextTokens } = params;
|
||||||
|
if (!cfg) {
|
||||||
|
return {
|
||||||
|
intent,
|
||||||
|
provider: defaultModelRef.provider,
|
||||||
|
model: defaultModelRef.model,
|
||||||
|
reason: "default",
|
||||||
|
isDefault: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const baseRoute = DEFAULT_INTENT_ROUTES[intent];
|
||||||
|
let primary = baseRoute?.primary ?? defaultModelRef;
|
||||||
|
let fallbacks = baseRoute?.fallbacks;
|
||||||
|
const defaultProvider = defaultModelRef.provider;
|
||||||
|
|
||||||
|
const intentRouting = cfg.routing?.intents?.[intent];
|
||||||
|
const overridePrimary = parseModelRefOrNull(intentRouting?.primary, defaultProvider);
|
||||||
|
if (overridePrimary) {
|
||||||
|
primary = overridePrimary;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (intentRouting && "fallbacks" in intentRouting) {
|
||||||
|
fallbacks = resolveFallbackRefs(intentRouting.fallbacks, defaultProvider) ?? fallbacks;
|
||||||
|
}
|
||||||
|
|
||||||
|
const shouldEscalate =
|
||||||
|
(intent === "chat" || intent === "strategy") &&
|
||||||
|
cfg.policy?.complexity?.contextTokensGe !== undefined &&
|
||||||
|
contextTokens !== undefined &&
|
||||||
|
contextTokens >= cfg.policy.complexity.contextTokensGe;
|
||||||
|
|
||||||
|
if (shouldEscalate) {
|
||||||
|
const target = parseModelRefOrNull(cfg.policy?.complexity?.target, defaultProvider) ?? {
|
||||||
|
provider: "anthropic",
|
||||||
|
model: "opus",
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
intent,
|
||||||
|
provider: target.provider,
|
||||||
|
model: target.model,
|
||||||
|
reason: "complexity",
|
||||||
|
isDefault: false,
|
||||||
|
...(fallbacks && fallbacks.length > 0 ? { fallbacks } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
intent,
|
||||||
|
provider: primary.provider,
|
||||||
|
model: primary.model,
|
||||||
|
reason: "intent",
|
||||||
|
isDefault: false,
|
||||||
|
...(fallbacks && fallbacks.length > 0 ? { fallbacks } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
141
src/routing/llm-router/llm-router.test.ts
Normal file
141
src/routing/llm-router/llm-router.test.ts
Normal file
@ -0,0 +1,141 @@
|
|||||||
|
import fs from "node:fs/promises";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { ModelRef } from "../../agents/model-selection.js";
|
||||||
|
import { loadRouterConfig, resolveRouteDecision } from "./index.js";
|
||||||
|
|
||||||
|
async function withTempDir<T>(runner: (dir: string) => Promise<T>): Promise<T> {
|
||||||
|
const dir = await fs.mkdtemp(path.join(os.tmpdir(), "moltbot-router-"));
|
||||||
|
try {
|
||||||
|
return await runner(dir);
|
||||||
|
} finally {
|
||||||
|
await fs.rm(dir, { recursive: true, force: true });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const defaultModelRef: ModelRef = { provider: "anthropic", model: "default" };
|
||||||
|
|
||||||
|
describe("llm-router", () => {
|
||||||
|
it("returns default when config dir is missing or empty", async () => {
|
||||||
|
const missingDir = path.join(os.tmpdir(), `moltbot-router-missing-${Date.now()}`);
|
||||||
|
const missingCfg = await loadRouterConfig(missingDir);
|
||||||
|
expect(missingCfg).toBeNull();
|
||||||
|
|
||||||
|
const decision = resolveRouteDecision({
|
||||||
|
cfg: missingCfg,
|
||||||
|
intent: "chat",
|
||||||
|
defaultModelRef,
|
||||||
|
});
|
||||||
|
expect(decision).toEqual({
|
||||||
|
intent: "chat",
|
||||||
|
provider: "anthropic",
|
||||||
|
model: "default",
|
||||||
|
reason: "default",
|
||||||
|
isDefault: true,
|
||||||
|
});
|
||||||
|
|
||||||
|
await withTempDir(async (dir) => {
|
||||||
|
const emptyCfg = await loadRouterConfig(dir);
|
||||||
|
expect(emptyCfg).toBeNull();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("resolves routing.yaml intents", async () => {
|
||||||
|
await withTempDir(async (dir) => {
|
||||||
|
await fs.writeFile(
|
||||||
|
path.join(dir, "routing.yaml"),
|
||||||
|
[
|
||||||
|
"intents:",
|
||||||
|
" chat:",
|
||||||
|
" primary: anthropic/haiku",
|
||||||
|
" fallbacks:",
|
||||||
|
" - anthropic/sonnet",
|
||||||
|
" strategy:",
|
||||||
|
" primary: anthropic/sonnet",
|
||||||
|
" fallbacks:",
|
||||||
|
" - anthropic/haiku",
|
||||||
|
" code:",
|
||||||
|
" primary: openai-codex/codex",
|
||||||
|
" summarize:",
|
||||||
|
" primary: anthropic/haiku",
|
||||||
|
" continuity:",
|
||||||
|
" primary: local/local_small",
|
||||||
|
"",
|
||||||
|
].join("\n"),
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
|
||||||
|
const cfg = await loadRouterConfig(dir);
|
||||||
|
expect(cfg).not.toBeNull();
|
||||||
|
|
||||||
|
expect(resolveRouteDecision({ cfg, intent: "chat", defaultModelRef }).fallbacks).toEqual([
|
||||||
|
{ provider: "anthropic", model: "sonnet" },
|
||||||
|
]);
|
||||||
|
expect(resolveRouteDecision({ cfg, intent: "chat", defaultModelRef }).provider).toBe(
|
||||||
|
"anthropic",
|
||||||
|
);
|
||||||
|
expect(resolveRouteDecision({ cfg, intent: "chat", defaultModelRef }).model).toBe("haiku");
|
||||||
|
|
||||||
|
const strategy = resolveRouteDecision({ cfg, intent: "strategy", defaultModelRef });
|
||||||
|
expect(strategy.provider).toBe("anthropic");
|
||||||
|
expect(strategy.model).toBe("sonnet");
|
||||||
|
|
||||||
|
const code = resolveRouteDecision({ cfg, intent: "code", defaultModelRef });
|
||||||
|
expect(code.provider).toBe("openai-codex");
|
||||||
|
expect(code.model).toBe("codex");
|
||||||
|
|
||||||
|
const summarize = resolveRouteDecision({ cfg, intent: "summarize", defaultModelRef });
|
||||||
|
expect(summarize.provider).toBe("anthropic");
|
||||||
|
expect(summarize.model).toBe("haiku");
|
||||||
|
|
||||||
|
const continuity = resolveRouteDecision({ cfg, intent: "continuity", defaultModelRef });
|
||||||
|
expect(continuity.provider).toBe("local");
|
||||||
|
expect(continuity.model).toBe("local_small");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("escalates chat/strategy to opus based on complexity policy", async () => {
|
||||||
|
await withTempDir(async (dir) => {
|
||||||
|
await fs.writeFile(
|
||||||
|
path.join(dir, "routing.yaml"),
|
||||||
|
[
|
||||||
|
"intents:",
|
||||||
|
" chat:",
|
||||||
|
" primary: anthropic/haiku",
|
||||||
|
" strategy:",
|
||||||
|
" primary: anthropic/sonnet",
|
||||||
|
"",
|
||||||
|
].join("\n"),
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
await fs.writeFile(
|
||||||
|
path.join(dir, "policy.yaml"),
|
||||||
|
["complexity:", " context_tokens_ge: 2000", " target: anthropic/opus", ""].join("\n"),
|
||||||
|
"utf-8",
|
||||||
|
);
|
||||||
|
|
||||||
|
const cfg = await loadRouterConfig(dir);
|
||||||
|
const chat = resolveRouteDecision({
|
||||||
|
cfg,
|
||||||
|
intent: "chat",
|
||||||
|
defaultModelRef,
|
||||||
|
contextTokens: 2000,
|
||||||
|
});
|
||||||
|
const strategy = resolveRouteDecision({
|
||||||
|
cfg,
|
||||||
|
intent: "strategy",
|
||||||
|
defaultModelRef,
|
||||||
|
contextTokens: 2500,
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(chat.provider).toBe("anthropic");
|
||||||
|
expect(chat.model).toBe("opus");
|
||||||
|
expect(chat.reason).toBe("complexity");
|
||||||
|
|
||||||
|
expect(strategy.provider).toBe("anthropic");
|
||||||
|
expect(strategy.model).toBe("opus");
|
||||||
|
expect(strategy.reason).toBe("complexity");
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
Loading…
Reference in New Issue
Block a user