diff --git a/docs/auto-think.md b/docs/auto-think.md new file mode 100644 index 000000000..4ad3154ca --- /dev/null +++ b/docs/auto-think.md @@ -0,0 +1,124 @@ +--- +summary: "Auto-think: Automatic thinking level classification based on message content" +read_when: + - You want to enable automatic thinking level selection + - You want to configure auto-think heuristics or rules +--- +# Auto-Think + +Auto-think automatically classifies incoming messages and selects an appropriate thinking level without requiring explicit `/think` directives from users. + +## Overview + +When enabled, auto-think analyzes each incoming message using heuristics to determine complexity: + +- **High complexity**: Debug requests, security reviews, architecture discussions, large code blocks +- **Medium complexity**: How-to questions, implementation requests, step-by-step guides, comparisons +- **Low complexity**: Simple lookups, definitions, format/translation requests +- **Minimal/Off**: Short messages, greetings, unclassified content + +## Configuration + +Enable auto-think in your agent config: + +```yaml +agents: + defaults: + autoThink: + enabled: true +``` + +### Full options + +```yaml +agents: + defaults: + autoThink: + enabled: true + floor: "off" # Never go below this level + ceiling: "high" # Never go above this level + rules: # Custom patterns (optional) + - match: "newsletter" + level: "medium" + - match: "security|audit" + level: "high" +``` + +### Config reference + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `enabled` | boolean | `false` | Enable auto-think classification | +| `floor` | ThinkLevel | `"off"` | Minimum thinking level | +| `ceiling` | ThinkLevel | `"high"` | Maximum thinking level | +| `rules` | Array | `[]` | Custom pattern rules | + +### Custom rules + +Rules are evaluated in order; the first match wins. Each rule has: + +- `match`: A regex pattern (case-insensitive) or plain string +- `level`: The thinking level to use when matched + +Example: + +```yaml +rules: + - match: "urgent|asap" + level: "high" + - match: "quick question" + level: "off" +``` + +## Override behavior + +Auto-think respects the existing directive precedence: + +1. **Inline directive** (`/t high` in the message) — always wins +2. **Session sticky** (previous `/think:high` directive-only message) +3. **Auto-think classification** — when enabled and no directive present +4. **Default level** (`thinkingDefault` config) + +Users can always override auto-think with explicit `/think` directives. + +## Built-in heuristics + +### High complexity signals + +- Debug/debugging requests +- Error messages, stack traces, exceptions +- Security, vulnerability, audit keywords +- Architecture, design pattern discussions +- Refactoring, optimization requests +- Code blocks > 500 characters + +### Medium complexity signals + +- "How do/would/should/can" questions +- "Explain", "analyze", "compare" requests +- Step-by-step guides +- Implementation/build/create requests +- Planning, strategy discussions +- Code blocks > 100 characters +- Messages > 2000 characters + +### Low complexity signals + +- Simple "What is X?" questions +- Translation/conversion requests +- List/enumerate requests +- Definition lookups + +### Fallbacks + +- Very short messages (< 50 chars): `off` +- Unclassified messages: `minimal` + +## Performance + +Auto-think uses pure regex heuristics with no additional API calls. Classification adds negligible latency (< 1ms). + +## Related + +- [Thinking levels](/tools/thinking) — Manual thinking control via `/think` directives +- [Token use](/token-use) — Cost implications of thinking levels diff --git a/src/auto-reply/auto-think.test.ts b/src/auto-reply/auto-think.test.ts new file mode 100644 index 000000000..b1e4fad0e --- /dev/null +++ b/src/auto-reply/auto-think.test.ts @@ -0,0 +1,184 @@ +import { describe, expect, it } from "vitest"; +import { classifyThinkLevel, type AutoThinkConfig } from "./auto-think.js"; + +describe("auto-think", () => { + describe("classifyThinkLevel", () => { + it("returns undefined when disabled", () => { + expect(classifyThinkLevel("debug this code", { enabled: false })).toBeUndefined(); + expect(classifyThinkLevel("debug this code", undefined)).toBeUndefined(); + expect(classifyThinkLevel("debug this code", {})).toBeUndefined(); + }); + + it("returns undefined for empty messages", () => { + const config: AutoThinkConfig = { enabled: true }; + expect(classifyThinkLevel("", config)).toBeUndefined(); + expect(classifyThinkLevel(" ", config)).toBeUndefined(); + }); + + describe("high complexity detection", () => { + const config: AutoThinkConfig = { enabled: true }; + + it("detects debug-related messages", () => { + expect(classifyThinkLevel("Can you debug this code?", config)).toBe("high"); + expect(classifyThinkLevel("I'm debugging an issue", config)).toBe("high"); + }); + + it("detects error-related messages", () => { + expect(classifyThinkLevel("Getting an error: TypeError", config)).toBe("high"); + expect(classifyThinkLevel("Here's the stack trace", config)).toBe("high"); + expect(classifyThinkLevel("Exception thrown at line 42", config)).toBe("high"); + }); + + it("detects security-related messages", () => { + expect(classifyThinkLevel("Review for security vulnerabilities", config)).toBe("high"); + expect(classifyThinkLevel("Is this code vulnerable to SQL injection?", config)).toBe( + "high", + ); + expect(classifyThinkLevel("Security audit needed", config)).toBe("high"); + }); + + it("detects architecture-related messages", () => { + expect(classifyThinkLevel("Help me architect this system", config)).toBe("high"); + expect(classifyThinkLevel("What design pattern should I use?", config)).toBe("high"); + expect(classifyThinkLevel("System design for a chat app", config)).toBe("high"); + }); + + it("detects large code blocks", () => { + const largeCode = "```\n" + "x".repeat(600) + "\n```"; + expect(classifyThinkLevel(largeCode, config)).toBe("high"); + }); + }); + + describe("medium complexity detection", () => { + const config: AutoThinkConfig = { enabled: true }; + + it("detects how-to questions", () => { + expect(classifyThinkLevel("How do I implement a linked list?", config)).toBe("medium"); + expect(classifyThinkLevel("How should I structure this?", config)).toBe("medium"); + expect(classifyThinkLevel("How can I improve performance?", config)).toBe("medium"); + }); + + it("detects analysis requests", () => { + expect(classifyThinkLevel("Explain how this algorithm works", config)).toBe("medium"); + expect(classifyThinkLevel("Analyze this data structure", config)).toBe("medium"); + expect(classifyThinkLevel("Compare these two approaches", config)).toBe("medium"); + }); + + it("detects step-by-step requests", () => { + expect(classifyThinkLevel("Walk me through this step by step", config)).toBe("medium"); + expect(classifyThinkLevel("Give me a step-by-step guide", config)).toBe("medium"); + }); + + it("detects implementation requests", () => { + expect(classifyThinkLevel("Implement a binary search tree", config)).toBe("medium"); + expect(classifyThinkLevel("Build a REST API for this", config)).toBe("medium"); + expect(classifyThinkLevel("Create a function that does X", config)).toBe("medium"); + }); + + it("detects medium code blocks", () => { + const mediumCode = "```\n" + "x".repeat(150) + "\n```"; + expect(classifyThinkLevel(mediumCode, config)).toBe("medium"); + }); + }); + + describe("low complexity detection", () => { + const config: AutoThinkConfig = { enabled: true }; + + it("detects simple what/when/where questions", () => { + expect(classifyThinkLevel("What is a closure?", config)).toBe("low"); + expect(classifyThinkLevel("When was Python released?", config)).toBe("low"); + expect(classifyThinkLevel("Where is the config file?", config)).toBe("low"); + }); + + it("detects translation/conversion requests", () => { + expect(classifyThinkLevel("Translate this to Spanish", config)).toBe("low"); + expect(classifyThinkLevel("Convert this to JSON", config)).toBe("low"); + expect(classifyThinkLevel("Reformat this code", config)).toBe("low"); + }); + + it("detects simple list requests", () => { + expect(classifyThinkLevel("List the programming languages", config)).toBe("low"); + expect(classifyThinkLevel("Name all the HTTP methods", config)).toBe("low"); + }); + }); + + describe("length-based fallbacks", () => { + const config: AutoThinkConfig = { enabled: true }; + + it("returns off for very short messages", () => { + expect(classifyThinkLevel("hi", config)).toBe("off"); + expect(classifyThinkLevel("thanks", config)).toBe("off"); + }); + + it("returns medium for very long messages", () => { + const longMessage = "a".repeat(2500); + expect(classifyThinkLevel(longMessage, config)).toBe("medium"); + }); + + it("returns minimal for unclassified messages", () => { + expect( + classifyThinkLevel("Here's some random text that doesn't match patterns", config), + ).toBe("minimal"); + }); + }); + + describe("floor and ceiling", () => { + it("respects floor setting", () => { + const config: AutoThinkConfig = { enabled: true, floor: "low" }; + expect(classifyThinkLevel("hi", config)).toBe("low"); // Would be "off" without floor + }); + + it("respects ceiling setting", () => { + const config: AutoThinkConfig = { enabled: true, ceiling: "medium" }; + expect(classifyThinkLevel("debug this security vulnerability", config)).toBe("medium"); // Would be "high" without ceiling + }); + + it("respects both floor and ceiling", () => { + const config: AutoThinkConfig = { enabled: true, floor: "low", ceiling: "medium" }; + expect(classifyThinkLevel("hi", config)).toBe("low"); + expect(classifyThinkLevel("debug this", config)).toBe("medium"); + }); + }); + + describe("custom rules", () => { + it("applies custom rules before built-in patterns", () => { + const config: AutoThinkConfig = { + enabled: true, + rules: [{ match: "newsletter", level: "medium" }], + }; + expect(classifyThinkLevel("Write the newsletter", config)).toBe("medium"); + }); + + it("supports regex patterns in rules", () => { + const config: AutoThinkConfig = { + enabled: true, + rules: [{ match: "urgent|asap|immediately", level: "high" }], + }; + expect(classifyThinkLevel("Fix this ASAP", config)).toBe("high"); + expect(classifyThinkLevel("This is urgent", config)).toBe("high"); + }); + + it("skips invalid regex patterns", () => { + const config: AutoThinkConfig = { + enabled: true, + rules: [ + { match: "[invalid(regex", level: "high" }, + { match: "valid", level: "medium" }, + ], + }; + expect(classifyThinkLevel("This is valid", config)).toBe("medium"); + }); + + it("first matching rule wins", () => { + const config: AutoThinkConfig = { + enabled: true, + rules: [ + { match: "first", level: "low" }, + { match: "first", level: "high" }, + ], + }; + expect(classifyThinkLevel("first match wins", config)).toBe("low"); + }); + }); + }); +}); diff --git a/src/auto-reply/auto-think.ts b/src/auto-reply/auto-think.ts new file mode 100644 index 000000000..461087b64 --- /dev/null +++ b/src/auto-reply/auto-think.ts @@ -0,0 +1,143 @@ +/** + * Auto-think: Heuristic-based thinking level classification. + * + * Analyzes incoming message content and determines an appropriate thinking level + * without requiring explicit user directives. + */ + +import type { ThinkLevel } from "./thinking.js"; + +export interface AutoThinkConfig { + /** Enable auto-think classification */ + enabled?: boolean; + /** Minimum thinking level (floor) */ + floor?: ThinkLevel; + /** Maximum thinking level (ceiling) */ + ceiling?: ThinkLevel; + /** Custom pattern rules (evaluated in order, first match wins) */ + rules?: AutoThinkRule[]; +} + +export interface AutoThinkRule { + /** Regex pattern or string to match (case-insensitive) */ + match: string; + /** Thinking level to use when matched */ + level: ThinkLevel; +} + +const THINK_LEVEL_ORDER: ThinkLevel[] = ["off", "minimal", "low", "medium", "high", "xhigh"]; + +function clampLevel(level: ThinkLevel, floor?: ThinkLevel, ceiling?: ThinkLevel): ThinkLevel { + const levelIdx = THINK_LEVEL_ORDER.indexOf(level); + const floorIdx = floor ? THINK_LEVEL_ORDER.indexOf(floor) : 0; + const ceilingIdx = ceiling ? THINK_LEVEL_ORDER.indexOf(ceiling) : THINK_LEVEL_ORDER.length - 1; + + if (levelIdx < floorIdx) return floor!; + if (levelIdx > ceilingIdx) return ceiling!; + return level; +} + +/** + * High-complexity patterns that warrant deeper thinking. + * Debug, security, architecture, multi-step problems. + */ +const HIGH_PATTERNS = [ + /\b(debug|debugging|debugger)\b/i, + /\b(error|exception|traceback|stack\s*trace)\b/i, + /\b(security|vulnerable|vulnerability|exploit|cve|audit)\b/i, + /\b(architect|architecture|design\s+pattern|system\s+design)\b/i, + /\b(refactor|rewrite|restructure)\b/i, + /\b(optimize|optimization|performance\s+issue)\b/i, + /\b(race\s+condition|deadlock|memory\s+leak)\b/i, + /\b(review|code\s+review|pr\s+review)\b/i, + /```[\s\S]{500,}/i, // Large code blocks (500+ chars) +]; + +/** + * Medium-complexity patterns that benefit from structured thinking. + * Multi-step tasks, comparisons, analysis, implementation. + */ +const MEDIUM_PATTERNS = [ + /\b(how\s+(do|would|should|can|to))\b/i, + /\b(explain|analyze|compare|contrast|evaluate)\b/i, + /\b(step[\s-]?by[\s-]?step|walkthrough|guide\s+me)\b/i, + /\b(implement|build|create|develop|write)\b/i, + /\b(plan|strategy|approach|roadmap)\b/i, + /\b(trade[\s-]?off|pros?\s+and\s+cons?|advantages?\s+and\s+disadvantages?)\b/i, + /\b(multiple|several|various|different)\s+(ways?|options?|approaches?|methods?)\b/i, + /```[\s\S]{100,}/i, // Medium code blocks (100+ chars) +]; + +/** + * Low-complexity patterns that need minimal thinking. + * Simple lookups, translations, formatting. + */ +const LOW_PATTERNS = [ + /^(what|when|where|who|which)\s+(is|are|was|were)\b/i, + /\b(translate|convert|format|reformat)\b/i, + /\b(list|enumerate|name)\s+(the|all|some)\b/i, + /\b(define|definition\s+of)\b/i, +]; + +/** + * Classify the thinking level for a message using heuristics. + * + * @param message - The user's message content + * @param config - Optional auto-think configuration + * @returns The classified thinking level, or undefined if auto-think is disabled + */ +export function classifyThinkLevel( + message: string, + config?: AutoThinkConfig, +): ThinkLevel | undefined { + if (!config?.enabled) return undefined; + if (!message?.trim()) return undefined; + + const text = message.trim(); + + // Check custom rules first (if any) + if (config.rules?.length) { + for (const rule of config.rules) { + try { + const pattern = new RegExp(rule.match, "i"); + if (pattern.test(text)) { + return clampLevel(rule.level, config.floor, config.ceiling); + } + } catch { + // Invalid regex, skip + } + } + } + + // Check for high complexity signals + for (const pattern of HIGH_PATTERNS) { + if (pattern.test(text)) { + return clampLevel("high", config.floor, config.ceiling); + } + } + + // Check for medium complexity signals + for (const pattern of MEDIUM_PATTERNS) { + if (pattern.test(text)) { + return clampLevel("medium", config.floor, config.ceiling); + } + } + + // Check for low complexity signals + for (const pattern of LOW_PATTERNS) { + if (pattern.test(text)) { + return clampLevel("low", config.floor, config.ceiling); + } + } + + // Length-based heuristics as fallback + if (text.length > 2000) { + return clampLevel("medium", config.floor, config.ceiling); + } + if (text.length < 50) { + return clampLevel("off", config.floor, config.ceiling); + } + + // Default: minimal thinking for unclassified messages + return clampLevel("minimal", config.floor, config.ceiling); +} diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index ba96023ce..d112cc250 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -21,6 +21,7 @@ import { isReasoningTagProvider } from "../../utils/provider-utils.js"; import { hasControlCommand } from "../command-detection.js"; import { buildInboundMediaNote } from "../media-note.js"; import type { MsgContext, TemplateContext } from "../templating.js"; +import { classifyThinkLevel } from "../auto-think.js"; import { type ElevatedLevel, formatXHighModelHint, @@ -261,6 +262,14 @@ export async function runPreparedReply( prefixedCommandBody = parts.slice(1).join(" ").trim(); } } + // Auto-think: classify thinking level based on message content + if (!resolvedThinkLevel && agentCfg?.autoThink?.enabled) { + const autoLevel = classifyThinkLevel(prefixedCommandBody, agentCfg.autoThink); + if (autoLevel) { + resolvedThinkLevel = autoLevel; + logVerbose(`Auto-think classified message as "${autoLevel}"`); + } + } if (!resolvedThinkLevel) { resolvedThinkLevel = await modelState.resolveDefaultThinkingLevel(); } diff --git a/src/config/types.agent-defaults.ts b/src/config/types.agent-defaults.ts index 9c6ce0211..3508a4614 100644 --- a/src/config/types.agent-defaults.ts +++ b/src/config/types.agent-defaults.ts @@ -134,6 +134,26 @@ export type AgentDefaultsConfig = { memorySearch?: MemorySearchConfig; /** Default thinking level when no /think directive is present. */ thinkingDefault?: "off" | "minimal" | "low" | "medium" | "high" | "xhigh"; + /** + * Auto-think: Automatically classify thinking level based on message content. + * When enabled, analyzes the incoming message using heuristics to determine + * the appropriate thinking level without requiring explicit /think directives. + */ + autoThink?: { + /** Enable auto-think classification. */ + enabled?: boolean; + /** Minimum thinking level (floor). */ + floor?: "off" | "minimal" | "low" | "medium" | "high"; + /** Maximum thinking level (ceiling). */ + ceiling?: "off" | "minimal" | "low" | "medium" | "high"; + /** Custom pattern rules (evaluated in order, first match wins). */ + rules?: Array<{ + /** Regex pattern or string to match (case-insensitive). */ + match: string; + /** Thinking level to use when matched. */ + level: "off" | "minimal" | "low" | "medium" | "high"; + }>; + }; /** Default verbose level when no /verbose directive is present. */ verboseDefault?: "off" | "on" | "full"; /** Default elevated level when no /elevated directive is present. */ diff --git a/src/config/zod-schema.agent-defaults.ts b/src/config/zod-schema.agent-defaults.ts index a849078ed..f17f4ea04 100644 --- a/src/config/zod-schema.agent-defaults.ts +++ b/src/config/zod-schema.agent-defaults.ts @@ -113,6 +113,46 @@ export const AgentDefaultsSchema = z z.literal("xhigh"), ]) .optional(), + autoThink: z + .object({ + enabled: z.boolean().optional(), + floor: z + .union([ + z.literal("off"), + z.literal("minimal"), + z.literal("low"), + z.literal("medium"), + z.literal("high"), + ]) + .optional(), + ceiling: z + .union([ + z.literal("off"), + z.literal("minimal"), + z.literal("low"), + z.literal("medium"), + z.literal("high"), + ]) + .optional(), + rules: z + .array( + z + .object({ + match: z.string(), + level: z.union([ + z.literal("off"), + z.literal("minimal"), + z.literal("low"), + z.literal("medium"), + z.literal("high"), + ]), + }) + .strict(), + ) + .optional(), + }) + .strict() + .optional(), verboseDefault: z.union([z.literal("off"), z.literal("on"), z.literal("full")]).optional(), elevatedDefault: z .union([z.literal("off"), z.literal("on"), z.literal("ask"), z.literal("full")])