From 1cb4af074f0b2292c599993227f01114422ccfb6 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Tue, 27 Jan 2026 21:57:15 -0800 Subject: [PATCH 1/9] feat (memory): Implement new (opt-in) QMD memory backend --- CHANGELOG.md | 3 + docs/concepts/memory.md | 78 +++ src/agents/cli-runner/helpers.ts | 1 + src/agents/pi-embedded-runner/compact.ts | 1 + src/agents/pi-embedded-runner/run/attempt.ts | 1 + .../pi-embedded-runner/system-prompt.ts | 3 + src/agents/system-prompt.ts | 28 +- .../tools/memory-tool.citations.test.ts | 65 ++ src/agents/tools/memory-tool.ts | 33 +- .../reply/commands-context-report.ts | 1 + src/commands/status.scan.ts | 10 +- src/config/schema.ts | 52 ++ src/config/types.clawdbot.ts | 2 + src/config/types.memory.ts | 46 ++ src/config/types.ts | 1 + src/config/zod-schema.session.ts | 51 +- src/config/zod-schema.ts | 63 +- src/memory/backend-config.test.ts | 58 ++ src/memory/backend-config.ts | 245 +++++++ src/memory/index.ts | 3 +- src/memory/manager.ts | 63 +- src/memory/qmd-manager.ts | 612 ++++++++++++++++++ src/memory/search-manager.test.ts | 62 ++ src/memory/search-manager.ts | 157 ++++- src/memory/types.ts | 72 +++ 25 files changed, 1621 insertions(+), 90 deletions(-) create mode 100644 src/agents/tools/memory-tool.citations.test.ts create mode 100644 src/config/types.memory.ts create mode 100644 src/memory/backend-config.test.ts create mode 100644 src/memory/backend-config.ts create mode 100644 src/memory/qmd-manager.ts create mode 100644 src/memory/search-manager.test.ts create mode 100644 src/memory/types.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5909c9899..5551f1e84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ Status: beta. - Docs: add GCP Compute Engine deployment guide. (#1848) Thanks @hougangdev. - Docs: add LINE channel guide. Thanks @thewilloftheshadow. - Docs: credit both contributors for Control UI refresh. (#1852) Thanks @EnzeD. +- Memory: add optional QMD-backed memory backend with fallbacks to the existing Markdown index. Thanks @vgnsh. +- Memory: surface QMD startup errors so we fall back to the legacy index when the CLI is missing, and allow extra QMD collections outside the workspace to return snippets via the `qmd//…` path prefix. +- Memory: export session transcripts into the QMD index when `memory.qmd.sessions.enabled` is set and honor `memory.citations` when formatting snippets. - Onboarding: add Venice API key to non-interactive flow. (#1893) Thanks @jonisjongithub. - Onboarding: strengthen security warning copy for beta + access control expectations. - Tlon: format thread reply IDs as @ud. (#1837) Thanks @wca4a. diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 8a386aba9..0a558a54d 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -96,6 +96,84 @@ embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or `models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint, set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`). +### QMD backend (experimental) + +Set `memory.backend = "qmd"` to swap the built-in SQLite indexer for +[QMD](https://github.com/tobi/qmd): a local-first search sidecar that combines +BM25 + vectors + reranking. Markdown stays the source of truth; Moltbot shells +out to QMD for retrieval. Key points: + +**Prereqs** +- Disabled by default. Opt in per-config (`memory.backend = "qmd"`). +- Install the QMD CLI separately (`bun install -g github.com/tobi/qmd` or grab + a release) and make sure the `qmd` binary is on the gateway’s `PATH`. +- QMD needs an SQLite build that allows extensions (`brew install sqlite` on + macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically. + +**How the sidecar runs** +- The gateway writes a self-contained QMD home under + `~/.clawdbot/agents//qmd/` (config + cache + sqlite DB). +- Collections are rewritten from `memory.qmd.paths` (plus default workspace + memory files) into `index.yml`, then `qmd update` + `qmd embed` run on boot and + on a configurable interval (`memory.qmd.update.interval`, default 5 m). +- Searches run via `qmd query --json`. If QMD fails or the binary is missing, + Moltbot automatically falls back to the builtin SQLite manager so memory tools + keep working. + +**Config surface (`memory.qmd.*`)** +- `command` (default `qmd`): override the executable path. +- `includeDefaultMemory` (default `true`): auto-index `MEMORY.md` + `memory/**/*.md`. +- `paths[]`: add extra directories/files (`path`, optional `pattern`, optional + stable `name`). +- `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`, + `exportDir`, `redactToolOutputs`—defaults to redacting tool payloads). +- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`). +- `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`, + `maxInjectedChars`, `timeoutMs`). +- `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy). + Default is DM-only (`deny` all, `allow` direct chats); loosen it to surface QMD + hits in groups/channels. +- Snippets sourced outside the workspace show up as + `qmd//` in `memory_search` results; `memory_get` + understands that prefix and reads from the configured QMD collection root. +- When `memory.qmd.sessions.enabled = true`, Moltbot exports sanitized session + transcripts (User/Assistant turns) into a dedicated QMD collection under + `~/.clawdbot/agents//qmd/sessions/`, so `memory_search` can recall recent + conversations without touching the builtin SQLite index. +- `memory_search` snippets now include a `Source: ` footer when + `memory.citations` is `auto`/`on`; set `memory.citations = "off"` to keep + the path metadata internal (the agent still receives the path for + `memory_get`, but the snippet text omits the footer and the system prompt + warns the agent not to cite it). + +**Example** + +```json5 +memory: { + backend: "qmd", + citations: "auto", + qmd: { + includeDefaultMemory: true, + update: { interval: "5m", debounceMs: 15000 }, + limits: { maxResults: 6, timeoutMs: 4000 }, + scope: { + default: "deny", + rules: [{ action: "allow", match: { chatType: "direct" } }] + }, + paths: [ + { name: "docs", path: "~/notes", pattern: "**/*.md" } + ] + } +} +``` + +**Citations & fallback** +- `memory.citations` applies regardless of backend (`auto`/`on`/`off`). +- When `qmd` runs, we tag `status().backend = "qmd"` so diagnostics show which + engine served the results. If the QMD subprocess exits or JSON output can’t be + parsed, the search manager logs a warning and returns the builtin provider + (existing Markdown embeddings) until QMD recovers. + ### Gemini embeddings (native) Set the provider to `gemini` to use the Gemini embeddings API directly: diff --git a/src/agents/cli-runner/helpers.ts b/src/agents/cli-runner/helpers.ts index 833771a1f..008ed45d1 100644 --- a/src/agents/cli-runner/helpers.ts +++ b/src/agents/cli-runner/helpers.ts @@ -212,6 +212,7 @@ export function buildSystemPrompt(params: { userTimeFormat, contextFiles: params.contextFiles, ttsHint, + memoryCitationsMode: params.config?.memory?.citations, }); } diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index dc68561c2..b592eded8 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -346,6 +346,7 @@ export async function compactEmbeddedPiSessionDirect( userTime, userTimeFormat, contextFiles, + memoryCitationsMode: params.config?.memory?.citations, }); const systemPrompt = createSystemPromptOverride(appendPrompt); diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index 46a53bd8f..9d2614a99 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -359,6 +359,7 @@ export async function runEmbeddedAttempt( userTime, userTimeFormat, contextFiles, + memoryCitationsMode: params.config?.memory?.citations, }); const systemPromptReport = buildSystemPromptReport({ source: "run", diff --git a/src/agents/pi-embedded-runner/system-prompt.ts b/src/agents/pi-embedded-runner/system-prompt.ts index 33c4bfc8b..63f6a8074 100644 --- a/src/agents/pi-embedded-runner/system-prompt.ts +++ b/src/agents/pi-embedded-runner/system-prompt.ts @@ -1,4 +1,5 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; +import type { MemoryCitationsMode } from "../../config/types.memory.js"; import type { ResolvedTimeFormat } from "../date-time.js"; import type { EmbeddedContextFile } from "../pi-embedded-helpers.js"; import { buildAgentSystemPrompt, type PromptMode } from "../system-prompt.js"; @@ -45,6 +46,7 @@ export function buildEmbeddedSystemPrompt(params: { userTime?: string; userTimeFormat?: ResolvedTimeFormat; contextFiles?: EmbeddedContextFile[]; + memoryCitationsMode?: MemoryCitationsMode; }): string { return buildAgentSystemPrompt({ workspaceDir: params.workspaceDir, @@ -70,6 +72,7 @@ export function buildEmbeddedSystemPrompt(params: { userTime: params.userTime, userTimeFormat: params.userTimeFormat, contextFiles: params.contextFiles, + memoryCitationsMode: params.memoryCitationsMode, }); } diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index ed97fd539..5a613f7af 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -1,6 +1,7 @@ import type { ReasoningLevel, ThinkLevel } from "../auto-reply/thinking.js"; import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import { listDeliverableMessageChannels } from "../utils/message-channel.js"; +import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { ResolvedTimeFormat } from "./date-time.js"; import type { EmbeddedContextFile } from "./pi-embedded-helpers.js"; @@ -32,16 +33,30 @@ function buildSkillsSection(params: { ]; } -function buildMemorySection(params: { isMinimal: boolean; availableTools: Set }) { +function buildMemorySection(params: { + isMinimal: boolean; + availableTools: Set; + citationsMode?: MemoryCitationsMode; +}) { if (params.isMinimal) return []; if (!params.availableTools.has("memory_search") && !params.availableTools.has("memory_get")) { return []; } - return [ + const lines = [ "## Memory Recall", "Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.", - "", ]; + if (params.citationsMode === "off") { + lines.push( + "Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.", + ); + } else { + lines.push( + "Citations: include Source: when it helps the user verify memory snippets.", + ); + } + lines.push(""); + return lines; } function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: boolean) { @@ -178,6 +193,7 @@ export function buildAgentSystemPrompt(params: { level: "minimal" | "extensive"; channel: string; }; + memoryCitationsMode?: MemoryCitationsMode; }) { const coreToolSummaries: Record = { read: "Read file contents", @@ -314,7 +330,11 @@ export function buildAgentSystemPrompt(params: { isMinimal, readToolName, }); - const memorySection = buildMemorySection({ isMinimal, availableTools }); + const memorySection = buildMemorySection({ + isMinimal, + availableTools, + citationsMode: params.memoryCitationsMode, + }); const docsSection = buildDocsSection({ docsPath: params.docsPath, isMinimal, diff --git a/src/agents/tools/memory-tool.citations.test.ts b/src/agents/tools/memory-tool.citations.test.ts new file mode 100644 index 000000000..111525822 --- /dev/null +++ b/src/agents/tools/memory-tool.citations.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const stubManager = { + search: vi.fn(async () => [ + { + path: "MEMORY.md", + startLine: 5, + endLine: 7, + score: 0.9, + snippet: "@@ -5,3 @@\nAssistant: noted", + source: "memory" as const, + }, + ]), + readFile: vi.fn(), + status: () => ({ + backend: "builtin" as const, + files: 1, + chunks: 1, + dirty: false, + workspaceDir: "/workspace", + dbPath: "/workspace/.memory/index.sqlite", + provider: "builtin", + model: "builtin", + requestedProvider: "builtin", + sources: ["memory" as const], + sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }], + }), + sync: vi.fn(), + probeVectorAvailability: vi.fn(async () => true), + close: vi.fn(), +}; + +vi.mock("../../memory/index.js", () => { + return { + getMemorySearchManager: async () => ({ manager: stubManager }), + }; +}); + +import { createMemorySearchTool } from "./memory-tool.js"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("memory search citations", () => { + it("appends source information when citations are enabled", async () => { + const cfg = { memory: { citations: "on" }, agents: { list: [{ id: "main", default: true }] } }; + const tool = createMemorySearchTool({ config: cfg }); + if (!tool) throw new Error("tool missing"); + const result = await tool.execute("call_citations_on", { query: "notes" }); + const details = result.details as { results: Array<{ snippet: string; citation?: string }> }; + expect(details.results[0]?.snippet).toMatch(/Source: MEMORY.md#L5-L7/); + expect(details.results[0]?.citation).toBe("MEMORY.md#L5-L7"); + }); + + it("leaves snippet untouched when citations are off", async () => { + const cfg = { memory: { citations: "off" }, agents: { list: [{ id: "main", default: true }] } }; + const tool = createMemorySearchTool({ config: cfg }); + if (!tool) throw new Error("tool missing"); + const result = await tool.execute("call_citations_off", { query: "notes" }); + const details = result.details as { results: Array<{ snippet: string; citation?: string }> }; + expect(details.results[0]?.snippet).not.toMatch(/Source:/); + expect(details.results[0]?.citation).toBeUndefined(); + }); +}); diff --git a/src/agents/tools/memory-tool.ts b/src/agents/tools/memory-tool.ts index b7b619af3..f6643bb67 100644 --- a/src/agents/tools/memory-tool.ts +++ b/src/agents/tools/memory-tool.ts @@ -1,7 +1,9 @@ import { Type } from "@sinclair/typebox"; import type { MoltbotConfig } from "../../config/config.js"; +import type { MemoryCitationsMode } from "../../config/types.memory.js"; import { getMemorySearchManager } from "../../memory/index.js"; +import type { MemorySearchResult } from "../../memory/types.js"; import { resolveSessionAgentId } from "../agent-scope.js"; import { resolveMemorySearchConfig } from "../memory-search.js"; import type { AnyAgentTool } from "./common.js"; @@ -48,17 +50,21 @@ export function createMemorySearchTool(options: { return jsonResult({ results: [], disabled: true, error }); } try { - const results = await manager.search(query, { + const citationsMode = resolveMemoryCitationsMode(cfg); + const includeCitations = citationsMode !== "off"; + const rawResults = await manager.search(query, { maxResults, minScore, sessionKey: options.agentSessionKey, }); const status = manager.status(); + const results = decorateCitations(rawResults, includeCitations); return jsonResult({ results, provider: status.provider, model: status.model, fallback: status.fallback, + citations: citationsMode, }); } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -110,3 +116,28 @@ export function createMemoryGetTool(options: { }, }; } + +function resolveMemoryCitationsMode(cfg: MoltbotConfig): MemoryCitationsMode { + const mode = cfg.memory?.citations; + if (mode === "on" || mode === "off" || mode === "auto") return mode; + return "auto"; +} + +function decorateCitations(results: MemorySearchResult[], include: boolean): MemorySearchResult[] { + if (!include) { + return results.map((entry) => ({ ...entry, citation: undefined })); + } + return results.map((entry) => { + const citation = formatCitation(entry); + const snippet = `${entry.snippet.trim()}\n\nSource: ${citation}`; + return { ...entry, citation, snippet }; + }); +} + +function formatCitation(entry: MemorySearchResult): string { + const lineRange = + entry.startLine === entry.endLine + ? `#L${entry.startLine}` + : `#L${entry.startLine}-L${entry.endLine}`; + return `${entry.path}${lineRange}`; +} diff --git a/src/auto-reply/reply/commands-context-report.ts b/src/auto-reply/reply/commands-context-report.ts index 8a65d42f5..975d5e331 100644 --- a/src/auto-reply/reply/commands-context-report.ts +++ b/src/auto-reply/reply/commands-context-report.ts @@ -150,6 +150,7 @@ async function resolveContextReport( ttsHint, runtimeInfo, sandboxInfo, + memoryCitationsMode: params.cfg?.memory?.citations, }); return buildSystemPromptReport({ diff --git a/src/commands/status.scan.ts b/src/commands/status.scan.ts index 13de0a383..5785e0ab8 100644 --- a/src/commands/status.scan.ts +++ b/src/commands/status.scan.ts @@ -6,7 +6,8 @@ import { probeGateway } from "../gateway/probe.js"; import { collectChannelStatusIssues } from "../infra/channels-status-issues.js"; import { resolveOsSummary } from "../infra/os-summary.js"; import { getTailnetHostname } from "../infra/tailscale.js"; -import type { MemoryIndexManager } from "../memory/manager.js"; +import { getMemorySearchManager } from "../memory/index.js"; +import type { MemoryProviderStatus } from "../memory/types.js"; import { runExec } from "../process/exec.js"; import type { RuntimeEnv } from "../runtime.js"; import { getAgentLocalStatuses } from "./status.agent-local.js"; @@ -15,7 +16,7 @@ import { getStatusSummary } from "./status.summary.js"; import { getUpdateCheckResult } from "./status.update.js"; import { buildChannelsTable } from "./status-all/channels.js"; -type MemoryStatusSnapshot = ReturnType & { +type MemoryStatusSnapshot = MemoryProviderStatus & { agentId: string; }; @@ -151,14 +152,13 @@ export async function scanStatus( if (!memoryPlugin.enabled) return null; if (memoryPlugin.slot !== "memory-core") return null; const agentId = agentStatus.defaultId ?? "main"; - const { MemoryIndexManager } = await import("../memory/manager.js"); - const manager = await MemoryIndexManager.get({ cfg, agentId }).catch(() => null); + const { manager } = await getMemorySearchManager({ cfg, agentId }); if (!manager) return null; try { await manager.probeVectorAvailability(); } catch {} const status = manager.status(); - await manager.close().catch(() => {}); + await manager.close?.().catch(() => {}); return { agentId, ...status }; })(); progress.tick(); diff --git a/src/config/schema.ts b/src/config/schema.ts index b4ec8723b..df38a6028 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -252,6 +252,27 @@ const FIELD_LABELS: Record = { "Memory Search Hybrid Candidate Multiplier", "agents.defaults.memorySearch.cache.enabled": "Memory Search Embedding Cache", "agents.defaults.memorySearch.cache.maxEntries": "Memory Search Embedding Cache Max Entries", + memory: "Memory", + "memory.backend": "Memory Backend", + "memory.citations": "Memory Citations Mode", + "memory.qmd.command": "QMD Binary", + "memory.qmd.includeDefaultMemory": "QMD Include Default Memory", + "memory.qmd.paths": "QMD Extra Paths", + "memory.qmd.paths.path": "QMD Path", + "memory.qmd.paths.pattern": "QMD Path Pattern", + "memory.qmd.paths.name": "QMD Path Name", + "memory.qmd.sessions.enabled": "QMD Session Indexing", + "memory.qmd.sessions.exportDir": "QMD Session Export Directory", + "memory.qmd.sessions.retentionDays": "QMD Session Retention (days)", + "memory.qmd.sessions.redactToolOutputs": "QMD Session Tool Redaction", + "memory.qmd.update.interval": "QMD Update Interval", + "memory.qmd.update.debounceMs": "QMD Update Debounce (ms)", + "memory.qmd.update.onBoot": "QMD Update on Startup", + "memory.qmd.limits.maxResults": "QMD Max Results", + "memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars", + "memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars", + "memory.qmd.limits.timeoutMs": "QMD Search Timeout (ms)", + "memory.qmd.scope": "QMD Surface Scope", "auth.profiles": "Auth Profiles", "auth.order": "Auth Profile Order", "auth.cooldowns.billingBackoffHours": "Billing Backoff (hours)", @@ -537,6 +558,37 @@ const FIELD_HELP: Record = { "Multiplier for candidate pool size (default: 4).", "agents.defaults.memorySearch.cache.enabled": "Cache chunk embeddings in SQLite to speed up reindexing and frequent updates (default: true).", + memory: "Memory backend configuration (global).", + "memory.backend": 'Memory backend ("builtin" for Moltbot embeddings, "qmd" for QMD sidecar).', + "memory.citations": 'Default citation behavior ("auto", "on", or "off").', + "memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).", + "memory.qmd.includeDefaultMemory": + "Whether to automatically index MEMORY.md + memory/**/*.md (default: true).", + "memory.qmd.paths": + "Additional directories/files to index with QMD (path + optional glob pattern).", + "memory.qmd.paths.path": "Absolute or ~-relative path to index via QMD.", + "memory.qmd.paths.pattern": "Glob pattern relative to the path root (default: **/*.md).", + "memory.qmd.paths.name": + "Optional stable name for the QMD collection (default derived from path).", + "memory.qmd.sessions.enabled": + "Enable QMD session transcript indexing (experimental, default: false).", + "memory.qmd.sessions.exportDir": + "Override directory for sanitized session exports before indexing.", + "memory.qmd.sessions.retentionDays": + "Retention window for exported sessions before pruning (default: unlimited).", + "memory.qmd.sessions.redactToolOutputs": + "Strip tool call payloads/results when exporting sessions (default: true).", + "memory.qmd.update.interval": + "How often the QMD sidecar refreshes indexes (duration string, default: 5m).", + "memory.qmd.update.debounceMs": + "Minimum delay between successive QMD refresh runs (default: 15000).", + "memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).", + "memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).", + "memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).", + "memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.", + "memory.qmd.limits.timeoutMs": "Per-query timeout for QMD searches (default: 4000).", + "memory.qmd.scope": + "Session/channel scope for QMD recall (same syntax as session.sendPolicy; default: direct-only).", "agents.defaults.memorySearch.cache.maxEntries": "Optional cap on cached embeddings (best-effort).", "agents.defaults.memorySearch.sync.onSearch": diff --git a/src/config/types.clawdbot.ts b/src/config/types.clawdbot.ts index c2b2f05e4..81f0ed1c8 100644 --- a/src/config/types.clawdbot.ts +++ b/src/config/types.clawdbot.ts @@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js"; import type { PluginsConfig } from "./types.plugins.js"; import type { SkillsConfig } from "./types.skills.js"; import type { ToolsConfig } from "./types.tools.js"; +import type { MemoryConfig } from "./types.memory.js"; export type MoltbotConfig = { meta?: { @@ -95,6 +96,7 @@ export type MoltbotConfig = { canvasHost?: CanvasHostConfig; talk?: TalkConfig; gateway?: GatewayConfig; + memory?: MemoryConfig; }; export type ConfigValidationIssue = { diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts new file mode 100644 index 000000000..a6bb5eeb3 --- /dev/null +++ b/src/config/types.memory.ts @@ -0,0 +1,46 @@ +import type { SessionSendPolicyConfig } from "./types.base.js"; + +export type MemoryBackend = "builtin" | "qmd"; +export type MemoryCitationsMode = "auto" | "on" | "off"; + +export type MemoryConfig = { + backend?: MemoryBackend; + citations?: MemoryCitationsMode; + qmd?: MemoryQmdConfig; +}; + +export type MemoryQmdConfig = { + command?: string; + includeDefaultMemory?: boolean; + paths?: MemoryQmdIndexPath[]; + sessions?: MemoryQmdSessionConfig; + update?: MemoryQmdUpdateConfig; + limits?: MemoryQmdLimitsConfig; + scope?: SessionSendPolicyConfig; +}; + +export type MemoryQmdIndexPath = { + path: string; + name?: string; + pattern?: string; +}; + +export type MemoryQmdSessionConfig = { + enabled?: boolean; + exportDir?: string; + retentionDays?: number; + redactToolOutputs?: boolean; +}; + +export type MemoryQmdUpdateConfig = { + interval?: string; + debounceMs?: number; + onBoot?: boolean; +}; + +export type MemoryQmdLimitsConfig = { + maxResults?: number; + maxSnippetChars?: number; + maxInjectedChars?: number; + timeoutMs?: number; +}; diff --git a/src/config/types.ts b/src/config/types.ts index 424ccaf26..c92e11004 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -28,3 +28,4 @@ export * from "./types.telegram.js"; export * from "./types.tts.js"; export * from "./types.tools.js"; export * from "./types.whatsapp.js"; +export * from "./types.memory.js"; diff --git a/src/config/zod-schema.session.ts b/src/config/zod-schema.session.ts index 4412f5515..6841a0a90 100644 --- a/src/config/zod-schema.session.ts +++ b/src/config/zod-schema.session.ts @@ -16,6 +16,31 @@ const SessionResetConfigSchema = z }) .strict(); +export const SessionSendPolicySchema = z + .object({ + default: z.union([z.literal("allow"), z.literal("deny")]).optional(), + rules: z + .array( + z + .object({ + action: z.union([z.literal("allow"), z.literal("deny")]), + match: z + .object({ + channel: z.string().optional(), + chatType: z + .union([z.literal("direct"), z.literal("group"), z.literal("channel")]) + .optional(), + keyPrefix: z.string().optional(), + }) + .strict() + .optional(), + }) + .strict(), + ) + .optional(), + }) + .strict(); + export const SessionSchema = z .object({ scope: z.union([z.literal("per-sender"), z.literal("global")]).optional(), @@ -51,31 +76,7 @@ export const SessionSchema = z ]) .optional(), mainKey: z.string().optional(), - sendPolicy: z - .object({ - default: z.union([z.literal("allow"), z.literal("deny")]).optional(), - rules: z - .array( - z - .object({ - action: z.union([z.literal("allow"), z.literal("deny")]), - match: z - .object({ - channel: z.string().optional(), - chatType: z - .union([z.literal("direct"), z.literal("group"), z.literal("channel")]) - .optional(), - keyPrefix: z.string().optional(), - }) - .strict() - .optional(), - }) - .strict(), - ) - .optional(), - }) - .strict() - .optional(), + sendPolicy: SessionSendPolicySchema.optional(), agentToAgent: z .object({ maxPingPongTurns: z.number().int().min(0).max(5).optional(), diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index ce4115517..46f4fc16c 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -5,7 +5,12 @@ import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zo import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js"; import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js"; import { ChannelsSchema } from "./zod-schema.providers.js"; -import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js"; +import { + CommandsSchema, + MessagesSchema, + SessionSchema, + SessionSendPolicySchema, +} from "./zod-schema.session.js"; const BrowserSnapshotDefaultsSchema = z .object({ @@ -27,6 +32,61 @@ const NodeHostSchema = z .strict() .optional(); +const MemoryQmdPathSchema = z + .object({ + path: z.string(), + name: z.string().optional(), + pattern: z.string().optional(), + }) + .strict(); + +const MemoryQmdSessionSchema = z + .object({ + enabled: z.boolean().optional(), + exportDir: z.string().optional(), + retentionDays: z.number().int().nonnegative().optional(), + redactToolOutputs: z.boolean().optional(), + }) + .strict(); + +const MemoryQmdUpdateSchema = z + .object({ + interval: z.string().optional(), + debounceMs: z.number().int().nonnegative().optional(), + onBoot: z.boolean().optional(), + }) + .strict(); + +const MemoryQmdLimitsSchema = z + .object({ + maxResults: z.number().int().positive().optional(), + maxSnippetChars: z.number().int().positive().optional(), + maxInjectedChars: z.number().int().positive().optional(), + timeoutMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +const MemoryQmdSchema = z + .object({ + command: z.string().optional(), + includeDefaultMemory: z.boolean().optional(), + paths: z.array(MemoryQmdPathSchema).optional(), + sessions: MemoryQmdSessionSchema.optional(), + update: MemoryQmdUpdateSchema.optional(), + limits: MemoryQmdLimitsSchema.optional(), + scope: SessionSendPolicySchema.optional(), + }) + .strict(); + +const MemorySchema = z + .object({ + backend: z.union([z.literal("builtin"), z.literal("qmd")]).optional(), + citations: z.union([z.literal("auto"), z.literal("on"), z.literal("off")]).optional(), + qmd: MemoryQmdSchema.optional(), + }) + .strict() + .optional(); + export const MoltbotSchema = z .object({ meta: z @@ -445,6 +505,7 @@ export const MoltbotSchema = z }) .strict() .optional(), + memory: MemorySchema, skills: z .object({ allowBundled: z.array(z.string()).optional(), diff --git a/src/memory/backend-config.test.ts b/src/memory/backend-config.test.ts new file mode 100644 index 000000000..4e9db881c --- /dev/null +++ b/src/memory/backend-config.test.ts @@ -0,0 +1,58 @@ +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { MoltbotConfig } from "../config/config.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveMemoryBackendConfig } from "./backend-config.js"; + +describe("resolveMemoryBackendConfig", () => { + it("defaults to builtin backend when config missing", () => { + const cfg = { agents: { defaults: { workspace: "/tmp/memory-test" } } } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + expect(resolved.backend).toBe("builtin"); + expect(resolved.citations).toBe("auto"); + expect(resolved.qmd).toBeUndefined(); + }); + + it("resolves qmd backend with default collections", () => { + const cfg = { + agents: { defaults: { workspace: "/tmp/memory-test" } }, + memory: { + backend: "qmd", + qmd: {}, + }, + } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + expect(resolved.backend).toBe("qmd"); + expect(resolved.qmd?.collections.length).toBeGreaterThanOrEqual(3); + expect(resolved.qmd?.command).toBe("qmd"); + expect(resolved.qmd?.update.intervalMs).toBeGreaterThan(0); + }); + + it("resolves custom paths relative to workspace", () => { + const cfg = { + agents: { + defaults: { workspace: "/workspace/root" }, + list: [{ id: "main", workspace: "/workspace/root" }], + }, + memory: { + backend: "qmd", + qmd: { + paths: [ + { + path: "notes", + name: "custom-notes", + pattern: "**/*.md", + }, + ], + }, + }, + } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + const custom = resolved.qmd?.collections.find((c) => c.name.startsWith("custom-notes")); + expect(custom).toBeDefined(); + const workspaceRoot = resolveAgentWorkspaceDir(cfg, "main"); + expect(custom?.path).toBe(path.resolve(workspaceRoot, "notes")); + }); +}); diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts new file mode 100644 index 000000000..c7c4a0122 --- /dev/null +++ b/src/memory/backend-config.ts @@ -0,0 +1,245 @@ +import path from "node:path"; + +import { parseDurationMs } from "../cli/parse-duration.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import type { MoltbotConfig } from "../config/config.js"; +import type { + MemoryBackend, + MemoryCitationsMode, + MemoryQmdConfig, + MemoryQmdIndexPath, +} from "../config/types.memory.js"; +import type { SessionSendPolicyConfig } from "../config/types.base.js"; +import { resolveUserPath } from "../utils.js"; + +export type ResolvedMemoryBackendConfig = { + backend: MemoryBackend; + citations: MemoryCitationsMode; + qmd?: ResolvedQmdConfig; +}; + +export type ResolvedQmdCollection = { + name: string; + path: string; + pattern: string; + kind: "memory" | "custom" | "sessions"; +}; + +export type ResolvedQmdUpdateConfig = { + intervalMs: number; + debounceMs: number; + onBoot: boolean; +}; + +export type ResolvedQmdLimitsConfig = { + maxResults: number; + maxSnippetChars: number; + maxInjectedChars: number; + timeoutMs: number; +}; + +export type ResolvedQmdSessionConfig = { + enabled: boolean; + exportDir?: string; + retentionDays?: number; + redactToolOutputs: boolean; +}; + +export type ResolvedQmdConfig = { + command: string; + collections: ResolvedQmdCollection[]; + sessions: ResolvedQmdSessionConfig; + update: ResolvedQmdUpdateConfig; + limits: ResolvedQmdLimitsConfig; + includeDefaultMemory: boolean; + scope?: SessionSendPolicyConfig; +}; + +const DEFAULT_BACKEND: MemoryBackend = "builtin"; +const DEFAULT_CITATIONS: MemoryCitationsMode = "auto"; +const DEFAULT_QMD_INTERVAL = "5m"; +const DEFAULT_QMD_DEBOUNCE_MS = 15_000; +const DEFAULT_QMD_TIMEOUT_MS = 4_000; +const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = { + maxResults: 6, + maxSnippetChars: 700, + maxInjectedChars: 4_000, + timeoutMs: DEFAULT_QMD_TIMEOUT_MS, +}; +const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = { + default: "deny", + rules: [ + { + action: "allow", + match: { chatType: "direct" }, + }, + ], +}; + +function sanitizeName(input: string): string { + const lower = input.toLowerCase().replace(/[^a-z0-9-]+/g, "-"); + const trimmed = lower.replace(/^-+|-+$/g, ""); + return trimmed || "collection"; +} + +function ensureUniqueName(base: string, existing: Set): string { + let name = sanitizeName(base); + if (!existing.has(name)) { + existing.add(name); + return name; + } + let suffix = 2; + while (existing.has(`${name}-${suffix}`)) { + suffix += 1; + } + const unique = `${name}-${suffix}`; + existing.add(unique); + return unique; +} + +function resolvePath(raw: string, workspaceDir: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error("path required"); + if (trimmed.startsWith("~") || path.isAbsolute(trimmed)) { + return path.normalize(resolveUserPath(trimmed)); + } + return path.normalize(path.resolve(workspaceDir, trimmed)); +} + +function resolveIntervalMs(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" }); + try { + return parseDurationMs(value, { defaultUnit: "m" }); + } catch { + return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" }); + } +} + +function resolveDebounceMs(raw: number | undefined): number { + if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) { + return Math.floor(raw); + } + return DEFAULT_QMD_DEBOUNCE_MS; +} + +function resolveLimits(raw?: MemoryQmdConfig["limits"]): ResolvedQmdLimitsConfig { + const parsed: ResolvedQmdLimitsConfig = { ...DEFAULT_QMD_LIMITS }; + if (raw?.maxResults && raw.maxResults > 0) parsed.maxResults = Math.floor(raw.maxResults); + if (raw?.maxSnippetChars && raw.maxSnippetChars > 0) { + parsed.maxSnippetChars = Math.floor(raw.maxSnippetChars); + } + if (raw?.maxInjectedChars && raw.maxInjectedChars > 0) { + parsed.maxInjectedChars = Math.floor(raw.maxInjectedChars); + } + if (raw?.timeoutMs && raw.timeoutMs > 0) { + parsed.timeoutMs = Math.floor(raw.timeoutMs); + } + return parsed; +} + +function resolveSessionConfig( + cfg: MemoryQmdConfig["sessions"], + workspaceDir: string, +): ResolvedQmdSessionConfig { + const enabled = Boolean(cfg?.enabled); + const exportDirRaw = cfg?.exportDir?.trim(); + const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined; + const retentionDays = + cfg?.retentionDays && cfg.retentionDays > 0 ? Math.floor(cfg.retentionDays) : undefined; + const redactToolOutputs = cfg?.redactToolOutputs !== false; + return { + enabled, + exportDir, + retentionDays, + redactToolOutputs, + }; +} + +function resolveCustomPaths( + rawPaths: MemoryQmdIndexPath[] | undefined, + workspaceDir: string, + existing: Set, +): ResolvedQmdCollection[] { + if (!rawPaths?.length) return []; + const collections: ResolvedQmdCollection[] = []; + rawPaths.forEach((entry, index) => { + const trimmedPath = entry?.path?.trim(); + if (!trimmedPath) return; + let resolved: string; + try { + resolved = resolvePath(trimmedPath, workspaceDir); + } catch { + return; + } + const pattern = entry.pattern?.trim() || "**/*.md"; + const baseName = entry.name?.trim() || `custom-${index + 1}`; + const name = ensureUniqueName(baseName, existing); + collections.push({ + name, + path: resolved, + pattern, + kind: "custom", + }); + }); + return collections; +} + +function resolveDefaultCollections( + include: boolean, + workspaceDir: string, + existing: Set, +): ResolvedQmdCollection[] { + if (!include) return []; + const entries: Array<{ path: string; pattern: string; base: string }> = [ + { path: workspaceDir, pattern: "MEMORY.md", base: "memory-root" }, + { path: workspaceDir, pattern: "memory.md", base: "memory-alt" }, + { path: path.join(workspaceDir, "memory"), pattern: "**/*.md", base: "memory-dir" }, + ]; + return entries.map((entry) => ({ + name: ensureUniqueName(entry.base, existing), + path: entry.path, + pattern: entry.pattern, + kind: "memory", + })); +} + +export function resolveMemoryBackendConfig(params: { + cfg: MoltbotConfig; + agentId: string; +}): ResolvedMemoryBackendConfig { + const backend = params.cfg.memory?.backend ?? DEFAULT_BACKEND; + const citations = params.cfg.memory?.citations ?? DEFAULT_CITATIONS; + if (backend !== "qmd") { + return { backend: "builtin", citations }; + } + + const workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId); + const qmdCfg = params.cfg.memory?.qmd; + const includeDefaultMemory = qmdCfg?.includeDefaultMemory !== false; + const nameSet = new Set(); + const collections = [ + ...resolveDefaultCollections(includeDefaultMemory, workspaceDir, nameSet), + ...resolveCustomPaths(qmdCfg?.paths, workspaceDir, nameSet), + ]; + + const resolved: ResolvedQmdConfig = { + command: qmdCfg?.command?.trim() || "qmd", + collections, + includeDefaultMemory, + sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir), + update: { + intervalMs: resolveIntervalMs(qmdCfg?.update?.interval), + debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs), + onBoot: qmdCfg?.update?.onBoot !== false, + }, + limits: resolveLimits(qmdCfg?.limits), + scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE, + }; + + return { + backend: "qmd", + citations, + qmd: resolved, + }; +} diff --git a/src/memory/index.ts b/src/memory/index.ts index 092c4b03c..a4d9a4da1 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,2 +1,3 @@ -export type { MemoryIndexManager, MemorySearchResult } from "./manager.js"; +export { MemoryIndexManager } from "./manager.js"; +export type { MemorySearchResult, MemorySearchManager } from "./types.js"; export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 9a9991d10..5d42c571b 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -45,17 +45,13 @@ import { searchKeyword, searchVector } from "./manager-search.js"; import { ensureMemoryIndexSchema } from "./memory-schema.js"; import { requireNodeSqlite } from "./sqlite.js"; import { loadSqliteVecExtension } from "./sqlite-vec.js"; - -type MemorySource = "memory" | "sessions"; - -export type MemorySearchResult = { - path: string; - startLine: number; - endLine: number; - score: number; - snippet: string; - source: MemorySource; -}; +import type { + MemoryProviderStatus, + MemorySearchManager, + MemorySearchResult, + MemorySource, + MemorySyncProgressUpdate, +} from "./types.js"; type MemoryIndexMeta = { model: string; @@ -75,12 +71,6 @@ type SessionFileEntry = { content: string; }; -type MemorySyncProgressUpdate = { - completed: number; - total: number; - label?: string; -}; - type MemorySyncProgressState = { completed: number; total: number; @@ -115,7 +105,7 @@ const INDEX_CACHE = new Map(); const vectorToBlob = (embedding: number[]): Buffer => Buffer.from(new Float32Array(embedding).buffer); -export class MemoryIndexManager { +export class MemoryIndexManager implements MemorySearchManager { private readonly cacheKey: string; private readonly cfg: MoltbotConfig; private readonly agentId: string; @@ -415,39 +405,7 @@ export class MemoryIndexManager { return { text: slice.join("\n"), path: relPath }; } - status(): { - files: number; - chunks: number; - dirty: boolean; - workspaceDir: string; - dbPath: string; - provider: string; - model: string; - requestedProvider: string; - sources: MemorySource[]; - sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>; - cache?: { enabled: boolean; entries?: number; maxEntries?: number }; - fts?: { enabled: boolean; available: boolean; error?: string }; - fallback?: { from: string; reason?: string }; - vector?: { - enabled: boolean; - available?: boolean; - extensionPath?: string; - loadError?: string; - dims?: number; - }; - batch?: { - enabled: boolean; - failures: number; - limit: number; - wait: boolean; - concurrency: number; - pollIntervalMs: number; - timeoutMs: number; - lastError?: string; - lastProvider?: string; - }; - } { + status(): MemoryProviderStatus { const sourceFilter = this.buildSourceFilter(); const files = this.db .prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`) @@ -489,9 +447,10 @@ export class MemoryIndexManager { return sources.map((source) => ({ source, ...bySource.get(source)! })); })(); return { + backend: "builtin", files: files?.c ?? 0, chunks: chunks?.c ?? 0, - dirty: this.dirty, + dirty: this.dirty || this.sessionsDirty, workspaceDir: this.workspaceDir, dbPath: this.settings.store.path, provider: this.provider.id, diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts new file mode 100644 index 000000000..28e68bca4 --- /dev/null +++ b/src/memory/qmd-manager.ts @@ -0,0 +1,612 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import YAML from "yaml"; + +import type { MoltbotConfig } from "../config/config.js"; +import { resolveStateDir } from "../config/paths.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { + listSessionFilesForAgent, + buildSessionEntry, + type SessionFileEntry, +} from "./session-files.js"; +import { requireNodeSqlite } from "./sqlite.js"; +import type { + MemoryProviderStatus, + MemorySearchManager, + MemorySearchResult, + MemorySource, + MemorySyncProgressUpdate, +} from "./types.js"; +import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js"; + +const log = createSubsystemLogger("memory"); + +const SNIPPET_HEADER_RE = /@@\s*-([0-9]+),([0-9]+)/; + +type QmdQueryResult = { + docid?: string; + score?: number; + file?: string; + snippet?: string; + body?: string; +}; + +type CollectionRoot = { + path: string; + kind: MemorySource; +}; + +type SessionExporterConfig = { + dir: string; + retentionMs?: number; + collectionName: string; +}; + +export class QmdMemoryManager implements MemorySearchManager { + static async create(params: { + cfg: MoltbotConfig; + agentId: string; + resolved: ResolvedMemoryBackendConfig; + }): Promise { + const resolved = params.resolved.qmd; + if (!resolved) return null; + const manager = new QmdMemoryManager({ cfg: params.cfg, agentId: params.agentId, resolved }); + await manager.initialize(); + return manager; + } + + private readonly cfg: MoltbotConfig; + private readonly agentId: string; + private readonly qmd: ResolvedQmdConfig; + private readonly workspaceDir: string; + private readonly stateDir: string; + private readonly agentStateDir: string; + private readonly qmdDir: string; + private readonly cacheDir: string; + private readonly configDir: string; + private readonly xdgConfigHome: string; + private readonly xdgCacheHome: string; + private readonly collectionsFile: string; + private readonly indexPath: string; + private readonly env: NodeJS.ProcessEnv; + private readonly collectionRoots = new Map(); + private readonly sources = new Set(); + private readonly docPathCache = new Map< + string, + { rel: string; abs: string; source: MemorySource } + >(); + private readonly sessionExporter: SessionExporterConfig | null; + private updateTimer: NodeJS.Timeout | null = null; + private pendingUpdate: Promise | null = null; + private closed = false; + private db: import("node:sqlite").DatabaseSync | null = null; + private lastUpdateAt: number | null = null; + + private constructor(params: { + cfg: MoltbotConfig; + agentId: string; + resolved: ResolvedQmdConfig; + }) { + this.cfg = params.cfg; + this.agentId = params.agentId; + this.qmd = params.resolved; + this.workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId); + this.stateDir = resolveStateDir(process.env, os.homedir); + this.agentStateDir = path.join(this.stateDir, "agents", this.agentId); + this.qmdDir = path.join(this.agentStateDir, "qmd"); + this.cacheDir = path.join(this.qmdDir, "cache"); + this.configDir = path.join(this.qmdDir, "config"); + this.xdgConfigHome = path.join(this.qmdDir, "xdg-config"); + this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache"); + this.collectionsFile = path.join(this.configDir, "index.yml"); + this.indexPath = path.join(this.cacheDir, "index.sqlite"); + this.env = { + ...process.env, + QMD_CONFIG_DIR: this.configDir, + XDG_CONFIG_HOME: this.xdgConfigHome, + XDG_CACHE_HOME: this.xdgCacheHome, + INDEX_PATH: this.indexPath, + NO_COLOR: "1", + }; + this.sessionExporter = this.qmd.sessions.enabled + ? { + dir: this.qmd.sessions.exportDir ?? path.join(this.qmdDir, "sessions"), + retentionMs: this.qmd.sessions.retentionDays + ? this.qmd.sessions.retentionDays * 24 * 60 * 60 * 1000 + : undefined, + collectionName: this.pickSessionCollectionName(), + } + : null; + if (this.sessionExporter) { + this.qmd.collections = [ + ...this.qmd.collections, + { + name: this.sessionExporter.collectionName, + path: this.sessionExporter.dir, + pattern: "**/*.md", + kind: "sessions", + }, + ]; + } + } + + private async initialize(): Promise { + await fs.mkdir(this.cacheDir, { recursive: true }); + await fs.mkdir(this.configDir, { recursive: true }); + await fs.mkdir(this.xdgConfigHome, { recursive: true }); + await fs.mkdir(this.xdgCacheHome, { recursive: true }); + + this.bootstrapCollections(); + await this.writeCollectionsConfig(); + + if (this.qmd.update.onBoot) { + await this.runUpdate("boot", true); + } + if (this.qmd.update.intervalMs > 0) { + this.updateTimer = setInterval(() => { + void this.runUpdate("interval").catch((err) => { + log.warn(`qmd update failed (${String(err)})`); + }); + }, this.qmd.update.intervalMs); + } + } + + private bootstrapCollections(): void { + this.collectionRoots.clear(); + this.sources.clear(); + for (const collection of this.qmd.collections) { + const kind: MemorySource = collection.kind === "sessions" ? "sessions" : "memory"; + this.collectionRoots.set(collection.name, { path: collection.path, kind }); + this.sources.add(kind); + } + } + + private async writeCollectionsConfig(): Promise { + const collections: Record = {}; + for (const collection of this.qmd.collections) { + collections[collection.name] = { + path: collection.path, + pattern: collection.pattern, + }; + } + const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 }); + await fs.writeFile(this.collectionsFile, yaml, "utf-8"); + } + + async search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ): Promise { + if (!this.isScopeAllowed(opts?.sessionKey)) return []; + const trimmed = query.trim(); + if (!trimmed) return []; + await this.pendingUpdate?.catch(() => undefined); + const limit = Math.min( + this.qmd.limits.maxResults, + opts?.maxResults ?? this.qmd.limits.maxResults, + ); + const args = ["query", trimmed, "--json", "-n", String(limit)]; + let stdout: string; + try { + const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs }); + stdout = result.stdout; + } catch (err) { + log.warn(`qmd query failed: ${String(err)}`); + throw err instanceof Error ? err : new Error(String(err)); + } + let parsed: QmdQueryResult[] = []; + try { + parsed = JSON.parse(stdout); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`qmd query returned invalid JSON: ${message}`); + throw new Error(`qmd query returned invalid JSON: ${message}`); + } + const results: MemorySearchResult[] = []; + for (const entry of parsed) { + const doc = await this.resolveDocLocation(entry.docid); + if (!doc) continue; + const snippet = entry.snippet?.slice(0, this.qmd.limits.maxSnippetChars) ?? ""; + const lines = this.extractSnippetLines(snippet); + const score = typeof entry.score === "number" ? entry.score : 0; + const minScore = opts?.minScore ?? 0; + if (score < minScore) continue; + results.push({ + path: doc.rel, + startLine: lines.startLine, + endLine: lines.endLine, + score, + snippet, + source: doc.source, + }); + } + return results.slice(0, limit); + } + + async sync(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }): Promise { + if (params?.progress) { + params.progress({ completed: 0, total: 1, label: "Updating QMD index…" }); + } + await this.runUpdate(params?.reason ?? "manual", params?.force); + if (params?.progress) { + params.progress({ completed: 1, total: 1, label: "QMD index updated" }); + } + } + + async readFile(params: { + relPath: string; + from?: number; + lines?: number; + }): Promise<{ text: string; path: string }> { + const relPath = params.relPath?.trim(); + if (!relPath) throw new Error("path required"); + const absPath = this.resolveReadPath(relPath); + const content = await fs.readFile(absPath, "utf-8"); + if (!params.from && !params.lines) { + return { text: content, path: relPath }; + } + const lines = content.split("\n"); + const start = Math.max(1, params.from ?? 1); + const count = Math.max(1, params.lines ?? lines.length); + const slice = lines.slice(start - 1, start - 1 + count); + return { text: slice.join("\n"), path: relPath }; + } + + status(): MemoryProviderStatus { + const counts = this.readCounts(); + return { + backend: "qmd", + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + files: counts.totalDocuments, + chunks: counts.totalDocuments, + dirty: false, + workspaceDir: this.workspaceDir, + dbPath: this.indexPath, + sources: Array.from(this.sources), + sourceCounts: counts.sourceCounts, + vector: { enabled: true, available: true }, + batch: { + enabled: false, + failures: 0, + limit: 0, + wait: false, + concurrency: 0, + pollIntervalMs: 0, + timeoutMs: 0, + }, + custom: { + qmd: { + collections: this.qmd.collections.length, + lastUpdateAt: this.lastUpdateAt, + }, + }, + }; + } + + async probeVectorAvailability(): Promise { + return true; + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + if (this.updateTimer) { + clearInterval(this.updateTimer); + this.updateTimer = null; + } + await this.pendingUpdate?.catch(() => undefined); + if (this.db) { + this.db.close(); + this.db = null; + } + } + + private async runUpdate(reason: string, force?: boolean): Promise { + if (this.pendingUpdate && !force) return this.pendingUpdate; + const run = async () => { + if (this.sessionExporter) { + await this.exportSessions(); + } + await this.runQmd(["update"], { timeoutMs: 120_000 }); + try { + await this.runQmd(["embed"], { timeoutMs: 120_000 }); + } catch (err) { + log.warn(`qmd embed failed (${reason}): ${String(err)}`); + } + this.lastUpdateAt = Date.now(); + this.docPathCache.clear(); + }; + this.pendingUpdate = run().finally(() => { + this.pendingUpdate = null; + }); + await this.pendingUpdate; + } + + private async runQmd( + args: string[], + opts?: { timeoutMs?: number }, + ): Promise<{ stdout: string; stderr: string }> { + return await new Promise((resolve, reject) => { + const child = spawn(this.qmd.command, args, { + env: this.env, + cwd: this.workspaceDir, + }); + let stdout = ""; + let stderr = ""; + const timer = opts?.timeoutMs + ? setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`qmd ${args.join(" ")} timed out after ${opts.timeoutMs}ms`)); + }, opts.timeoutMs) + : null; + child.stdout.on("data", (data) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data) => { + stderr += data.toString(); + }); + child.on("error", (err) => { + if (timer) clearTimeout(timer); + reject(err); + }); + child.on("close", (code) => { + if (timer) clearTimeout(timer); + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject(new Error(`qmd ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`)); + } + }); + }); + } + + private ensureDb() { + if (this.db) return this.db; + const sqlite = requireNodeSqlite(); + this.db = sqlite.open(this.indexPath, { readonly: true }); + return this.db; + } + + private async exportSessions(): Promise { + if (!this.sessionExporter) return; + const exportDir = this.sessionExporter.dir; + await fs.mkdir(exportDir, { recursive: true }); + const files = await listSessionFilesForAgent(this.agentId); + const keep = new Set(); + const cutoff = this.sessionExporter.retentionMs + ? Date.now() - this.sessionExporter.retentionMs + : null; + for (const sessionFile of files) { + const entry = await buildSessionEntry(sessionFile); + if (!entry) continue; + if (cutoff && entry.mtimeMs < cutoff) continue; + const target = path.join(exportDir, `${path.basename(sessionFile, ".jsonl")}.md`); + await fs.writeFile(target, this.renderSessionMarkdown(entry), "utf-8"); + keep.add(target); + } + const exported = await fs.readdir(exportDir).catch(() => []); + for (const name of exported) { + if (!name.endsWith(".md")) continue; + const full = path.join(exportDir, name); + if (!keep.has(full)) { + await fs.rm(full, { force: true }); + } + } + } + + private renderSessionMarkdown(entry: SessionFileEntry): string { + const header = `# Session ${path.basename(entry.absPath, path.extname(entry.absPath))}`; + const body = entry.content?.trim().length ? entry.content.trim() : "(empty)"; + return `${header}\n\n${body}\n`; + } + + private pickSessionCollectionName(): string { + const existing = new Set(this.qmd.collections.map((collection) => collection.name)); + if (!existing.has("sessions")) return "sessions"; + let counter = 2; + let candidate = `sessions-${counter}`; + while (existing.has(candidate)) { + counter += 1; + candidate = `sessions-${counter}`; + } + return candidate; + } + + private async resolveDocLocation( + docid?: string, + ): Promise<{ rel: string; abs: string; source: MemorySource } | null> { + if (!docid) return null; + const normalized = docid.startsWith("#") ? docid.slice(1) : docid; + if (!normalized) return null; + const cached = this.docPathCache.get(normalized); + if (cached) return cached; + const db = this.ensureDb(); + const row = db + .prepare("SELECT collection, path FROM documents WHERE hash LIKE ? AND active = 1 LIMIT 1") + .get(`${normalized}%`) as { collection: string; path: string } | undefined; + if (!row) return null; + const location = this.toDocLocation(row.collection, row.path); + if (!location) return null; + this.docPathCache.set(normalized, location); + return location; + } + + private extractSnippetLines(snippet: string): { startLine: number; endLine: number } { + const match = SNIPPET_HEADER_RE.exec(snippet); + if (match) { + const start = Number(match[1]); + const count = Number(match[2]); + if (Number.isFinite(start) && Number.isFinite(count)) { + return { startLine: start, endLine: start + count - 1 }; + } + } + const lines = snippet.split("\n").length; + return { startLine: 1, endLine: lines }; + } + + private readCounts(): { + totalDocuments: number; + sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>; + } { + try { + const db = this.ensureDb(); + const rows = db + .prepare( + "SELECT collection, COUNT(*) as c FROM documents WHERE active = 1 GROUP BY collection", + ) + .all() as Array<{ collection: string; c: number }>; + const bySource = new Map(); + for (const source of this.sources) { + bySource.set(source, { files: 0, chunks: 0 }); + } + let total = 0; + for (const row of rows) { + const root = this.collectionRoots.get(row.collection); + const source = root?.kind ?? "memory"; + const entry = bySource.get(source) ?? { files: 0, chunks: 0 }; + entry.files += row.c ?? 0; + entry.chunks += row.c ?? 0; + bySource.set(source, entry); + total += row.c ?? 0; + } + return { + totalDocuments: total, + sourceCounts: Array.from(bySource.entries()).map(([source, value]) => ({ + source, + files: value.files, + chunks: value.chunks, + })), + }; + } catch (err) { + log.warn(`failed to read qmd index stats: ${String(err)}`); + return { + totalDocuments: 0, + sourceCounts: Array.from(this.sources).map((source) => ({ source, files: 0, chunks: 0 })), + }; + } + } + + private isScopeAllowed(sessionKey?: string): boolean { + const scope = this.qmd.scope; + if (!scope) return true; + const channel = this.deriveChannelFromKey(sessionKey); + const chatType = this.deriveChatTypeFromKey(sessionKey); + const normalizedKey = sessionKey ?? ""; + for (const rule of scope.rules ?? []) { + if (!rule) continue; + const match = rule.match ?? {}; + if (match.channel && match.channel !== channel) continue; + if (match.chatType && match.chatType !== chatType) continue; + if (match.keyPrefix && !normalizedKey.startsWith(match.keyPrefix)) continue; + return rule.action === "allow"; + } + const fallback = scope.default ?? "allow"; + return fallback === "allow"; + } + + private deriveChannelFromKey(key?: string) { + if (!key) return undefined; + const parts = key.split(":").filter(Boolean); + if (parts.length >= 3 && (parts[1] === "group" || parts[1] === "channel")) { + return parts[0]?.toLowerCase(); + } + return undefined; + } + + private deriveChatTypeFromKey(key?: string) { + if (!key) return undefined; + if (key.includes(":group:")) return "group"; + if (key.includes(":channel:")) return "channel"; + return "direct"; + } + + private toDocLocation( + collection: string, + collectionRelativePath: string, + ): { rel: string; abs: string; source: MemorySource } | null { + const root = this.collectionRoots.get(collection); + if (!root) return null; + const normalizedRelative = collectionRelativePath.replace(/\\/g, "/"); + const absPath = path.normalize(path.resolve(root.path, collectionRelativePath)); + const relativeToWorkspace = path.relative(this.workspaceDir, absPath); + const relPath = this.buildSearchPath( + collection, + normalizedRelative, + relativeToWorkspace, + absPath, + ); + return { rel: relPath, abs: absPath, source: root.kind }; + } + + private buildSearchPath( + collection: string, + collectionRelativePath: string, + relativeToWorkspace: string, + absPath: string, + ): string { + const insideWorkspace = this.isInsideWorkspace(relativeToWorkspace); + if (insideWorkspace) { + const normalized = relativeToWorkspace.replace(/\\/g, "/"); + if (!normalized) return path.basename(absPath); + return normalized; + } + const sanitized = collectionRelativePath.replace(/^\/+/, ""); + return `qmd/${collection}/${sanitized}`; + } + + private isInsideWorkspace(relativePath: string): boolean { + if (!relativePath) return true; + if (relativePath.startsWith("..")) return false; + if (relativePath.startsWith(`..${path.sep}`)) return false; + return !path.isAbsolute(relativePath); + } + + private resolveReadPath(relPath: string): string { + if (relPath.startsWith("qmd/")) { + const [, collection, ...rest] = relPath.split("/"); + if (!collection || rest.length === 0) { + throw new Error("invalid qmd path"); + } + const root = this.collectionRoots.get(collection); + if (!root) throw new Error(`unknown qmd collection: ${collection}`); + const joined = rest.join("/"); + const resolved = path.resolve(root.path, joined); + if (!this.isWithinRoot(root.path, resolved)) { + throw new Error("qmd path escapes collection"); + } + return resolved; + } + const absPath = path.resolve(this.workspaceDir, relPath); + if (!this.isWithinWorkspace(absPath)) { + throw new Error("path escapes workspace"); + } + return absPath; + } + + private isWithinWorkspace(absPath: string): boolean { + const normalizedWorkspace = this.workspaceDir.endsWith(path.sep) + ? this.workspaceDir + : `${this.workspaceDir}${path.sep}`; + if (absPath === this.workspaceDir) return true; + const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`; + return candidate.startsWith(normalizedWorkspace); + } + + private isWithinRoot(root: string, candidate: string): boolean { + const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`; + if (candidate === root) return true; + const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`; + return next.startsWith(normalizedRoot); + } +} diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts new file mode 100644 index 000000000..7dd822fa5 --- /dev/null +++ b/src/memory/search-manager.test.ts @@ -0,0 +1,62 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mockPrimary = { + search: vi.fn(async () => []), + readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), + status: vi.fn(() => ({ + backend: "qmd" as const, + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + files: 0, + chunks: 0, + dirty: false, + workspaceDir: "/tmp", + dbPath: "/tmp/index.sqlite", + sources: ["memory" as const], + sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }], + })), + sync: vi.fn(async () => {}), + probeVectorAvailability: vi.fn(async () => true), + close: vi.fn(async () => {}), +}; + +vi.mock("./qmd-manager.js", () => ({ + QmdMemoryManager: { + create: vi.fn(async () => mockPrimary), + }, +})); + +vi.mock("./manager.js", () => ({ + MemoryIndexManager: { + get: vi.fn(async () => null), + }, +})); + +import { QmdMemoryManager } from "./qmd-manager.js"; +import { getMemorySearchManager } from "./search-manager.js"; + +beforeEach(() => { + mockPrimary.search.mockClear(); + mockPrimary.readFile.mockClear(); + mockPrimary.status.mockClear(); + mockPrimary.sync.mockClear(); + mockPrimary.probeVectorAvailability.mockClear(); + mockPrimary.close.mockClear(); + QmdMemoryManager.create.mockClear(); +}); + +describe("getMemorySearchManager caching", () => { + it("reuses the same QMD manager instance for repeated calls", async () => { + const cfg = { + memory: { backend: "qmd", qmd: {} }, + agents: { list: [{ id: "main", default: true, workspace: "/tmp/workspace" }] }, + } as const; + + const first = await getMemorySearchManager({ cfg, agentId: "main" }); + const second = await getMemorySearchManager({ cfg, agentId: "main" }); + + expect(first.manager).toBe(second.manager); + expect(QmdMemoryManager.create).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index 9bcd529f3..f3a0f07af 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -1,8 +1,15 @@ +import { createSubsystemLogger } from "../logging/subsystem.js"; import type { MoltbotConfig } from "../config/config.js"; +import { resolveMemoryBackendConfig } from "./backend-config.js"; +import type { ResolvedQmdConfig } from "./backend-config.js"; import type { MemoryIndexManager } from "./manager.js"; +import type { MemorySearchManager, MemorySyncProgressUpdate } from "./types.js"; + +const log = createSubsystemLogger("memory"); +const QMD_MANAGER_CACHE = new Map(); export type MemorySearchManagerResult = { - manager: MemoryIndexManager | null; + manager: MemorySearchManager | null; error?: string; }; @@ -10,6 +17,40 @@ export async function getMemorySearchManager(params: { cfg: MoltbotConfig; agentId: string; }): Promise { + const resolved = resolveMemoryBackendConfig(params); + if (resolved.backend === "qmd" && resolved.qmd) { + const cacheKey = buildQmdCacheKey(params.agentId, resolved.qmd); + const cached = QMD_MANAGER_CACHE.get(cacheKey); + if (cached) { + return { manager: cached }; + } + try { + const { QmdMemoryManager } = await import("./qmd-manager.js"); + const primary = await QmdMemoryManager.create({ + cfg: params.cfg, + agentId: params.agentId, + resolved, + }); + if (primary) { + const wrapper = new FallbackMemoryManager( + { + primary, + fallbackFactory: async () => { + const { MemoryIndexManager } = await import("./manager.js"); + return await MemoryIndexManager.get(params); + }, + }, + () => QMD_MANAGER_CACHE.delete(cacheKey), + ); + QMD_MANAGER_CACHE.set(cacheKey, wrapper); + return { manager: wrapper }; + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`qmd memory unavailable; falling back to builtin: ${message}`); + } + } + try { const { MemoryIndexManager } = await import("./manager.js"); const manager = await MemoryIndexManager.get(params); @@ -19,3 +60,117 @@ export async function getMemorySearchManager(params: { return { manager: null, error: message }; } } + +class FallbackMemoryManager implements MemorySearchManager { + private fallback: MemorySearchManager | null = null; + private primaryFailed = false; + private lastError?: string; + + constructor( + private readonly deps: { + primary: MemorySearchManager; + fallbackFactory: () => Promise; + }, + private readonly onClose?: () => void, + ) {} + + async search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ) { + if (!this.primaryFailed) { + try { + return await this.deps.primary.search(query, opts); + } catch (err) { + this.primaryFailed = true; + this.lastError = err instanceof Error ? err.message : String(err); + log.warn(`qmd memory failed; switching to builtin index: ${this.lastError}`); + await this.deps.primary.close?.().catch(() => {}); + } + } + const fallback = await this.ensureFallback(); + if (fallback) { + return await fallback.search(query, opts); + } + throw new Error(this.lastError ?? "memory search unavailable"); + } + + async readFile(params: { relPath: string; from?: number; lines?: number }) { + if (!this.primaryFailed) { + return await this.deps.primary.readFile(params); + } + const fallback = await this.ensureFallback(); + if (fallback) { + return await fallback.readFile(params); + } + throw new Error(this.lastError ?? "memory read unavailable"); + } + + status() { + if (!this.primaryFailed) { + return this.deps.primary.status(); + } + const fallbackStatus = this.fallback?.status(); + if (fallbackStatus) { + const custom = fallbackStatus.custom ?? {}; + return { + ...fallbackStatus, + custom: { + ...custom, + fallback: { disabled: true, reason: this.lastError ?? "unknown" }, + }, + }; + } + const primaryStatus = this.deps.primary.status(); + const custom = primaryStatus.custom ?? {}; + return { + ...primaryStatus, + custom: { + ...custom, + fallback: { disabled: true, reason: this.lastError ?? "unknown" }, + }, + }; + } + + async sync(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }) { + if (!this.primaryFailed) { + await this.deps.primary.sync?.(params); + return; + } + const fallback = await this.ensureFallback(); + await fallback?.sync?.(params); + } + + async probeVectorAvailability() { + if (!this.primaryFailed) { + return await this.deps.primary.probeVectorAvailability(); + } + const fallback = await this.ensureFallback(); + return (await fallback?.probeVectorAvailability()) ?? false; + } + + async close() { + await this.deps.primary.close?.(); + await this.fallback?.close?.(); + this.onClose?.(); + } + + private async ensureFallback(): Promise { + if (this.fallback) return this.fallback; + const fallback = await this.deps.fallbackFactory(); + if (!fallback) { + log.warn("memory fallback requested but builtin index is unavailable"); + return null; + } + this.fallback = fallback; + return this.fallback; + } +} + +function buildQmdCacheKey(agentId: string, config: ResolvedQmdConfig): string { + return `${agentId}:${JSON.stringify(config)}`; +} diff --git a/src/memory/types.ts b/src/memory/types.ts new file mode 100644 index 000000000..c26940ede --- /dev/null +++ b/src/memory/types.ts @@ -0,0 +1,72 @@ +export type MemorySource = "memory" | "sessions"; + +export type MemorySearchResult = { + path: string; + startLine: number; + endLine: number; + score: number; + snippet: string; + source: MemorySource; + citation?: string; +}; + +export type MemorySyncProgressUpdate = { + completed: number; + total: number; + label?: string; +}; + +export type MemoryProviderStatus = { + backend: "builtin" | "qmd"; + provider: string; + model?: string; + requestedProvider?: string; + files?: number; + chunks?: number; + dirty?: boolean; + workspaceDir?: string; + dbPath?: string; + sources?: MemorySource[]; + cache?: { enabled: boolean; entries?: number; maxEntries?: number }; + fts?: { enabled: boolean; available: boolean; error?: string }; + fallback?: { from: string; reason?: string }; + vector?: { + enabled: boolean; + available?: boolean; + extensionPath?: string; + loadError?: string; + dims?: number; + }; + batch?: { + enabled: boolean; + failures: number; + limit: number; + wait: boolean; + concurrency: number; + pollIntervalMs: number; + timeoutMs: number; + lastError?: string; + lastProvider?: string; + }; + custom?: Record; +}; + +export interface MemorySearchManager { + search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ): Promise; + readFile(params: { + relPath: string; + from?: number; + lines?: number; + }): Promise<{ text: string; path: string }>; + status(): MemoryProviderStatus; + sync?(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }): Promise; + probeVectorAvailability(): Promise; + close?(): Promise; +} From 4e8000e745eee80e8e13ed97aa081781856d2c6b Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Tue, 27 Jan 2026 22:17:56 -0800 Subject: [PATCH 2/9] Make memory more resilient to failure --- docs/concepts/memory.md | 2 +- src/config/schema.ts | 3 --- src/config/types.memory.ts | 1 - src/config/zod-schema.ts | 1 - src/memory/backend-config.ts | 3 --- src/memory/qmd-manager.ts | 22 +++++++++++++++++++++- src/memory/search-manager.ts | 3 +++ src/memory/types.ts | 1 + 8 files changed, 26 insertions(+), 10 deletions(-) diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 0a558a54d..7781033fa 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -126,7 +126,7 @@ out to QMD for retrieval. Key points: - `paths[]`: add extra directories/files (`path`, optional `pattern`, optional stable `name`). - `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`, - `exportDir`, `redactToolOutputs`—defaults to redacting tool payloads). + `exportDir`). - `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`). - `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`, `maxInjectedChars`, `timeoutMs`). diff --git a/src/config/schema.ts b/src/config/schema.ts index df38a6028..e41fa905e 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -264,7 +264,6 @@ const FIELD_LABELS: Record = { "memory.qmd.sessions.enabled": "QMD Session Indexing", "memory.qmd.sessions.exportDir": "QMD Session Export Directory", "memory.qmd.sessions.retentionDays": "QMD Session Retention (days)", - "memory.qmd.sessions.redactToolOutputs": "QMD Session Tool Redaction", "memory.qmd.update.interval": "QMD Update Interval", "memory.qmd.update.debounceMs": "QMD Update Debounce (ms)", "memory.qmd.update.onBoot": "QMD Update on Startup", @@ -576,8 +575,6 @@ const FIELD_HELP: Record = { "Override directory for sanitized session exports before indexing.", "memory.qmd.sessions.retentionDays": "Retention window for exported sessions before pruning (default: unlimited).", - "memory.qmd.sessions.redactToolOutputs": - "Strip tool call payloads/results when exporting sessions (default: true).", "memory.qmd.update.interval": "How often the QMD sidecar refreshes indexes (duration string, default: 5m).", "memory.qmd.update.debounceMs": diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts index a6bb5eeb3..dad2f9784 100644 --- a/src/config/types.memory.ts +++ b/src/config/types.memory.ts @@ -29,7 +29,6 @@ export type MemoryQmdSessionConfig = { enabled?: boolean; exportDir?: string; retentionDays?: number; - redactToolOutputs?: boolean; }; export type MemoryQmdUpdateConfig = { diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 46f4fc16c..1fb4c923c 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -45,7 +45,6 @@ const MemoryQmdSessionSchema = z enabled: z.boolean().optional(), exportDir: z.string().optional(), retentionDays: z.number().int().nonnegative().optional(), - redactToolOutputs: z.boolean().optional(), }) .strict(); diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts index c7c4a0122..15851ef29 100644 --- a/src/memory/backend-config.ts +++ b/src/memory/backend-config.ts @@ -42,7 +42,6 @@ export type ResolvedQmdSessionConfig = { enabled: boolean; exportDir?: string; retentionDays?: number; - redactToolOutputs: boolean; }; export type ResolvedQmdConfig = { @@ -147,12 +146,10 @@ function resolveSessionConfig( const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined; const retentionDays = cfg?.retentionDays && cfg.retentionDays > 0 ? Math.floor(cfg.retentionDays) : undefined; - const redactToolOutputs = cfg?.redactToolOutputs !== false; return { enabled, exportDir, retentionDays, - redactToolOutputs, }; } diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 28e68bca4..a54390112 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -225,7 +225,7 @@ export class QmdMemoryManager implements MemorySearchManager { source: doc.source, }); } - return results.slice(0, limit); + return this.clampResultsByInjectedChars(results.slice(0, limit)); } async sync(params?: { @@ -609,4 +609,24 @@ export class QmdMemoryManager implements MemorySearchManager { const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`; return next.startsWith(normalizedRoot); } + + private clampResultsByInjectedChars(results: MemorySearchResult[]): MemorySearchResult[] { + const budget = this.qmd.limits.maxInjectedChars; + if (!budget || budget <= 0) return results; + let remaining = budget; + const clamped: MemorySearchResult[] = []; + for (const entry of results) { + if (remaining <= 0) break; + const snippet = entry.snippet ?? ""; + if (snippet.length <= remaining) { + clamped.push(entry); + remaining -= snippet.length; + } else { + const trimmed = snippet.slice(0, Math.max(0, remaining)); + clamped.push({ ...entry, snippet: trimmed }); + break; + } + } + return clamped; + } } diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index f3a0f07af..a642f4658 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -111,10 +111,12 @@ class FallbackMemoryManager implements MemorySearchManager { return this.deps.primary.status(); } const fallbackStatus = this.fallback?.status(); + const fallbackInfo = { from: "qmd", reason: this.lastError ?? "unknown" }; if (fallbackStatus) { const custom = fallbackStatus.custom ?? {}; return { ...fallbackStatus, + fallback: fallbackInfo, custom: { ...custom, fallback: { disabled: true, reason: this.lastError ?? "unknown" }, @@ -125,6 +127,7 @@ class FallbackMemoryManager implements MemorySearchManager { const custom = primaryStatus.custom ?? {}; return { ...primaryStatus, + fallback: fallbackInfo, custom: { ...custom, fallback: { disabled: true, reason: this.lastError ?? "unknown" }, diff --git a/src/memory/types.ts b/src/memory/types.ts index c26940ede..40abf99ea 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -27,6 +27,7 @@ export type MemoryProviderStatus = { workspaceDir?: string; dbPath?: string; sources?: MemorySource[]; + sourceCounts?: Array<{ source: MemorySource; files: number; chunks: number }>; cache?: { enabled: boolean; entries?: number; maxEntries?: number }; fts?: { enabled: boolean; available: boolean; error?: string }; fallback?: { from: string; reason?: string }; From f3f1640c7be6bcc2ae6d65ac25317f4ed35ff4f0 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Tue, 27 Jan 2026 22:46:11 -0800 Subject: [PATCH 3/9] Add more tests; make fall back more resilient and visible --- CHANGELOG.md | 3 - src/cli/memory-cli.test.ts | 6 +- src/cli/memory-cli.ts | 6 +- src/memory/index.ts | 6 +- src/memory/manager.ts | 3 +- src/memory/qmd-manager.test.ts | 95 +++++++++++++++++++++++++++++++ src/memory/qmd-manager.ts | 16 ++++++ src/memory/search-manager.test.ts | 2 + src/memory/search-manager.ts | 17 +++++- src/memory/types.ts | 6 ++ 10 files changed, 148 insertions(+), 12 deletions(-) create mode 100644 src/memory/qmd-manager.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5551f1e84..5909c9899 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,9 +35,6 @@ Status: beta. - Docs: add GCP Compute Engine deployment guide. (#1848) Thanks @hougangdev. - Docs: add LINE channel guide. Thanks @thewilloftheshadow. - Docs: credit both contributors for Control UI refresh. (#1852) Thanks @EnzeD. -- Memory: add optional QMD-backed memory backend with fallbacks to the existing Markdown index. Thanks @vgnsh. -- Memory: surface QMD startup errors so we fall back to the legacy index when the CLI is missing, and allow extra QMD collections outside the workspace to return snippets via the `qmd//…` path prefix. -- Memory: export session transcripts into the QMD index when `memory.qmd.sessions.enabled` is set and honor `memory.citations` when formatting snippets. - Onboarding: add Venice API key to non-interactive flow. (#1893) Thanks @jonisjongithub. - Onboarding: strengthen security warning copy for beta + access control expectations. - Tlon: format thread reply IDs as @ud. (#1837) Thanks @wca4a. diff --git a/src/cli/memory-cli.test.ts b/src/cli/memory-cli.test.ts index 263b23fc9..b55c983d4 100644 --- a/src/cli/memory-cli.test.ts +++ b/src/cli/memory-cli.test.ts @@ -242,7 +242,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "status", "--index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), ); expect(probeEmbeddingAvailability).toHaveBeenCalled(); expect(close).toHaveBeenCalled(); @@ -267,7 +267,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(log).toHaveBeenCalledWith("Memory index updated (main)."); @@ -294,7 +294,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(error).toHaveBeenCalledWith( diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index 68894adf5..5301306ec 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -240,6 +240,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { try { await manager.sync({ reason: "cli", + force: true, progress: (syncUpdate) => { update({ completed: syncUpdate.completed, @@ -443,9 +444,8 @@ export function registerMemoryCli(program: Command) { .command("index") .description("Reindex memory files") .option("--agent ", "Agent id (default: default agent)") - .option("--force", "Force full reindex", false) .option("--verbose", "Verbose logging", false) - .action(async (opts: MemoryCommandOptions & { force?: boolean }) => { + .action(async (opts: MemoryCommandOptions) => { setVerbose(Boolean(opts.verbose)); const cfg = loadConfig(); const agentIds = resolveAgentIds(cfg, opts.agent); @@ -527,7 +527,7 @@ export function registerMemoryCli(program: Command) { try { await manager.sync({ reason: "cli", - force: opts.force, + force: true, progress: (syncUpdate) => { if (syncUpdate.label) lastLabel = syncUpdate.label; lastCompleted = syncUpdate.completed; diff --git a/src/memory/index.ts b/src/memory/index.ts index a4d9a4da1..4d2df05a3 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,3 +1,7 @@ export { MemoryIndexManager } from "./manager.js"; -export type { MemorySearchResult, MemorySearchManager } from "./types.js"; +export type { + MemoryEmbeddingProbeResult, + MemorySearchManager, + MemorySearchResult, +} from "./types.js"; export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 5d42c571b..724b1320c 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -46,6 +46,7 @@ import { ensureMemoryIndexSchema } from "./memory-schema.js"; import { requireNodeSqlite } from "./sqlite.js"; import { loadSqliteVecExtension } from "./sqlite-vec.js"; import type { + MemoryEmbeddingProbeResult, MemoryProviderStatus, MemorySearchManager, MemorySearchResult, @@ -504,7 +505,7 @@ export class MemoryIndexManager implements MemorySearchManager { return this.ensureVectorReady(); } - async probeEmbeddingAvailability(): Promise<{ ok: boolean; error?: string }> { + async probeEmbeddingAvailability(): Promise { try { await this.embedBatchWithRetry(["ping"]); return { ok: true }; diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts new file mode 100644 index 000000000..2fd2e2c1b --- /dev/null +++ b/src/memory/qmd-manager.test.ts @@ -0,0 +1,95 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { EventEmitter } from "node:events"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("node:child_process", () => { + const spawn = vi.fn((cmd: string, _args: string[]) => { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const child = new EventEmitter() as { + stdout: EventEmitter; + stderr: EventEmitter; + kill: () => void; + emit: (event: string, code: number) => boolean; + }; + child.stdout = stdout; + child.stderr = stderr; + child.kill = () => { + child.emit("close", 0); + }; + setImmediate(() => { + stdout.emit("data", ""); + stderr.emit("data", ""); + child.emit("close", 0); + }); + return child; + }); + return { spawn }; +}); + +import { spawn as mockedSpawn } from "node:child_process"; +import type { MoltbotConfig } from "../config/config.js"; +import { resolveMemoryBackendConfig } from "./backend-config.js"; +import { QmdMemoryManager } from "./qmd-manager.js"; + +const spawnMock = mockedSpawn as unknown as vi.Mock; + +describe("QmdMemoryManager", () => { + let tmpRoot: string; + let workspaceDir: string; + let stateDir: string; + let cfg: MoltbotConfig; + const agentId = "main"; + + beforeEach(async () => { + spawnMock.mockClear(); + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "qmd-manager-test-")); + workspaceDir = path.join(tmpRoot, "workspace"); + await fs.mkdir(workspaceDir, { recursive: true }); + stateDir = path.join(tmpRoot, "state"); + await fs.mkdir(stateDir, { recursive: true }); + process.env.MOLTBOT_STATE_DIR = stateDir; + cfg = { + agents: { + list: [{ id: agentId, default: true, workspace: workspaceDir }], + }, + memory: { + backend: "qmd", + qmd: { + includeDefaultMemory: false, + update: { interval: "0s", debounceMs: 60_000, onBoot: false }, + paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }], + }, + }, + } as MoltbotConfig; + }); + + afterEach(async () => { + delete process.env.MOLTBOT_STATE_DIR; + await fs.rm(tmpRoot, { recursive: true, force: true }); + }); + + it("debounces back-to-back sync calls", async () => { + const resolved = resolveMemoryBackendConfig({ cfg, agentId }); + const manager = await QmdMemoryManager.create({ cfg, agentId, resolved }); + expect(manager).toBeTruthy(); + if (!manager) throw new Error("manager missing"); + + await manager.sync({ reason: "manual" }); + expect(spawnMock.mock.calls.length).toBe(2); + + await manager.sync({ reason: "manual-again" }); + expect(spawnMock.mock.calls.length).toBe(2); + + (manager as unknown as { lastUpdateAt: number | null }).lastUpdateAt = + Date.now() - (resolved.qmd?.update.debounceMs ?? 0) - 10; + + await manager.sync({ reason: "after-wait" }); + expect(spawnMock.mock.calls.length).toBe(4); + + await manager.close(); + }); +}); diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index a54390112..77159da3a 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -16,6 +16,7 @@ import { } from "./session-files.js"; import { requireNodeSqlite } from "./sqlite.js"; import type { + MemoryEmbeddingProbeResult, MemoryProviderStatus, MemorySearchManager, MemorySearchResult, @@ -294,6 +295,10 @@ export class QmdMemoryManager implements MemorySearchManager { }; } + async probeEmbeddingAvailability(): Promise { + return { ok: true }; + } + async probeVectorAvailability(): Promise { return true; } @@ -314,6 +319,9 @@ export class QmdMemoryManager implements MemorySearchManager { private async runUpdate(reason: string, force?: boolean): Promise { if (this.pendingUpdate && !force) return this.pendingUpdate; + if (this.shouldSkipUpdate(force)) { + return; + } const run = async () => { if (this.sessionExporter) { await this.exportSessions(); @@ -629,4 +637,12 @@ export class QmdMemoryManager implements MemorySearchManager { } return clamped; } + + private shouldSkipUpdate(force?: boolean): boolean { + if (force) return false; + const debounceMs = this.qmd.update.debounceMs; + if (debounceMs <= 0) return false; + if (!this.lastUpdateAt) return false; + return Date.now() - this.lastUpdateAt < debounceMs; + } } diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts index 7dd822fa5..f7119caa1 100644 --- a/src/memory/search-manager.test.ts +++ b/src/memory/search-manager.test.ts @@ -17,6 +17,7 @@ const mockPrimary = { sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }], })), sync: vi.fn(async () => {}), + probeEmbeddingAvailability: vi.fn(async () => ({ ok: true })), probeVectorAvailability: vi.fn(async () => true), close: vi.fn(async () => {}), }; @@ -41,6 +42,7 @@ beforeEach(() => { mockPrimary.readFile.mockClear(); mockPrimary.status.mockClear(); mockPrimary.sync.mockClear(); + mockPrimary.probeEmbeddingAvailability.mockClear(); mockPrimary.probeVectorAvailability.mockClear(); mockPrimary.close.mockClear(); QmdMemoryManager.create.mockClear(); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index a642f4658..5d98f6451 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -3,7 +3,11 @@ import type { MoltbotConfig } from "../config/config.js"; import { resolveMemoryBackendConfig } from "./backend-config.js"; import type { ResolvedQmdConfig } from "./backend-config.js"; import type { MemoryIndexManager } from "./manager.js"; -import type { MemorySearchManager, MemorySyncProgressUpdate } from "./types.js"; +import type { + MemoryEmbeddingProbeResult, + MemorySearchManager, + MemorySyncProgressUpdate, +} from "./types.js"; const log = createSubsystemLogger("memory"); const QMD_MANAGER_CACHE = new Map(); @@ -148,6 +152,17 @@ class FallbackMemoryManager implements MemorySearchManager { await fallback?.sync?.(params); } + async probeEmbeddingAvailability(): Promise { + if (!this.primaryFailed) { + return await this.deps.primary.probeEmbeddingAvailability(); + } + const fallback = await this.ensureFallback(); + if (fallback) { + return await fallback.probeEmbeddingAvailability(); + } + return { ok: false, error: this.lastError ?? "memory embeddings unavailable" }; + } + async probeVectorAvailability() { if (!this.primaryFailed) { return await this.deps.primary.probeVectorAvailability(); diff --git a/src/memory/types.ts b/src/memory/types.ts index 40abf99ea..2013fcc3e 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -10,6 +10,11 @@ export type MemorySearchResult = { citation?: string; }; +export type MemoryEmbeddingProbeResult = { + ok: boolean; + error?: string; +}; + export type MemorySyncProgressUpdate = { completed: number; total: number; @@ -68,6 +73,7 @@ export interface MemorySearchManager { force?: boolean; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise; + probeEmbeddingAvailability(): Promise; probeVectorAvailability(): Promise; close?(): Promise; } From 62e80d030cbf05473441c47acc1bcaceed060782 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Tue, 27 Jan 2026 23:08:43 -0800 Subject: [PATCH 4/9] Fix build errors --- docs/concepts/memory.md | 7 ++++ src/cli/memory-cli.ts | 70 +++++++++++++++++++++++++-------------- src/discord/targets.ts | 4 +++ src/memory/qmd-manager.ts | 10 +++--- 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 7781033fa..034e8754d 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -109,6 +109,13 @@ out to QMD for retrieval. Key points: a release) and make sure the `qmd` binary is on the gateway’s `PATH`. - QMD needs an SQLite build that allows extensions (`brew install sqlite` on macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically. +- QMD shells out to its CLI, which depends on an Ollama daemon listening on + `http://localhost:11434` to load the bundled models (`embeddinggemma`, + `ExpedientFalcon/qwen3-reranker:0.6b-q8_0`, `qwen3:0.6b`). Install/run Ollama + separately before enabling the backend. +- OS support: macOS and Linux work out of the box once Bun + SQLite + Ollama are + installed. Windows requires WSL2 (or building the experimental Ollama port) + until Ollama ships native binaries. **How the sidecar runs** - The gateway writes a self-contained QMD home under diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index 5301306ec..ddb8e79e7 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -213,13 +213,16 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { onMissing: (error) => defaultRuntime.log(error ?? "Memory search disabled."), onCloseError: (err) => defaultRuntime.error(`Memory manager close failed: ${formatErrorMessage(err)}`), - close: (manager) => manager.close(), + close: async (manager) => { + await manager.close?.(); + }, run: async (manager) => { const deep = Boolean(opts.deep || opts.index); let embeddingProbe: | Awaited> | undefined; let indexError: string | undefined; + const syncFn = manager.sync; if (deep) { await withProgress({ label: "Checking memory…", total: 2 }, async (progress) => { progress.setLabel("Probing vector…"); @@ -229,7 +232,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { embeddingProbe = await manager.probeEmbeddingAvailability(); progress.tick(); }); - if (opts.index) { + if (opts.index && syncFn) { await withProgressTotals( { label: "Indexing memory…", @@ -238,7 +241,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { }, async (update, progress) => { try { - await manager.sync({ + await syncFn({ reason: "cli", force: true, progress: (syncUpdate) => { @@ -257,6 +260,8 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { } }, ); + } else if (opts.index && !syncFn) { + defaultRuntime.log("Memory backend does not support manual reindex."); } } else { await manager.probeVectorAvailability(); @@ -265,11 +270,10 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { const sources = ( status.sources?.length ? status.sources : ["memory"] ) as MemorySourceName[]; - const scan = await scanMemorySources({ - workspaceDir: status.workspaceDir, - agentId, - sources, - }); + const workspaceDir = status.workspaceDir; + const scan = workspaceDir + ? await scanMemorySources({ workspaceDir, agentId, sources }) + : undefined; allResults.push({ agentId, status, embeddingProbe, indexError, scan }); }, }); @@ -291,26 +295,31 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { for (const result of allResults) { const { agentId, status, embeddingProbe, indexError, scan } = result; + const filesIndexed = status.files ?? 0; + const chunksIndexed = status.chunks ?? 0; const totalFiles = scan?.totalFiles ?? null; const indexedLabel = totalFiles === null - ? `${status.files}/? files · ${status.chunks} chunks` - : `${status.files}/${totalFiles} files · ${status.chunks} chunks`; + ? `${filesIndexed}/? files · ${chunksIndexed} chunks` + : `${filesIndexed}/${totalFiles} files · ${chunksIndexed} chunks`; if (opts.index) { const line = indexError ? `Memory index failed: ${indexError}` : "Memory index complete."; defaultRuntime.log(line); } + const requestedProvider = status.requestedProvider ?? status.provider; + const modelLabel = status.model ?? status.provider; + const storePath = status.dbPath ? shortenHomePath(status.dbPath) : ""; + const workspacePath = status.workspaceDir ? shortenHomePath(status.workspaceDir) : ""; + const sourceList = status.sources?.length ? status.sources.join(", ") : null; const lines = [ `${heading("Memory Search")} ${muted(`(${agentId})`)}`, - `${label("Provider")} ${info(status.provider)} ${muted( - `(requested: ${status.requestedProvider})`, - )}`, - `${label("Model")} ${info(status.model)}`, - status.sources?.length ? `${label("Sources")} ${info(status.sources.join(", "))}` : null, + `${label("Provider")} ${info(status.provider)} ${muted(`(requested: ${requestedProvider})`)}`, + `${label("Model")} ${info(modelLabel)}`, + sourceList ? `${label("Sources")} ${info(sourceList)}` : null, `${label("Indexed")} ${success(indexedLabel)}`, `${label("Dirty")} ${status.dirty ? warn("yes") : muted("no")}`, - `${label("Store")} ${info(shortenHomePath(status.dbPath))}`, - `${label("Workspace")} ${info(shortenHomePath(status.workspaceDir))}`, + `${label("Store")} ${info(storePath)}`, + `${label("Workspace")} ${info(workspacePath)}`, ].filter(Boolean) as string[]; if (embeddingProbe) { const state = embeddingProbe.ok ? "ready" : "unavailable"; @@ -323,7 +332,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { if (status.sourceCounts?.length) { lines.push(label("By source")); for (const entry of status.sourceCounts) { - const total = scan?.sources.find( + const total = scan?.sources?.find( (scanEntry) => scanEntry.source === entry.source, )?.totalFiles; const counts = @@ -455,9 +464,12 @@ export function registerMemoryCli(program: Command) { onMissing: (error) => defaultRuntime.log(error ?? "Memory search disabled."), onCloseError: (err) => defaultRuntime.error(`Memory manager close failed: ${formatErrorMessage(err)}`), - close: (manager) => manager.close(), + close: async (manager) => { + await manager.close?.(); + }, run: async (manager) => { try { + const syncFn = manager.sync; if (opts.verbose) { const status = manager.status(); const rich = isRich(); @@ -466,15 +478,17 @@ export function registerMemoryCli(program: Command) { const info = (text: string) => colorize(rich, theme.info, text); const warn = (text: string) => colorize(rich, theme.warn, text); const label = (text: string) => muted(`${text}:`); - const sourceLabels = status.sources.map((source) => - formatSourceLabel(source, status.workspaceDir, agentId), + const sourceLabels = (status.sources ?? []).map((source) => + formatSourceLabel(source, status.workspaceDir ?? "", agentId), ); + const requestedProvider = status.requestedProvider ?? status.provider; + const modelLabel = status.model ?? status.provider; const lines = [ `${heading("Memory Index")} ${muted(`(${agentId})`)}`, `${label("Provider")} ${info(status.provider)} ${muted( - `(requested: ${status.requestedProvider})`, + `(requested: ${requestedProvider})`, )}`, - `${label("Model")} ${info(status.model)}`, + `${label("Model")} ${info(modelLabel)}`, sourceLabels.length ? `${label("Sources")} ${info(sourceLabels.join(", "))}` : null, @@ -514,6 +528,10 @@ export function registerMemoryCli(program: Command) { ? `${lastLabel} · elapsed ${elapsed} · eta ${eta}` : `${lastLabel} · elapsed ${elapsed}`; }; + if (!syncFn) { + defaultRuntime.log("Memory backend does not support manual reindex."); + return; + } await withProgressTotals( { label: "Indexing memory…", @@ -525,7 +543,7 @@ export function registerMemoryCli(program: Command) { progress.setLabel(buildLabel()); }, 1000); try { - await manager.sync({ + await syncFn({ reason: "cli", force: true, progress: (syncUpdate) => { @@ -579,7 +597,9 @@ export function registerMemoryCli(program: Command) { onMissing: (error) => defaultRuntime.log(error ?? "Memory search disabled."), onCloseError: (err) => defaultRuntime.error(`Memory manager close failed: ${formatErrorMessage(err)}`), - close: (manager) => manager.close(), + close: async (manager) => { + await manager.close?.(); + }, run: async (manager) => { let results: Awaited>; try { diff --git a/src/discord/targets.ts b/src/discord/targets.ts index 5ea6f5b1b..b43b7adbc 100644 --- a/src/discord/targets.ts +++ b/src/discord/targets.ts @@ -84,10 +84,14 @@ export async function resolveDiscordTarget( const likelyUsername = isLikelyUsername(trimmed); const shouldLookup = isExplicitUserLookup(trimmed, parseOptions) || likelyUsername; + + // Parse directly if it's already a known format. Use a safe parse so ambiguous + // numeric targets don't throw when we still want to attempt username lookup. const directParse = safeParseDiscordTarget(trimmed, parseOptions); if (directParse && directParse.kind !== "channel" && !likelyUsername) { return directParse; } + if (!shouldLookup) { return directParse ?? parseDiscordTarget(trimmed, parseOptions); } diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 77159da3a..c11496b6e 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -23,6 +23,8 @@ import type { MemorySource, MemorySyncProgressUpdate, } from "./types.js"; + +type SqliteDatabase = import("node:sqlite").DatabaseSync; import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js"; const log = createSubsystemLogger("memory"); @@ -85,7 +87,7 @@ export class QmdMemoryManager implements MemorySearchManager { private updateTimer: NodeJS.Timeout | null = null; private pendingUpdate: Promise | null = null; private closed = false; - private db: import("node:sqlite").DatabaseSync | null = null; + private db: SqliteDatabase | null = null; private lastUpdateAt: number | null = null; private constructor(params: { @@ -379,10 +381,10 @@ export class QmdMemoryManager implements MemorySearchManager { }); } - private ensureDb() { + private ensureDb(): SqliteDatabase { if (this.db) return this.db; - const sqlite = requireNodeSqlite(); - this.db = sqlite.open(this.indexPath, { readonly: true }); + const { DatabaseSync } = requireNodeSqlite(); + this.db = new DatabaseSync(this.indexPath, { readOnly: true }); return this.db; } From eff75c395b0fb85a2c3581ce25231e440a0ea15d Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Tue, 27 Jan 2026 23:42:52 -0800 Subject: [PATCH 5/9] fix(qmd): use XDG dirs for qmd home; drop ollama docs --- docs/concepts/memory.md | 16 ++++++++-------- src/memory/qmd-manager.ts | 17 +++++++---------- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 034e8754d..0db9995cb 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -108,14 +108,14 @@ out to QMD for retrieval. Key points: - Install the QMD CLI separately (`bun install -g github.com/tobi/qmd` or grab a release) and make sure the `qmd` binary is on the gateway’s `PATH`. - QMD needs an SQLite build that allows extensions (`brew install sqlite` on - macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically. -- QMD shells out to its CLI, which depends on an Ollama daemon listening on - `http://localhost:11434` to load the bundled models (`embeddinggemma`, - `ExpedientFalcon/qwen3-reranker:0.6b-q8_0`, `qwen3:0.6b`). Install/run Ollama - separately before enabling the backend. -- OS support: macOS and Linux work out of the box once Bun + SQLite + Ollama are - installed. Windows requires WSL2 (or building the experimental Ollama port) - until Ollama ships native binaries. + macOS). +- QMD runs fully locally via Bun + `node-llama-cpp` and auto-downloads GGUF + models from HuggingFace on first use (no separate Ollama daemon required). +- The gateway runs QMD in a self-contained XDG home under + `~/.clawdbot/agents//qmd/` by setting `XDG_CONFIG_HOME` and + `XDG_CACHE_HOME`. +- OS support: macOS and Linux work out of the box once Bun + SQLite are + installed. Windows is best supported via WSL2. **How the sidecar runs** - The gateway writes a self-contained QMD home under diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index c11496b6e..87fe22729 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -70,8 +70,6 @@ export class QmdMemoryManager implements MemorySearchManager { private readonly stateDir: string; private readonly agentStateDir: string; private readonly qmdDir: string; - private readonly cacheDir: string; - private readonly configDir: string; private readonly xdgConfigHome: string; private readonly xdgCacheHome: string; private readonly collectionsFile: string; @@ -102,18 +100,17 @@ export class QmdMemoryManager implements MemorySearchManager { this.stateDir = resolveStateDir(process.env, os.homedir); this.agentStateDir = path.join(this.stateDir, "agents", this.agentId); this.qmdDir = path.join(this.agentStateDir, "qmd"); - this.cacheDir = path.join(this.qmdDir, "cache"); - this.configDir = path.join(this.qmdDir, "config"); + // QMD respects XDG base dirs: + // - config: $XDG_CONFIG_HOME/qmd/index.yml + // - cache: $XDG_CACHE_HOME/qmd/index.sqlite this.xdgConfigHome = path.join(this.qmdDir, "xdg-config"); this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache"); - this.collectionsFile = path.join(this.configDir, "index.yml"); - this.indexPath = path.join(this.cacheDir, "index.sqlite"); + this.collectionsFile = path.join(this.xdgConfigHome, "qmd", "index.yml"); + this.indexPath = path.join(this.xdgCacheHome, "qmd", "index.sqlite"); this.env = { ...process.env, - QMD_CONFIG_DIR: this.configDir, XDG_CONFIG_HOME: this.xdgConfigHome, XDG_CACHE_HOME: this.xdgCacheHome, - INDEX_PATH: this.indexPath, NO_COLOR: "1", }; this.sessionExporter = this.qmd.sessions.enabled @@ -139,10 +136,10 @@ export class QmdMemoryManager implements MemorySearchManager { } private async initialize(): Promise { - await fs.mkdir(this.cacheDir, { recursive: true }); - await fs.mkdir(this.configDir, { recursive: true }); await fs.mkdir(this.xdgConfigHome, { recursive: true }); await fs.mkdir(this.xdgCacheHome, { recursive: true }); + await fs.mkdir(path.dirname(this.collectionsFile), { recursive: true }); + await fs.mkdir(path.dirname(this.indexPath), { recursive: true }); this.bootstrapCollections(); await this.writeCollectionsConfig(); From 06d675d4ae511bf85676c1646b3f1031c0004eb6 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Wed, 28 Jan 2026 00:12:18 -0800 Subject: [PATCH 6/9] fix(memory-qmd): write XDG index.yml + legacy compat --- src/cli/memory-cli.ts | 4 +-- src/memory/qmd-manager.test.ts | 2 +- src/memory/qmd-manager.ts | 49 +++++++++++++++++++++++++++++++ src/memory/search-manager.test.ts | 1 + src/memory/search-manager.ts | 1 - 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index ddb8e79e7..7642ea61d 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -222,7 +222,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { | Awaited> | undefined; let indexError: string | undefined; - const syncFn = manager.sync; + const syncFn = manager.sync ? manager.sync.bind(manager) : undefined; if (deep) { await withProgress({ label: "Checking memory…", total: 2 }, async (progress) => { progress.setLabel("Probing vector…"); @@ -469,7 +469,7 @@ export function registerMemoryCli(program: Command) { }, run: async (manager) => { try { - const syncFn = manager.sync; + const syncFn = manager.sync ? manager.sync.bind(manager) : undefined; if (opts.verbose) { const status = manager.status(); const rich = isRich(); diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index 2fd2e2c1b..a465260b4 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -6,7 +6,7 @@ import { EventEmitter } from "node:events"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("node:child_process", () => { - const spawn = vi.fn((cmd: string, _args: string[]) => { + const spawn = vi.fn((_cmd: string, _args: string[]) => { const stdout = new EventEmitter(); const stderr = new EventEmitter(); const child = new EventEmitter() as { diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 87fe22729..433ddbcba 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -74,6 +74,8 @@ export class QmdMemoryManager implements MemorySearchManager { private readonly xdgCacheHome: string; private readonly collectionsFile: string; private readonly indexPath: string; + private readonly legacyCollectionsFile: string; + private readonly legacyIndexPath: string; private readonly env: NodeJS.ProcessEnv; private readonly collectionRoots = new Map(); private readonly sources = new Set(); @@ -107,6 +109,12 @@ export class QmdMemoryManager implements MemorySearchManager { this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache"); this.collectionsFile = path.join(this.xdgConfigHome, "qmd", "index.yml"); this.indexPath = path.join(this.xdgCacheHome, "qmd", "index.sqlite"); + + // Legacy locations (older builds wrote here). Keep them in sync via symlinks + // so upgrades don't strand an empty index. + this.legacyCollectionsFile = path.join(this.qmdDir, "config", "index.yml"); + this.legacyIndexPath = path.join(this.qmdDir, "cache", "index.sqlite"); + this.env = { ...process.env, XDG_CONFIG_HOME: this.xdgConfigHome, @@ -141,8 +149,13 @@ export class QmdMemoryManager implements MemorySearchManager { await fs.mkdir(path.dirname(this.collectionsFile), { recursive: true }); await fs.mkdir(path.dirname(this.indexPath), { recursive: true }); + // Legacy dirs + await fs.mkdir(path.dirname(this.legacyCollectionsFile), { recursive: true }); + await fs.mkdir(path.dirname(this.legacyIndexPath), { recursive: true }); + this.bootstrapCollections(); await this.writeCollectionsConfig(); + await this.ensureLegacyCompatSymlinks(); if (this.qmd.update.onBoot) { await this.runUpdate("boot", true); @@ -176,6 +189,42 @@ export class QmdMemoryManager implements MemorySearchManager { } const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 }); await fs.writeFile(this.collectionsFile, yaml, "utf-8"); + + // Also write legacy path so older qmd homes remain usable. + await fs.writeFile(this.legacyCollectionsFile, yaml, "utf-8"); + } + + private async ensureLegacyCompatSymlinks(): Promise { + // Best-effort: keep legacy locations pointing at the XDG locations. + // This helps when users have old state dirs on disk. + try { + await fs.rm(this.legacyCollectionsFile, { force: true }); + } catch {} + try { + await fs.symlink(this.collectionsFile, this.legacyCollectionsFile); + } catch {} + + try { + // If a legacy index already exists (from an older version), prefer it by + // linking the XDG path to the legacy DB. + const legacyExists = await fs + .stat(this.legacyIndexPath) + .then(() => true) + .catch(() => false); + const xdgExists = await fs + .stat(this.indexPath) + .then(() => true) + .catch(() => false); + + if (legacyExists && !xdgExists) { + await fs.symlink(this.legacyIndexPath, this.indexPath); + } else if (!legacyExists && xdgExists) { + // nothing to do + } else if (!legacyExists && !xdgExists) { + // Create an empty file so the path exists for read-only opens later. + await fs.writeFile(this.indexPath, ""); + } + } catch {} } async search( diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts index f7119caa1..8419d7f2c 100644 --- a/src/memory/search-manager.test.ts +++ b/src/memory/search-manager.test.ts @@ -59,6 +59,7 @@ describe("getMemorySearchManager caching", () => { const second = await getMemorySearchManager({ cfg, agentId: "main" }); expect(first.manager).toBe(second.manager); + // eslint-disable-next-line @typescript-eslint/unbound-method expect(QmdMemoryManager.create).toHaveBeenCalledTimes(1); }); }); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index 5d98f6451..afd392673 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -2,7 +2,6 @@ import { createSubsystemLogger } from "../logging/subsystem.js"; import type { MoltbotConfig } from "../config/config.js"; import { resolveMemoryBackendConfig } from "./backend-config.js"; import type { ResolvedQmdConfig } from "./backend-config.js"; -import type { MemoryIndexManager } from "./manager.js"; import type { MemoryEmbeddingProbeResult, MemorySearchManager, From 3211b6cb106c9b0130b2ae4185f8932baea8063b Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Wed, 28 Jan 2026 00:33:17 -0800 Subject: [PATCH 7/9] fix(memory-qmd): create collections via qmd CLI (no YAML) --- src/memory/qmd-manager.test.ts | 8 ++-- src/memory/qmd-manager.ts | 88 ++++++++++------------------------ 2 files changed, 29 insertions(+), 67 deletions(-) diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index a465260b4..4fcbb0c9f 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -78,17 +78,19 @@ describe("QmdMemoryManager", () => { expect(manager).toBeTruthy(); if (!manager) throw new Error("manager missing"); + const baselineCalls = spawnMock.mock.calls.length; + await manager.sync({ reason: "manual" }); - expect(spawnMock.mock.calls.length).toBe(2); + expect(spawnMock.mock.calls.length).toBe(baselineCalls + 2); await manager.sync({ reason: "manual-again" }); - expect(spawnMock.mock.calls.length).toBe(2); + expect(spawnMock.mock.calls.length).toBe(baselineCalls + 2); (manager as unknown as { lastUpdateAt: number | null }).lastUpdateAt = Date.now() - (resolved.qmd?.update.debounceMs ?? 0) - 10; await manager.sync({ reason: "after-wait" }); - expect(spawnMock.mock.calls.length).toBe(4); + expect(spawnMock.mock.calls.length).toBe(baselineCalls + 4); await manager.close(); }); diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 433ddbcba..003a6ca36 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -3,8 +3,6 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import YAML from "yaml"; - import type { MoltbotConfig } from "../config/config.js"; import { resolveStateDir } from "../config/paths.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; @@ -72,10 +70,7 @@ export class QmdMemoryManager implements MemorySearchManager { private readonly qmdDir: string; private readonly xdgConfigHome: string; private readonly xdgCacheHome: string; - private readonly collectionsFile: string; private readonly indexPath: string; - private readonly legacyCollectionsFile: string; - private readonly legacyIndexPath: string; private readonly env: NodeJS.ProcessEnv; private readonly collectionRoots = new Map(); private readonly sources = new Set(); @@ -102,19 +97,14 @@ export class QmdMemoryManager implements MemorySearchManager { this.stateDir = resolveStateDir(process.env, os.homedir); this.agentStateDir = path.join(this.stateDir, "agents", this.agentId); this.qmdDir = path.join(this.agentStateDir, "qmd"); - // QMD respects XDG base dirs: - // - config: $XDG_CONFIG_HOME/qmd/index.yml + // QMD uses XDG base dirs for its internal state. + // Collections are managed via `qmd collection add` and stored inside the index DB. + // - config: $XDG_CONFIG_HOME (contexts, etc.) // - cache: $XDG_CACHE_HOME/qmd/index.sqlite this.xdgConfigHome = path.join(this.qmdDir, "xdg-config"); this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache"); - this.collectionsFile = path.join(this.xdgConfigHome, "qmd", "index.yml"); this.indexPath = path.join(this.xdgCacheHome, "qmd", "index.sqlite"); - // Legacy locations (older builds wrote here). Keep them in sync via symlinks - // so upgrades don't strand an empty index. - this.legacyCollectionsFile = path.join(this.qmdDir, "config", "index.yml"); - this.legacyIndexPath = path.join(this.qmdDir, "cache", "index.sqlite"); - this.env = { ...process.env, XDG_CONFIG_HOME: this.xdgConfigHome, @@ -146,16 +136,10 @@ export class QmdMemoryManager implements MemorySearchManager { private async initialize(): Promise { await fs.mkdir(this.xdgConfigHome, { recursive: true }); await fs.mkdir(this.xdgCacheHome, { recursive: true }); - await fs.mkdir(path.dirname(this.collectionsFile), { recursive: true }); await fs.mkdir(path.dirname(this.indexPath), { recursive: true }); - // Legacy dirs - await fs.mkdir(path.dirname(this.legacyCollectionsFile), { recursive: true }); - await fs.mkdir(path.dirname(this.legacyIndexPath), { recursive: true }); - this.bootstrapCollections(); - await this.writeCollectionsConfig(); - await this.ensureLegacyCompatSymlinks(); + await this.ensureCollections(); if (this.qmd.update.onBoot) { await this.runUpdate("boot", true); @@ -179,52 +163,28 @@ export class QmdMemoryManager implements MemorySearchManager { } } - private async writeCollectionsConfig(): Promise { - const collections: Record = {}; + private async ensureCollections(): Promise { + // QMD collections are persisted inside the index database and must be created + // via the CLI. The YAML file format is not supported by the QMD builds we + // target, so we ensure collections exist by running `qmd collection add`. for (const collection of this.qmd.collections) { - collections[collection.name] = { - path: collection.path, - pattern: collection.pattern, - }; - } - const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 }); - await fs.writeFile(this.collectionsFile, yaml, "utf-8"); - - // Also write legacy path so older qmd homes remain usable. - await fs.writeFile(this.legacyCollectionsFile, yaml, "utf-8"); - } - - private async ensureLegacyCompatSymlinks(): Promise { - // Best-effort: keep legacy locations pointing at the XDG locations. - // This helps when users have old state dirs on disk. - try { - await fs.rm(this.legacyCollectionsFile, { force: true }); - } catch {} - try { - await fs.symlink(this.collectionsFile, this.legacyCollectionsFile); - } catch {} - - try { - // If a legacy index already exists (from an older version), prefer it by - // linking the XDG path to the legacy DB. - const legacyExists = await fs - .stat(this.legacyIndexPath) - .then(() => true) - .catch(() => false); - const xdgExists = await fs - .stat(this.indexPath) - .then(() => true) - .catch(() => false); - - if (legacyExists && !xdgExists) { - await fs.symlink(this.legacyIndexPath, this.indexPath); - } else if (!legacyExists && xdgExists) { - // nothing to do - } else if (!legacyExists && !xdgExists) { - // Create an empty file so the path exists for read-only opens later. - await fs.writeFile(this.indexPath, ""); + try { + await this.runQmd([ + "collection", + "add", + collection.path, + "--name", + collection.name, + "--mask", + collection.pattern, + ]); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + // Idempotency: qmd exits non-zero if the collection name already exists. + if (message.includes("already exists")) continue; + log.warn(`qmd collection add failed for ${collection.name}: ${message}`); } - } catch {} + } } async search( From ebf6553c9ef9f9e5dfaa7995a841a37da6e85246 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Wed, 28 Jan 2026 00:57:46 -0800 Subject: [PATCH 8/9] Add how to trigger model downloads for qmd in documentation --- docs/concepts/memory.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 0db9995cb..c2e8139fd 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -126,6 +126,34 @@ out to QMD for retrieval. Key points: - Searches run via `qmd query --json`. If QMD fails or the binary is missing, Moltbot automatically falls back to the builtin SQLite manager so memory tools keep working. +- **First search may be slow**: QMD may download local GGUF models (reranker/query + expansion) on the first `qmd query` run. + - Moltbot sets `XDG_CONFIG_HOME`/`XDG_CACHE_HOME` automatically when it runs QMD. + - If you want to pre-download models manually (and warm the same index Moltbot + uses), run a one-off query with the agent’s XDG dirs. + + Moltbot’s QMD state lives under your **state dir** (usually `~/.clawdbot`, or + `~/.moltbot` on newer installs). You can point `qmd` at the exact same index + by exporting the same XDG vars Moltbot uses: + + ```bash + # Pick the same state dir Moltbot uses + STATE_DIR="${MOLTBOT_STATE_DIR:-${CLAWDBOT_STATE_DIR:-$HOME/.clawdbot}}" + if [ -d "$HOME/.moltbot" ] && [ ! -d "$HOME/.clawdbot" ] \ + && [ -z "${MOLTBOT_STATE_DIR:-}" ] && [ -z "${CLAWDBOT_STATE_DIR:-}" ]; then + STATE_DIR="$HOME/.moltbot" + fi + + export XDG_CONFIG_HOME="$STATE_DIR/agents/main/qmd/xdg-config" + export XDG_CACHE_HOME="$STATE_DIR/agents/main/qmd/xdg-cache" + + # (Optional) force an index refresh + embeddings + qmd update + qmd embed + + # Warm up / trigger first-time model downloads + qmd query "test" -c memory-root --json >/dev/null 2>&1 + ``` **Config surface (`memory.qmd.*`)** - `command` (default `qmd`): override the executable path. From d16d7218986d2ebfdf6958174c7b9ab238fd9380 Mon Sep 17 00:00:00 2001 From: vignesh07 Date: Wed, 28 Jan 2026 02:05:58 -0800 Subject: [PATCH 9/9] fix(memory/qmd): throttle embed + citations auto + restore --force --- docs/concepts/memory.md | 2 +- src/agents/tools/memory-tool.ts | 23 +++++++++++++++++- src/cli/memory-cli.test.ts | 6 ++--- src/cli/memory-cli.ts | 8 ++++--- src/config/schema.ts | 3 +++ src/config/types.memory.ts | 1 + src/config/zod-schema.ts | 1 + src/memory/backend-config.ts | 15 +++++++++++- src/memory/qmd-manager.test.ts | 3 ++- src/memory/qmd-manager.ts | 42 +++++++++++++++++++++++++++------ src/memory/session-files.ts | 4 +++- 11 files changed, 90 insertions(+), 18 deletions(-) diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index c2e8139fd..3c5850965 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -162,7 +162,7 @@ out to QMD for retrieval. Key points: stable `name`). - `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`, `exportDir`). -- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`). +- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`, `embedInterval`). - `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`, `maxInjectedChars`, `timeoutMs`). - `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy). diff --git a/src/agents/tools/memory-tool.ts b/src/agents/tools/memory-tool.ts index f6643bb67..1cf5b3356 100644 --- a/src/agents/tools/memory-tool.ts +++ b/src/agents/tools/memory-tool.ts @@ -51,7 +51,10 @@ export function createMemorySearchTool(options: { } try { const citationsMode = resolveMemoryCitationsMode(cfg); - const includeCitations = citationsMode !== "off"; + const includeCitations = shouldIncludeCitations({ + mode: citationsMode, + sessionKey: options.agentSessionKey, + }); const rawResults = await manager.search(query, { maxResults, minScore, @@ -141,3 +144,21 @@ function formatCitation(entry: MemorySearchResult): string { : `#L${entry.startLine}-L${entry.endLine}`; return `${entry.path}${lineRange}`; } + +function shouldIncludeCitations(params: { + mode: MemoryCitationsMode; + sessionKey?: string; +}): boolean { + if (params.mode === "on") return true; + if (params.mode === "off") return false; + // auto: show citations in direct chats; suppress in groups/channels by default. + const chatType = deriveChatTypeFromSessionKey(params.sessionKey); + return chatType === "direct"; +} + +function deriveChatTypeFromSessionKey(sessionKey?: string): "direct" | "group" | "channel" { + if (!sessionKey) return "direct"; + if (sessionKey.includes(":group:")) return "group"; + if (sessionKey.includes(":channel:")) return "channel"; + return "direct"; +} diff --git a/src/cli/memory-cli.test.ts b/src/cli/memory-cli.test.ts index b55c983d4..e13fa9a5c 100644 --- a/src/cli/memory-cli.test.ts +++ b/src/cli/memory-cli.test.ts @@ -242,7 +242,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "status", "--index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(probeEmbeddingAvailability).toHaveBeenCalled(); expect(close).toHaveBeenCalled(); @@ -267,7 +267,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(log).toHaveBeenCalledWith("Memory index updated (main)."); @@ -294,7 +294,7 @@ describe("memory cli", () => { await program.parseAsync(["memory", "index"], { from: "user" }); expect(sync).toHaveBeenCalledWith( - expect.objectContaining({ reason: "cli", force: true, progress: expect.any(Function) }), + expect.objectContaining({ reason: "cli", force: false, progress: expect.any(Function) }), ); expect(close).toHaveBeenCalled(); expect(error).toHaveBeenCalledWith( diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index 7642ea61d..683e4c209 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -24,6 +24,7 @@ type MemoryCommandOptions = { json?: boolean; deep?: boolean; index?: boolean; + force?: boolean; verbose?: boolean; }; @@ -243,7 +244,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { try { await syncFn({ reason: "cli", - force: true, + force: Boolean(opts.force), progress: (syncUpdate) => { update({ completed: syncUpdate.completed, @@ -445,7 +446,7 @@ export function registerMemoryCli(program: Command) { .option("--deep", "Probe embedding provider availability") .option("--index", "Reindex if dirty (implies --deep)") .option("--verbose", "Verbose logging", false) - .action(async (opts: MemoryCommandOptions) => { + .action(async (opts: MemoryCommandOptions & { force?: boolean }) => { await runMemoryStatus(opts); }); @@ -453,6 +454,7 @@ export function registerMemoryCli(program: Command) { .command("index") .description("Reindex memory files") .option("--agent ", "Agent id (default: default agent)") + .option("--force", "Force full reindex", false) .option("--verbose", "Verbose logging", false) .action(async (opts: MemoryCommandOptions) => { setVerbose(Boolean(opts.verbose)); @@ -545,7 +547,7 @@ export function registerMemoryCli(program: Command) { try { await syncFn({ reason: "cli", - force: true, + force: Boolean(opts.force), progress: (syncUpdate) => { if (syncUpdate.label) lastLabel = syncUpdate.label; lastCompleted = syncUpdate.completed; diff --git a/src/config/schema.ts b/src/config/schema.ts index e41fa905e..65bf4aa55 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -267,6 +267,7 @@ const FIELD_LABELS: Record = { "memory.qmd.update.interval": "QMD Update Interval", "memory.qmd.update.debounceMs": "QMD Update Debounce (ms)", "memory.qmd.update.onBoot": "QMD Update on Startup", + "memory.qmd.update.embedInterval": "QMD Embed Interval", "memory.qmd.limits.maxResults": "QMD Max Results", "memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars", "memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars", @@ -580,6 +581,8 @@ const FIELD_HELP: Record = { "memory.qmd.update.debounceMs": "Minimum delay between successive QMD refresh runs (default: 15000).", "memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).", + "memory.qmd.update.embedInterval": + "How often QMD embeddings are refreshed (duration string, default: 60m). Set to 0 to disable periodic embed.", "memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).", "memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).", "memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.", diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts index dad2f9784..2fc185200 100644 --- a/src/config/types.memory.ts +++ b/src/config/types.memory.ts @@ -35,6 +35,7 @@ export type MemoryQmdUpdateConfig = { interval?: string; debounceMs?: number; onBoot?: boolean; + embedInterval?: string; }; export type MemoryQmdLimitsConfig = { diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 1fb4c923c..af6668979 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -53,6 +53,7 @@ const MemoryQmdUpdateSchema = z interval: z.string().optional(), debounceMs: z.number().int().nonnegative().optional(), onBoot: z.boolean().optional(), + embedInterval: z.string().optional(), }) .strict(); diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts index 15851ef29..2510d16c3 100644 --- a/src/memory/backend-config.ts +++ b/src/memory/backend-config.ts @@ -29,6 +29,7 @@ export type ResolvedQmdUpdateConfig = { intervalMs: number; debounceMs: number; onBoot: boolean; + embedIntervalMs: number; }; export type ResolvedQmdLimitsConfig = { @@ -59,6 +60,7 @@ const DEFAULT_CITATIONS: MemoryCitationsMode = "auto"; const DEFAULT_QMD_INTERVAL = "5m"; const DEFAULT_QMD_DEBOUNCE_MS = 15_000; const DEFAULT_QMD_TIMEOUT_MS = 4_000; +const DEFAULT_QMD_EMBED_INTERVAL = "60m"; const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = { maxResults: 6, maxSnippetChars: 700, @@ -115,6 +117,16 @@ function resolveIntervalMs(raw: string | undefined): number { } } +function resolveEmbedIntervalMs(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" }); + try { + return parseDurationMs(value, { defaultUnit: "m" }); + } catch { + return parseDurationMs(DEFAULT_QMD_EMBED_INTERVAL, { defaultUnit: "m" }); + } +} + function resolveDebounceMs(raw: number | undefined): number { if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) { return Math.floor(raw); @@ -221,7 +233,7 @@ export function resolveMemoryBackendConfig(params: { ]; const resolved: ResolvedQmdConfig = { - command: qmdCfg?.command?.trim() || "qmd", + command: (qmdCfg?.command?.trim() || "qmd").split(/\s+/)[0] || "qmd", collections, includeDefaultMemory, sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir), @@ -229,6 +241,7 @@ export function resolveMemoryBackendConfig(params: { intervalMs: resolveIntervalMs(qmdCfg?.update?.interval), debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs), onBoot: qmdCfg?.update?.onBoot !== false, + embedIntervalMs: resolveEmbedIntervalMs(qmdCfg?.update?.embedInterval), }, limits: resolveLimits(qmdCfg?.limits), scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE, diff --git a/src/memory/qmd-manager.test.ts b/src/memory/qmd-manager.test.ts index 4fcbb0c9f..c91a4dc4e 100644 --- a/src/memory/qmd-manager.test.ts +++ b/src/memory/qmd-manager.test.ts @@ -90,7 +90,8 @@ describe("QmdMemoryManager", () => { Date.now() - (resolved.qmd?.update.debounceMs ?? 0) - 10; await manager.sync({ reason: "after-wait" }); - expect(spawnMock.mock.calls.length).toBe(baselineCalls + 4); + // By default we refresh embeddings less frequently than index updates. + expect(spawnMock.mock.calls.length).toBe(baselineCalls + 3); await manager.close(); }); diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts index 003a6ca36..c984aa238 100644 --- a/src/memory/qmd-manager.ts +++ b/src/memory/qmd-manager.ts @@ -84,6 +84,7 @@ export class QmdMemoryManager implements MemorySearchManager { private closed = false; private db: SqliteDatabase | null = null; private lastUpdateAt: number | null = null; + private lastEmbedAt: number | null = null; private constructor(params: { cfg: MoltbotConfig; @@ -165,9 +166,27 @@ export class QmdMemoryManager implements MemorySearchManager { private async ensureCollections(): Promise { // QMD collections are persisted inside the index database and must be created - // via the CLI. The YAML file format is not supported by the QMD builds we - // target, so we ensure collections exist by running `qmd collection add`. + // via the CLI. Prefer listing existing collections when supported, otherwise + // fall back to best-effort idempotent `qmd collection add`. + const existing = new Set(); + try { + const result = await this.runQmd(["collection", "list", "--json"]); + const parsed = JSON.parse(result.stdout) as unknown; + if (Array.isArray(parsed)) { + for (const entry of parsed) { + if (typeof entry === "string") existing.add(entry); + else if (entry && typeof entry === "object") { + const name = (entry as { name?: unknown }).name; + if (typeof name === "string") existing.add(name); + } + } + } + } catch { + // ignore; older qmd versions might not support list --json. + } + for (const collection of this.qmd.collections) { + if (existing.has(collection.name)) continue; try { await this.runQmd([ "collection", @@ -181,7 +200,8 @@ export class QmdMemoryManager implements MemorySearchManager { } catch (err) { const message = err instanceof Error ? err.message : String(err); // Idempotency: qmd exits non-zero if the collection name already exists. - if (message.includes("already exists")) continue; + if (message.toLowerCase().includes("already exists")) continue; + if (message.toLowerCase().includes("exists")) continue; log.warn(`qmd collection add failed for ${collection.name}: ${message}`); } } @@ -335,10 +355,18 @@ export class QmdMemoryManager implements MemorySearchManager { await this.exportSessions(); } await this.runQmd(["update"], { timeoutMs: 120_000 }); - try { - await this.runQmd(["embed"], { timeoutMs: 120_000 }); - } catch (err) { - log.warn(`qmd embed failed (${reason}): ${String(err)}`); + const embedIntervalMs = this.qmd.update.embedIntervalMs; + const shouldEmbed = + Boolean(force) || + this.lastEmbedAt === null || + (embedIntervalMs > 0 && Date.now() - this.lastEmbedAt > embedIntervalMs); + if (shouldEmbed) { + try { + await this.runQmd(["embed"], { timeoutMs: 120_000 }); + this.lastEmbedAt = Date.now(); + } catch (err) { + log.warn(`qmd embed failed (${reason}): ${String(err)}`); + } } this.lastUpdateAt = Date.now(); this.docPathCache.clear(); diff --git a/src/memory/session-files.ts b/src/memory/session-files.ts index 82f8c1e09..374b0362d 100644 --- a/src/memory/session-files.ts +++ b/src/memory/session-files.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import { redactSensitiveText } from "../logging/redact.js"; import { hashText } from "./internal.js"; const log = createSubsystemLogger("memory"); @@ -87,8 +88,9 @@ export async function buildSessionEntry(absPath: string): Promise