diff --git a/CHANGELOG.md b/CHANGELOG.md index 5909c9899..5551f1e84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ Status: beta. - Docs: add GCP Compute Engine deployment guide. (#1848) Thanks @hougangdev. - Docs: add LINE channel guide. Thanks @thewilloftheshadow. - Docs: credit both contributors for Control UI refresh. (#1852) Thanks @EnzeD. +- Memory: add optional QMD-backed memory backend with fallbacks to the existing Markdown index. Thanks @vgnsh. +- Memory: surface QMD startup errors so we fall back to the legacy index when the CLI is missing, and allow extra QMD collections outside the workspace to return snippets via the `qmd//…` path prefix. +- Memory: export session transcripts into the QMD index when `memory.qmd.sessions.enabled` is set and honor `memory.citations` when formatting snippets. - Onboarding: add Venice API key to non-interactive flow. (#1893) Thanks @jonisjongithub. - Onboarding: strengthen security warning copy for beta + access control expectations. - Tlon: format thread reply IDs as @ud. (#1837) Thanks @wca4a. diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 8a386aba9..0a558a54d 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -96,6 +96,84 @@ embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or `models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint, set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`). +### QMD backend (experimental) + +Set `memory.backend = "qmd"` to swap the built-in SQLite indexer for +[QMD](https://github.com/tobi/qmd): a local-first search sidecar that combines +BM25 + vectors + reranking. Markdown stays the source of truth; Moltbot shells +out to QMD for retrieval. Key points: + +**Prereqs** +- Disabled by default. Opt in per-config (`memory.backend = "qmd"`). +- Install the QMD CLI separately (`bun install -g github.com/tobi/qmd` or grab + a release) and make sure the `qmd` binary is on the gateway’s `PATH`. +- QMD needs an SQLite build that allows extensions (`brew install sqlite` on + macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically. + +**How the sidecar runs** +- The gateway writes a self-contained QMD home under + `~/.clawdbot/agents//qmd/` (config + cache + sqlite DB). +- Collections are rewritten from `memory.qmd.paths` (plus default workspace + memory files) into `index.yml`, then `qmd update` + `qmd embed` run on boot and + on a configurable interval (`memory.qmd.update.interval`, default 5 m). +- Searches run via `qmd query --json`. If QMD fails or the binary is missing, + Moltbot automatically falls back to the builtin SQLite manager so memory tools + keep working. + +**Config surface (`memory.qmd.*`)** +- `command` (default `qmd`): override the executable path. +- `includeDefaultMemory` (default `true`): auto-index `MEMORY.md` + `memory/**/*.md`. +- `paths[]`: add extra directories/files (`path`, optional `pattern`, optional + stable `name`). +- `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`, + `exportDir`, `redactToolOutputs`—defaults to redacting tool payloads). +- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`). +- `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`, + `maxInjectedChars`, `timeoutMs`). +- `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy). + Default is DM-only (`deny` all, `allow` direct chats); loosen it to surface QMD + hits in groups/channels. +- Snippets sourced outside the workspace show up as + `qmd//` in `memory_search` results; `memory_get` + understands that prefix and reads from the configured QMD collection root. +- When `memory.qmd.sessions.enabled = true`, Moltbot exports sanitized session + transcripts (User/Assistant turns) into a dedicated QMD collection under + `~/.clawdbot/agents//qmd/sessions/`, so `memory_search` can recall recent + conversations without touching the builtin SQLite index. +- `memory_search` snippets now include a `Source: ` footer when + `memory.citations` is `auto`/`on`; set `memory.citations = "off"` to keep + the path metadata internal (the agent still receives the path for + `memory_get`, but the snippet text omits the footer and the system prompt + warns the agent not to cite it). + +**Example** + +```json5 +memory: { + backend: "qmd", + citations: "auto", + qmd: { + includeDefaultMemory: true, + update: { interval: "5m", debounceMs: 15000 }, + limits: { maxResults: 6, timeoutMs: 4000 }, + scope: { + default: "deny", + rules: [{ action: "allow", match: { chatType: "direct" } }] + }, + paths: [ + { name: "docs", path: "~/notes", pattern: "**/*.md" } + ] + } +} +``` + +**Citations & fallback** +- `memory.citations` applies regardless of backend (`auto`/`on`/`off`). +- When `qmd` runs, we tag `status().backend = "qmd"` so diagnostics show which + engine served the results. If the QMD subprocess exits or JSON output can’t be + parsed, the search manager logs a warning and returns the builtin provider + (existing Markdown embeddings) until QMD recovers. + ### Gemini embeddings (native) Set the provider to `gemini` to use the Gemini embeddings API directly: diff --git a/src/agents/cli-runner/helpers.ts b/src/agents/cli-runner/helpers.ts index 833771a1f..008ed45d1 100644 --- a/src/agents/cli-runner/helpers.ts +++ b/src/agents/cli-runner/helpers.ts @@ -212,6 +212,7 @@ export function buildSystemPrompt(params: { userTimeFormat, contextFiles: params.contextFiles, ttsHint, + memoryCitationsMode: params.config?.memory?.citations, }); } diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index dc68561c2..b592eded8 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -346,6 +346,7 @@ export async function compactEmbeddedPiSessionDirect( userTime, userTimeFormat, contextFiles, + memoryCitationsMode: params.config?.memory?.citations, }); const systemPrompt = createSystemPromptOverride(appendPrompt); diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index 46a53bd8f..9d2614a99 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -359,6 +359,7 @@ export async function runEmbeddedAttempt( userTime, userTimeFormat, contextFiles, + memoryCitationsMode: params.config?.memory?.citations, }); const systemPromptReport = buildSystemPromptReport({ source: "run", diff --git a/src/agents/pi-embedded-runner/system-prompt.ts b/src/agents/pi-embedded-runner/system-prompt.ts index 33c4bfc8b..63f6a8074 100644 --- a/src/agents/pi-embedded-runner/system-prompt.ts +++ b/src/agents/pi-embedded-runner/system-prompt.ts @@ -1,4 +1,5 @@ import type { AgentTool } from "@mariozechner/pi-agent-core"; +import type { MemoryCitationsMode } from "../../config/types.memory.js"; import type { ResolvedTimeFormat } from "../date-time.js"; import type { EmbeddedContextFile } from "../pi-embedded-helpers.js"; import { buildAgentSystemPrompt, type PromptMode } from "../system-prompt.js"; @@ -45,6 +46,7 @@ export function buildEmbeddedSystemPrompt(params: { userTime?: string; userTimeFormat?: ResolvedTimeFormat; contextFiles?: EmbeddedContextFile[]; + memoryCitationsMode?: MemoryCitationsMode; }): string { return buildAgentSystemPrompt({ workspaceDir: params.workspaceDir, @@ -70,6 +72,7 @@ export function buildEmbeddedSystemPrompt(params: { userTime: params.userTime, userTimeFormat: params.userTimeFormat, contextFiles: params.contextFiles, + memoryCitationsMode: params.memoryCitationsMode, }); } diff --git a/src/agents/system-prompt.ts b/src/agents/system-prompt.ts index ed97fd539..5a613f7af 100644 --- a/src/agents/system-prompt.ts +++ b/src/agents/system-prompt.ts @@ -1,6 +1,7 @@ import type { ReasoningLevel, ThinkLevel } from "../auto-reply/thinking.js"; import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; import { listDeliverableMessageChannels } from "../utils/message-channel.js"; +import type { MemoryCitationsMode } from "../config/types.memory.js"; import type { ResolvedTimeFormat } from "./date-time.js"; import type { EmbeddedContextFile } from "./pi-embedded-helpers.js"; @@ -32,16 +33,30 @@ function buildSkillsSection(params: { ]; } -function buildMemorySection(params: { isMinimal: boolean; availableTools: Set }) { +function buildMemorySection(params: { + isMinimal: boolean; + availableTools: Set; + citationsMode?: MemoryCitationsMode; +}) { if (params.isMinimal) return []; if (!params.availableTools.has("memory_search") && !params.availableTools.has("memory_get")) { return []; } - return [ + const lines = [ "## Memory Recall", "Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.", - "", ]; + if (params.citationsMode === "off") { + lines.push( + "Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.", + ); + } else { + lines.push( + "Citations: include Source: when it helps the user verify memory snippets.", + ); + } + lines.push(""); + return lines; } function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: boolean) { @@ -178,6 +193,7 @@ export function buildAgentSystemPrompt(params: { level: "minimal" | "extensive"; channel: string; }; + memoryCitationsMode?: MemoryCitationsMode; }) { const coreToolSummaries: Record = { read: "Read file contents", @@ -314,7 +330,11 @@ export function buildAgentSystemPrompt(params: { isMinimal, readToolName, }); - const memorySection = buildMemorySection({ isMinimal, availableTools }); + const memorySection = buildMemorySection({ + isMinimal, + availableTools, + citationsMode: params.memoryCitationsMode, + }); const docsSection = buildDocsSection({ docsPath: params.docsPath, isMinimal, diff --git a/src/agents/tools/memory-tool.citations.test.ts b/src/agents/tools/memory-tool.citations.test.ts new file mode 100644 index 000000000..111525822 --- /dev/null +++ b/src/agents/tools/memory-tool.citations.test.ts @@ -0,0 +1,65 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const stubManager = { + search: vi.fn(async () => [ + { + path: "MEMORY.md", + startLine: 5, + endLine: 7, + score: 0.9, + snippet: "@@ -5,3 @@\nAssistant: noted", + source: "memory" as const, + }, + ]), + readFile: vi.fn(), + status: () => ({ + backend: "builtin" as const, + files: 1, + chunks: 1, + dirty: false, + workspaceDir: "/workspace", + dbPath: "/workspace/.memory/index.sqlite", + provider: "builtin", + model: "builtin", + requestedProvider: "builtin", + sources: ["memory" as const], + sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }], + }), + sync: vi.fn(), + probeVectorAvailability: vi.fn(async () => true), + close: vi.fn(), +}; + +vi.mock("../../memory/index.js", () => { + return { + getMemorySearchManager: async () => ({ manager: stubManager }), + }; +}); + +import { createMemorySearchTool } from "./memory-tool.js"; + +beforeEach(() => { + vi.clearAllMocks(); +}); + +describe("memory search citations", () => { + it("appends source information when citations are enabled", async () => { + const cfg = { memory: { citations: "on" }, agents: { list: [{ id: "main", default: true }] } }; + const tool = createMemorySearchTool({ config: cfg }); + if (!tool) throw new Error("tool missing"); + const result = await tool.execute("call_citations_on", { query: "notes" }); + const details = result.details as { results: Array<{ snippet: string; citation?: string }> }; + expect(details.results[0]?.snippet).toMatch(/Source: MEMORY.md#L5-L7/); + expect(details.results[0]?.citation).toBe("MEMORY.md#L5-L7"); + }); + + it("leaves snippet untouched when citations are off", async () => { + const cfg = { memory: { citations: "off" }, agents: { list: [{ id: "main", default: true }] } }; + const tool = createMemorySearchTool({ config: cfg }); + if (!tool) throw new Error("tool missing"); + const result = await tool.execute("call_citations_off", { query: "notes" }); + const details = result.details as { results: Array<{ snippet: string; citation?: string }> }; + expect(details.results[0]?.snippet).not.toMatch(/Source:/); + expect(details.results[0]?.citation).toBeUndefined(); + }); +}); diff --git a/src/agents/tools/memory-tool.ts b/src/agents/tools/memory-tool.ts index b7b619af3..f6643bb67 100644 --- a/src/agents/tools/memory-tool.ts +++ b/src/agents/tools/memory-tool.ts @@ -1,7 +1,9 @@ import { Type } from "@sinclair/typebox"; import type { MoltbotConfig } from "../../config/config.js"; +import type { MemoryCitationsMode } from "../../config/types.memory.js"; import { getMemorySearchManager } from "../../memory/index.js"; +import type { MemorySearchResult } from "../../memory/types.js"; import { resolveSessionAgentId } from "../agent-scope.js"; import { resolveMemorySearchConfig } from "../memory-search.js"; import type { AnyAgentTool } from "./common.js"; @@ -48,17 +50,21 @@ export function createMemorySearchTool(options: { return jsonResult({ results: [], disabled: true, error }); } try { - const results = await manager.search(query, { + const citationsMode = resolveMemoryCitationsMode(cfg); + const includeCitations = citationsMode !== "off"; + const rawResults = await manager.search(query, { maxResults, minScore, sessionKey: options.agentSessionKey, }); const status = manager.status(); + const results = decorateCitations(rawResults, includeCitations); return jsonResult({ results, provider: status.provider, model: status.model, fallback: status.fallback, + citations: citationsMode, }); } catch (err) { const message = err instanceof Error ? err.message : String(err); @@ -110,3 +116,28 @@ export function createMemoryGetTool(options: { }, }; } + +function resolveMemoryCitationsMode(cfg: MoltbotConfig): MemoryCitationsMode { + const mode = cfg.memory?.citations; + if (mode === "on" || mode === "off" || mode === "auto") return mode; + return "auto"; +} + +function decorateCitations(results: MemorySearchResult[], include: boolean): MemorySearchResult[] { + if (!include) { + return results.map((entry) => ({ ...entry, citation: undefined })); + } + return results.map((entry) => { + const citation = formatCitation(entry); + const snippet = `${entry.snippet.trim()}\n\nSource: ${citation}`; + return { ...entry, citation, snippet }; + }); +} + +function formatCitation(entry: MemorySearchResult): string { + const lineRange = + entry.startLine === entry.endLine + ? `#L${entry.startLine}` + : `#L${entry.startLine}-L${entry.endLine}`; + return `${entry.path}${lineRange}`; +} diff --git a/src/auto-reply/reply/commands-context-report.ts b/src/auto-reply/reply/commands-context-report.ts index 8a65d42f5..975d5e331 100644 --- a/src/auto-reply/reply/commands-context-report.ts +++ b/src/auto-reply/reply/commands-context-report.ts @@ -150,6 +150,7 @@ async function resolveContextReport( ttsHint, runtimeInfo, sandboxInfo, + memoryCitationsMode: params.cfg?.memory?.citations, }); return buildSystemPromptReport({ diff --git a/src/commands/status.scan.ts b/src/commands/status.scan.ts index 13de0a383..5785e0ab8 100644 --- a/src/commands/status.scan.ts +++ b/src/commands/status.scan.ts @@ -6,7 +6,8 @@ import { probeGateway } from "../gateway/probe.js"; import { collectChannelStatusIssues } from "../infra/channels-status-issues.js"; import { resolveOsSummary } from "../infra/os-summary.js"; import { getTailnetHostname } from "../infra/tailscale.js"; -import type { MemoryIndexManager } from "../memory/manager.js"; +import { getMemorySearchManager } from "../memory/index.js"; +import type { MemoryProviderStatus } from "../memory/types.js"; import { runExec } from "../process/exec.js"; import type { RuntimeEnv } from "../runtime.js"; import { getAgentLocalStatuses } from "./status.agent-local.js"; @@ -15,7 +16,7 @@ import { getStatusSummary } from "./status.summary.js"; import { getUpdateCheckResult } from "./status.update.js"; import { buildChannelsTable } from "./status-all/channels.js"; -type MemoryStatusSnapshot = ReturnType & { +type MemoryStatusSnapshot = MemoryProviderStatus & { agentId: string; }; @@ -151,14 +152,13 @@ export async function scanStatus( if (!memoryPlugin.enabled) return null; if (memoryPlugin.slot !== "memory-core") return null; const agentId = agentStatus.defaultId ?? "main"; - const { MemoryIndexManager } = await import("../memory/manager.js"); - const manager = await MemoryIndexManager.get({ cfg, agentId }).catch(() => null); + const { manager } = await getMemorySearchManager({ cfg, agentId }); if (!manager) return null; try { await manager.probeVectorAvailability(); } catch {} const status = manager.status(); - await manager.close().catch(() => {}); + await manager.close?.().catch(() => {}); return { agentId, ...status }; })(); progress.tick(); diff --git a/src/config/schema.ts b/src/config/schema.ts index b4ec8723b..df38a6028 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -252,6 +252,27 @@ const FIELD_LABELS: Record = { "Memory Search Hybrid Candidate Multiplier", "agents.defaults.memorySearch.cache.enabled": "Memory Search Embedding Cache", "agents.defaults.memorySearch.cache.maxEntries": "Memory Search Embedding Cache Max Entries", + memory: "Memory", + "memory.backend": "Memory Backend", + "memory.citations": "Memory Citations Mode", + "memory.qmd.command": "QMD Binary", + "memory.qmd.includeDefaultMemory": "QMD Include Default Memory", + "memory.qmd.paths": "QMD Extra Paths", + "memory.qmd.paths.path": "QMD Path", + "memory.qmd.paths.pattern": "QMD Path Pattern", + "memory.qmd.paths.name": "QMD Path Name", + "memory.qmd.sessions.enabled": "QMD Session Indexing", + "memory.qmd.sessions.exportDir": "QMD Session Export Directory", + "memory.qmd.sessions.retentionDays": "QMD Session Retention (days)", + "memory.qmd.sessions.redactToolOutputs": "QMD Session Tool Redaction", + "memory.qmd.update.interval": "QMD Update Interval", + "memory.qmd.update.debounceMs": "QMD Update Debounce (ms)", + "memory.qmd.update.onBoot": "QMD Update on Startup", + "memory.qmd.limits.maxResults": "QMD Max Results", + "memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars", + "memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars", + "memory.qmd.limits.timeoutMs": "QMD Search Timeout (ms)", + "memory.qmd.scope": "QMD Surface Scope", "auth.profiles": "Auth Profiles", "auth.order": "Auth Profile Order", "auth.cooldowns.billingBackoffHours": "Billing Backoff (hours)", @@ -537,6 +558,37 @@ const FIELD_HELP: Record = { "Multiplier for candidate pool size (default: 4).", "agents.defaults.memorySearch.cache.enabled": "Cache chunk embeddings in SQLite to speed up reindexing and frequent updates (default: true).", + memory: "Memory backend configuration (global).", + "memory.backend": 'Memory backend ("builtin" for Moltbot embeddings, "qmd" for QMD sidecar).', + "memory.citations": 'Default citation behavior ("auto", "on", or "off").', + "memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).", + "memory.qmd.includeDefaultMemory": + "Whether to automatically index MEMORY.md + memory/**/*.md (default: true).", + "memory.qmd.paths": + "Additional directories/files to index with QMD (path + optional glob pattern).", + "memory.qmd.paths.path": "Absolute or ~-relative path to index via QMD.", + "memory.qmd.paths.pattern": "Glob pattern relative to the path root (default: **/*.md).", + "memory.qmd.paths.name": + "Optional stable name for the QMD collection (default derived from path).", + "memory.qmd.sessions.enabled": + "Enable QMD session transcript indexing (experimental, default: false).", + "memory.qmd.sessions.exportDir": + "Override directory for sanitized session exports before indexing.", + "memory.qmd.sessions.retentionDays": + "Retention window for exported sessions before pruning (default: unlimited).", + "memory.qmd.sessions.redactToolOutputs": + "Strip tool call payloads/results when exporting sessions (default: true).", + "memory.qmd.update.interval": + "How often the QMD sidecar refreshes indexes (duration string, default: 5m).", + "memory.qmd.update.debounceMs": + "Minimum delay between successive QMD refresh runs (default: 15000).", + "memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).", + "memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).", + "memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).", + "memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.", + "memory.qmd.limits.timeoutMs": "Per-query timeout for QMD searches (default: 4000).", + "memory.qmd.scope": + "Session/channel scope for QMD recall (same syntax as session.sendPolicy; default: direct-only).", "agents.defaults.memorySearch.cache.maxEntries": "Optional cap on cached embeddings (best-effort).", "agents.defaults.memorySearch.sync.onSearch": diff --git a/src/config/types.clawdbot.ts b/src/config/types.clawdbot.ts index c2b2f05e4..81f0ed1c8 100644 --- a/src/config/types.clawdbot.ts +++ b/src/config/types.clawdbot.ts @@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js"; import type { PluginsConfig } from "./types.plugins.js"; import type { SkillsConfig } from "./types.skills.js"; import type { ToolsConfig } from "./types.tools.js"; +import type { MemoryConfig } from "./types.memory.js"; export type MoltbotConfig = { meta?: { @@ -95,6 +96,7 @@ export type MoltbotConfig = { canvasHost?: CanvasHostConfig; talk?: TalkConfig; gateway?: GatewayConfig; + memory?: MemoryConfig; }; export type ConfigValidationIssue = { diff --git a/src/config/types.memory.ts b/src/config/types.memory.ts new file mode 100644 index 000000000..a6bb5eeb3 --- /dev/null +++ b/src/config/types.memory.ts @@ -0,0 +1,46 @@ +import type { SessionSendPolicyConfig } from "./types.base.js"; + +export type MemoryBackend = "builtin" | "qmd"; +export type MemoryCitationsMode = "auto" | "on" | "off"; + +export type MemoryConfig = { + backend?: MemoryBackend; + citations?: MemoryCitationsMode; + qmd?: MemoryQmdConfig; +}; + +export type MemoryQmdConfig = { + command?: string; + includeDefaultMemory?: boolean; + paths?: MemoryQmdIndexPath[]; + sessions?: MemoryQmdSessionConfig; + update?: MemoryQmdUpdateConfig; + limits?: MemoryQmdLimitsConfig; + scope?: SessionSendPolicyConfig; +}; + +export type MemoryQmdIndexPath = { + path: string; + name?: string; + pattern?: string; +}; + +export type MemoryQmdSessionConfig = { + enabled?: boolean; + exportDir?: string; + retentionDays?: number; + redactToolOutputs?: boolean; +}; + +export type MemoryQmdUpdateConfig = { + interval?: string; + debounceMs?: number; + onBoot?: boolean; +}; + +export type MemoryQmdLimitsConfig = { + maxResults?: number; + maxSnippetChars?: number; + maxInjectedChars?: number; + timeoutMs?: number; +}; diff --git a/src/config/types.ts b/src/config/types.ts index 424ccaf26..c92e11004 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -28,3 +28,4 @@ export * from "./types.telegram.js"; export * from "./types.tts.js"; export * from "./types.tools.js"; export * from "./types.whatsapp.js"; +export * from "./types.memory.js"; diff --git a/src/config/zod-schema.session.ts b/src/config/zod-schema.session.ts index 4412f5515..6841a0a90 100644 --- a/src/config/zod-schema.session.ts +++ b/src/config/zod-schema.session.ts @@ -16,6 +16,31 @@ const SessionResetConfigSchema = z }) .strict(); +export const SessionSendPolicySchema = z + .object({ + default: z.union([z.literal("allow"), z.literal("deny")]).optional(), + rules: z + .array( + z + .object({ + action: z.union([z.literal("allow"), z.literal("deny")]), + match: z + .object({ + channel: z.string().optional(), + chatType: z + .union([z.literal("direct"), z.literal("group"), z.literal("channel")]) + .optional(), + keyPrefix: z.string().optional(), + }) + .strict() + .optional(), + }) + .strict(), + ) + .optional(), + }) + .strict(); + export const SessionSchema = z .object({ scope: z.union([z.literal("per-sender"), z.literal("global")]).optional(), @@ -51,31 +76,7 @@ export const SessionSchema = z ]) .optional(), mainKey: z.string().optional(), - sendPolicy: z - .object({ - default: z.union([z.literal("allow"), z.literal("deny")]).optional(), - rules: z - .array( - z - .object({ - action: z.union([z.literal("allow"), z.literal("deny")]), - match: z - .object({ - channel: z.string().optional(), - chatType: z - .union([z.literal("direct"), z.literal("group"), z.literal("channel")]) - .optional(), - keyPrefix: z.string().optional(), - }) - .strict() - .optional(), - }) - .strict(), - ) - .optional(), - }) - .strict() - .optional(), + sendPolicy: SessionSendPolicySchema.optional(), agentToAgent: z .object({ maxPingPongTurns: z.number().int().min(0).max(5).optional(), diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index ce4115517..46f4fc16c 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -5,7 +5,12 @@ import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zo import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js"; import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js"; import { ChannelsSchema } from "./zod-schema.providers.js"; -import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js"; +import { + CommandsSchema, + MessagesSchema, + SessionSchema, + SessionSendPolicySchema, +} from "./zod-schema.session.js"; const BrowserSnapshotDefaultsSchema = z .object({ @@ -27,6 +32,61 @@ const NodeHostSchema = z .strict() .optional(); +const MemoryQmdPathSchema = z + .object({ + path: z.string(), + name: z.string().optional(), + pattern: z.string().optional(), + }) + .strict(); + +const MemoryQmdSessionSchema = z + .object({ + enabled: z.boolean().optional(), + exportDir: z.string().optional(), + retentionDays: z.number().int().nonnegative().optional(), + redactToolOutputs: z.boolean().optional(), + }) + .strict(); + +const MemoryQmdUpdateSchema = z + .object({ + interval: z.string().optional(), + debounceMs: z.number().int().nonnegative().optional(), + onBoot: z.boolean().optional(), + }) + .strict(); + +const MemoryQmdLimitsSchema = z + .object({ + maxResults: z.number().int().positive().optional(), + maxSnippetChars: z.number().int().positive().optional(), + maxInjectedChars: z.number().int().positive().optional(), + timeoutMs: z.number().int().nonnegative().optional(), + }) + .strict(); + +const MemoryQmdSchema = z + .object({ + command: z.string().optional(), + includeDefaultMemory: z.boolean().optional(), + paths: z.array(MemoryQmdPathSchema).optional(), + sessions: MemoryQmdSessionSchema.optional(), + update: MemoryQmdUpdateSchema.optional(), + limits: MemoryQmdLimitsSchema.optional(), + scope: SessionSendPolicySchema.optional(), + }) + .strict(); + +const MemorySchema = z + .object({ + backend: z.union([z.literal("builtin"), z.literal("qmd")]).optional(), + citations: z.union([z.literal("auto"), z.literal("on"), z.literal("off")]).optional(), + qmd: MemoryQmdSchema.optional(), + }) + .strict() + .optional(); + export const MoltbotSchema = z .object({ meta: z @@ -445,6 +505,7 @@ export const MoltbotSchema = z }) .strict() .optional(), + memory: MemorySchema, skills: z .object({ allowBundled: z.array(z.string()).optional(), diff --git a/src/memory/backend-config.test.ts b/src/memory/backend-config.test.ts new file mode 100644 index 000000000..4e9db881c --- /dev/null +++ b/src/memory/backend-config.test.ts @@ -0,0 +1,58 @@ +import path from "node:path"; + +import { describe, expect, it } from "vitest"; + +import type { MoltbotConfig } from "../config/config.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveMemoryBackendConfig } from "./backend-config.js"; + +describe("resolveMemoryBackendConfig", () => { + it("defaults to builtin backend when config missing", () => { + const cfg = { agents: { defaults: { workspace: "/tmp/memory-test" } } } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + expect(resolved.backend).toBe("builtin"); + expect(resolved.citations).toBe("auto"); + expect(resolved.qmd).toBeUndefined(); + }); + + it("resolves qmd backend with default collections", () => { + const cfg = { + agents: { defaults: { workspace: "/tmp/memory-test" } }, + memory: { + backend: "qmd", + qmd: {}, + }, + } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + expect(resolved.backend).toBe("qmd"); + expect(resolved.qmd?.collections.length).toBeGreaterThanOrEqual(3); + expect(resolved.qmd?.command).toBe("qmd"); + expect(resolved.qmd?.update.intervalMs).toBeGreaterThan(0); + }); + + it("resolves custom paths relative to workspace", () => { + const cfg = { + agents: { + defaults: { workspace: "/workspace/root" }, + list: [{ id: "main", workspace: "/workspace/root" }], + }, + memory: { + backend: "qmd", + qmd: { + paths: [ + { + path: "notes", + name: "custom-notes", + pattern: "**/*.md", + }, + ], + }, + }, + } as MoltbotConfig; + const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" }); + const custom = resolved.qmd?.collections.find((c) => c.name.startsWith("custom-notes")); + expect(custom).toBeDefined(); + const workspaceRoot = resolveAgentWorkspaceDir(cfg, "main"); + expect(custom?.path).toBe(path.resolve(workspaceRoot, "notes")); + }); +}); diff --git a/src/memory/backend-config.ts b/src/memory/backend-config.ts new file mode 100644 index 000000000..c7c4a0122 --- /dev/null +++ b/src/memory/backend-config.ts @@ -0,0 +1,245 @@ +import path from "node:path"; + +import { parseDurationMs } from "../cli/parse-duration.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import type { MoltbotConfig } from "../config/config.js"; +import type { + MemoryBackend, + MemoryCitationsMode, + MemoryQmdConfig, + MemoryQmdIndexPath, +} from "../config/types.memory.js"; +import type { SessionSendPolicyConfig } from "../config/types.base.js"; +import { resolveUserPath } from "../utils.js"; + +export type ResolvedMemoryBackendConfig = { + backend: MemoryBackend; + citations: MemoryCitationsMode; + qmd?: ResolvedQmdConfig; +}; + +export type ResolvedQmdCollection = { + name: string; + path: string; + pattern: string; + kind: "memory" | "custom" | "sessions"; +}; + +export type ResolvedQmdUpdateConfig = { + intervalMs: number; + debounceMs: number; + onBoot: boolean; +}; + +export type ResolvedQmdLimitsConfig = { + maxResults: number; + maxSnippetChars: number; + maxInjectedChars: number; + timeoutMs: number; +}; + +export type ResolvedQmdSessionConfig = { + enabled: boolean; + exportDir?: string; + retentionDays?: number; + redactToolOutputs: boolean; +}; + +export type ResolvedQmdConfig = { + command: string; + collections: ResolvedQmdCollection[]; + sessions: ResolvedQmdSessionConfig; + update: ResolvedQmdUpdateConfig; + limits: ResolvedQmdLimitsConfig; + includeDefaultMemory: boolean; + scope?: SessionSendPolicyConfig; +}; + +const DEFAULT_BACKEND: MemoryBackend = "builtin"; +const DEFAULT_CITATIONS: MemoryCitationsMode = "auto"; +const DEFAULT_QMD_INTERVAL = "5m"; +const DEFAULT_QMD_DEBOUNCE_MS = 15_000; +const DEFAULT_QMD_TIMEOUT_MS = 4_000; +const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = { + maxResults: 6, + maxSnippetChars: 700, + maxInjectedChars: 4_000, + timeoutMs: DEFAULT_QMD_TIMEOUT_MS, +}; +const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = { + default: "deny", + rules: [ + { + action: "allow", + match: { chatType: "direct" }, + }, + ], +}; + +function sanitizeName(input: string): string { + const lower = input.toLowerCase().replace(/[^a-z0-9-]+/g, "-"); + const trimmed = lower.replace(/^-+|-+$/g, ""); + return trimmed || "collection"; +} + +function ensureUniqueName(base: string, existing: Set): string { + let name = sanitizeName(base); + if (!existing.has(name)) { + existing.add(name); + return name; + } + let suffix = 2; + while (existing.has(`${name}-${suffix}`)) { + suffix += 1; + } + const unique = `${name}-${suffix}`; + existing.add(unique); + return unique; +} + +function resolvePath(raw: string, workspaceDir: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error("path required"); + if (trimmed.startsWith("~") || path.isAbsolute(trimmed)) { + return path.normalize(resolveUserPath(trimmed)); + } + return path.normalize(path.resolve(workspaceDir, trimmed)); +} + +function resolveIntervalMs(raw: string | undefined): number { + const value = raw?.trim(); + if (!value) return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" }); + try { + return parseDurationMs(value, { defaultUnit: "m" }); + } catch { + return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" }); + } +} + +function resolveDebounceMs(raw: number | undefined): number { + if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) { + return Math.floor(raw); + } + return DEFAULT_QMD_DEBOUNCE_MS; +} + +function resolveLimits(raw?: MemoryQmdConfig["limits"]): ResolvedQmdLimitsConfig { + const parsed: ResolvedQmdLimitsConfig = { ...DEFAULT_QMD_LIMITS }; + if (raw?.maxResults && raw.maxResults > 0) parsed.maxResults = Math.floor(raw.maxResults); + if (raw?.maxSnippetChars && raw.maxSnippetChars > 0) { + parsed.maxSnippetChars = Math.floor(raw.maxSnippetChars); + } + if (raw?.maxInjectedChars && raw.maxInjectedChars > 0) { + parsed.maxInjectedChars = Math.floor(raw.maxInjectedChars); + } + if (raw?.timeoutMs && raw.timeoutMs > 0) { + parsed.timeoutMs = Math.floor(raw.timeoutMs); + } + return parsed; +} + +function resolveSessionConfig( + cfg: MemoryQmdConfig["sessions"], + workspaceDir: string, +): ResolvedQmdSessionConfig { + const enabled = Boolean(cfg?.enabled); + const exportDirRaw = cfg?.exportDir?.trim(); + const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined; + const retentionDays = + cfg?.retentionDays && cfg.retentionDays > 0 ? Math.floor(cfg.retentionDays) : undefined; + const redactToolOutputs = cfg?.redactToolOutputs !== false; + return { + enabled, + exportDir, + retentionDays, + redactToolOutputs, + }; +} + +function resolveCustomPaths( + rawPaths: MemoryQmdIndexPath[] | undefined, + workspaceDir: string, + existing: Set, +): ResolvedQmdCollection[] { + if (!rawPaths?.length) return []; + const collections: ResolvedQmdCollection[] = []; + rawPaths.forEach((entry, index) => { + const trimmedPath = entry?.path?.trim(); + if (!trimmedPath) return; + let resolved: string; + try { + resolved = resolvePath(trimmedPath, workspaceDir); + } catch { + return; + } + const pattern = entry.pattern?.trim() || "**/*.md"; + const baseName = entry.name?.trim() || `custom-${index + 1}`; + const name = ensureUniqueName(baseName, existing); + collections.push({ + name, + path: resolved, + pattern, + kind: "custom", + }); + }); + return collections; +} + +function resolveDefaultCollections( + include: boolean, + workspaceDir: string, + existing: Set, +): ResolvedQmdCollection[] { + if (!include) return []; + const entries: Array<{ path: string; pattern: string; base: string }> = [ + { path: workspaceDir, pattern: "MEMORY.md", base: "memory-root" }, + { path: workspaceDir, pattern: "memory.md", base: "memory-alt" }, + { path: path.join(workspaceDir, "memory"), pattern: "**/*.md", base: "memory-dir" }, + ]; + return entries.map((entry) => ({ + name: ensureUniqueName(entry.base, existing), + path: entry.path, + pattern: entry.pattern, + kind: "memory", + })); +} + +export function resolveMemoryBackendConfig(params: { + cfg: MoltbotConfig; + agentId: string; +}): ResolvedMemoryBackendConfig { + const backend = params.cfg.memory?.backend ?? DEFAULT_BACKEND; + const citations = params.cfg.memory?.citations ?? DEFAULT_CITATIONS; + if (backend !== "qmd") { + return { backend: "builtin", citations }; + } + + const workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId); + const qmdCfg = params.cfg.memory?.qmd; + const includeDefaultMemory = qmdCfg?.includeDefaultMemory !== false; + const nameSet = new Set(); + const collections = [ + ...resolveDefaultCollections(includeDefaultMemory, workspaceDir, nameSet), + ...resolveCustomPaths(qmdCfg?.paths, workspaceDir, nameSet), + ]; + + const resolved: ResolvedQmdConfig = { + command: qmdCfg?.command?.trim() || "qmd", + collections, + includeDefaultMemory, + sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir), + update: { + intervalMs: resolveIntervalMs(qmdCfg?.update?.interval), + debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs), + onBoot: qmdCfg?.update?.onBoot !== false, + }, + limits: resolveLimits(qmdCfg?.limits), + scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE, + }; + + return { + backend: "qmd", + citations, + qmd: resolved, + }; +} diff --git a/src/memory/index.ts b/src/memory/index.ts index 092c4b03c..a4d9a4da1 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,2 +1,3 @@ -export type { MemoryIndexManager, MemorySearchResult } from "./manager.js"; +export { MemoryIndexManager } from "./manager.js"; +export type { MemorySearchResult, MemorySearchManager } from "./types.js"; export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js"; diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 9a9991d10..5d42c571b 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -45,17 +45,13 @@ import { searchKeyword, searchVector } from "./manager-search.js"; import { ensureMemoryIndexSchema } from "./memory-schema.js"; import { requireNodeSqlite } from "./sqlite.js"; import { loadSqliteVecExtension } from "./sqlite-vec.js"; - -type MemorySource = "memory" | "sessions"; - -export type MemorySearchResult = { - path: string; - startLine: number; - endLine: number; - score: number; - snippet: string; - source: MemorySource; -}; +import type { + MemoryProviderStatus, + MemorySearchManager, + MemorySearchResult, + MemorySource, + MemorySyncProgressUpdate, +} from "./types.js"; type MemoryIndexMeta = { model: string; @@ -75,12 +71,6 @@ type SessionFileEntry = { content: string; }; -type MemorySyncProgressUpdate = { - completed: number; - total: number; - label?: string; -}; - type MemorySyncProgressState = { completed: number; total: number; @@ -115,7 +105,7 @@ const INDEX_CACHE = new Map(); const vectorToBlob = (embedding: number[]): Buffer => Buffer.from(new Float32Array(embedding).buffer); -export class MemoryIndexManager { +export class MemoryIndexManager implements MemorySearchManager { private readonly cacheKey: string; private readonly cfg: MoltbotConfig; private readonly agentId: string; @@ -415,39 +405,7 @@ export class MemoryIndexManager { return { text: slice.join("\n"), path: relPath }; } - status(): { - files: number; - chunks: number; - dirty: boolean; - workspaceDir: string; - dbPath: string; - provider: string; - model: string; - requestedProvider: string; - sources: MemorySource[]; - sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>; - cache?: { enabled: boolean; entries?: number; maxEntries?: number }; - fts?: { enabled: boolean; available: boolean; error?: string }; - fallback?: { from: string; reason?: string }; - vector?: { - enabled: boolean; - available?: boolean; - extensionPath?: string; - loadError?: string; - dims?: number; - }; - batch?: { - enabled: boolean; - failures: number; - limit: number; - wait: boolean; - concurrency: number; - pollIntervalMs: number; - timeoutMs: number; - lastError?: string; - lastProvider?: string; - }; - } { + status(): MemoryProviderStatus { const sourceFilter = this.buildSourceFilter(); const files = this.db .prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`) @@ -489,9 +447,10 @@ export class MemoryIndexManager { return sources.map((source) => ({ source, ...bySource.get(source)! })); })(); return { + backend: "builtin", files: files?.c ?? 0, chunks: chunks?.c ?? 0, - dirty: this.dirty, + dirty: this.dirty || this.sessionsDirty, workspaceDir: this.workspaceDir, dbPath: this.settings.store.path, provider: this.provider.id, diff --git a/src/memory/qmd-manager.ts b/src/memory/qmd-manager.ts new file mode 100644 index 000000000..28e68bca4 --- /dev/null +++ b/src/memory/qmd-manager.ts @@ -0,0 +1,612 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +import YAML from "yaml"; + +import type { MoltbotConfig } from "../config/config.js"; +import { resolveStateDir } from "../config/paths.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { + listSessionFilesForAgent, + buildSessionEntry, + type SessionFileEntry, +} from "./session-files.js"; +import { requireNodeSqlite } from "./sqlite.js"; +import type { + MemoryProviderStatus, + MemorySearchManager, + MemorySearchResult, + MemorySource, + MemorySyncProgressUpdate, +} from "./types.js"; +import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js"; + +const log = createSubsystemLogger("memory"); + +const SNIPPET_HEADER_RE = /@@\s*-([0-9]+),([0-9]+)/; + +type QmdQueryResult = { + docid?: string; + score?: number; + file?: string; + snippet?: string; + body?: string; +}; + +type CollectionRoot = { + path: string; + kind: MemorySource; +}; + +type SessionExporterConfig = { + dir: string; + retentionMs?: number; + collectionName: string; +}; + +export class QmdMemoryManager implements MemorySearchManager { + static async create(params: { + cfg: MoltbotConfig; + agentId: string; + resolved: ResolvedMemoryBackendConfig; + }): Promise { + const resolved = params.resolved.qmd; + if (!resolved) return null; + const manager = new QmdMemoryManager({ cfg: params.cfg, agentId: params.agentId, resolved }); + await manager.initialize(); + return manager; + } + + private readonly cfg: MoltbotConfig; + private readonly agentId: string; + private readonly qmd: ResolvedQmdConfig; + private readonly workspaceDir: string; + private readonly stateDir: string; + private readonly agentStateDir: string; + private readonly qmdDir: string; + private readonly cacheDir: string; + private readonly configDir: string; + private readonly xdgConfigHome: string; + private readonly xdgCacheHome: string; + private readonly collectionsFile: string; + private readonly indexPath: string; + private readonly env: NodeJS.ProcessEnv; + private readonly collectionRoots = new Map(); + private readonly sources = new Set(); + private readonly docPathCache = new Map< + string, + { rel: string; abs: string; source: MemorySource } + >(); + private readonly sessionExporter: SessionExporterConfig | null; + private updateTimer: NodeJS.Timeout | null = null; + private pendingUpdate: Promise | null = null; + private closed = false; + private db: import("node:sqlite").DatabaseSync | null = null; + private lastUpdateAt: number | null = null; + + private constructor(params: { + cfg: MoltbotConfig; + agentId: string; + resolved: ResolvedQmdConfig; + }) { + this.cfg = params.cfg; + this.agentId = params.agentId; + this.qmd = params.resolved; + this.workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId); + this.stateDir = resolveStateDir(process.env, os.homedir); + this.agentStateDir = path.join(this.stateDir, "agents", this.agentId); + this.qmdDir = path.join(this.agentStateDir, "qmd"); + this.cacheDir = path.join(this.qmdDir, "cache"); + this.configDir = path.join(this.qmdDir, "config"); + this.xdgConfigHome = path.join(this.qmdDir, "xdg-config"); + this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache"); + this.collectionsFile = path.join(this.configDir, "index.yml"); + this.indexPath = path.join(this.cacheDir, "index.sqlite"); + this.env = { + ...process.env, + QMD_CONFIG_DIR: this.configDir, + XDG_CONFIG_HOME: this.xdgConfigHome, + XDG_CACHE_HOME: this.xdgCacheHome, + INDEX_PATH: this.indexPath, + NO_COLOR: "1", + }; + this.sessionExporter = this.qmd.sessions.enabled + ? { + dir: this.qmd.sessions.exportDir ?? path.join(this.qmdDir, "sessions"), + retentionMs: this.qmd.sessions.retentionDays + ? this.qmd.sessions.retentionDays * 24 * 60 * 60 * 1000 + : undefined, + collectionName: this.pickSessionCollectionName(), + } + : null; + if (this.sessionExporter) { + this.qmd.collections = [ + ...this.qmd.collections, + { + name: this.sessionExporter.collectionName, + path: this.sessionExporter.dir, + pattern: "**/*.md", + kind: "sessions", + }, + ]; + } + } + + private async initialize(): Promise { + await fs.mkdir(this.cacheDir, { recursive: true }); + await fs.mkdir(this.configDir, { recursive: true }); + await fs.mkdir(this.xdgConfigHome, { recursive: true }); + await fs.mkdir(this.xdgCacheHome, { recursive: true }); + + this.bootstrapCollections(); + await this.writeCollectionsConfig(); + + if (this.qmd.update.onBoot) { + await this.runUpdate("boot", true); + } + if (this.qmd.update.intervalMs > 0) { + this.updateTimer = setInterval(() => { + void this.runUpdate("interval").catch((err) => { + log.warn(`qmd update failed (${String(err)})`); + }); + }, this.qmd.update.intervalMs); + } + } + + private bootstrapCollections(): void { + this.collectionRoots.clear(); + this.sources.clear(); + for (const collection of this.qmd.collections) { + const kind: MemorySource = collection.kind === "sessions" ? "sessions" : "memory"; + this.collectionRoots.set(collection.name, { path: collection.path, kind }); + this.sources.add(kind); + } + } + + private async writeCollectionsConfig(): Promise { + const collections: Record = {}; + for (const collection of this.qmd.collections) { + collections[collection.name] = { + path: collection.path, + pattern: collection.pattern, + }; + } + const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 }); + await fs.writeFile(this.collectionsFile, yaml, "utf-8"); + } + + async search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ): Promise { + if (!this.isScopeAllowed(opts?.sessionKey)) return []; + const trimmed = query.trim(); + if (!trimmed) return []; + await this.pendingUpdate?.catch(() => undefined); + const limit = Math.min( + this.qmd.limits.maxResults, + opts?.maxResults ?? this.qmd.limits.maxResults, + ); + const args = ["query", trimmed, "--json", "-n", String(limit)]; + let stdout: string; + try { + const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs }); + stdout = result.stdout; + } catch (err) { + log.warn(`qmd query failed: ${String(err)}`); + throw err instanceof Error ? err : new Error(String(err)); + } + let parsed: QmdQueryResult[] = []; + try { + parsed = JSON.parse(stdout); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`qmd query returned invalid JSON: ${message}`); + throw new Error(`qmd query returned invalid JSON: ${message}`); + } + const results: MemorySearchResult[] = []; + for (const entry of parsed) { + const doc = await this.resolveDocLocation(entry.docid); + if (!doc) continue; + const snippet = entry.snippet?.slice(0, this.qmd.limits.maxSnippetChars) ?? ""; + const lines = this.extractSnippetLines(snippet); + const score = typeof entry.score === "number" ? entry.score : 0; + const minScore = opts?.minScore ?? 0; + if (score < minScore) continue; + results.push({ + path: doc.rel, + startLine: lines.startLine, + endLine: lines.endLine, + score, + snippet, + source: doc.source, + }); + } + return results.slice(0, limit); + } + + async sync(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }): Promise { + if (params?.progress) { + params.progress({ completed: 0, total: 1, label: "Updating QMD index…" }); + } + await this.runUpdate(params?.reason ?? "manual", params?.force); + if (params?.progress) { + params.progress({ completed: 1, total: 1, label: "QMD index updated" }); + } + } + + async readFile(params: { + relPath: string; + from?: number; + lines?: number; + }): Promise<{ text: string; path: string }> { + const relPath = params.relPath?.trim(); + if (!relPath) throw new Error("path required"); + const absPath = this.resolveReadPath(relPath); + const content = await fs.readFile(absPath, "utf-8"); + if (!params.from && !params.lines) { + return { text: content, path: relPath }; + } + const lines = content.split("\n"); + const start = Math.max(1, params.from ?? 1); + const count = Math.max(1, params.lines ?? lines.length); + const slice = lines.slice(start - 1, start - 1 + count); + return { text: slice.join("\n"), path: relPath }; + } + + status(): MemoryProviderStatus { + const counts = this.readCounts(); + return { + backend: "qmd", + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + files: counts.totalDocuments, + chunks: counts.totalDocuments, + dirty: false, + workspaceDir: this.workspaceDir, + dbPath: this.indexPath, + sources: Array.from(this.sources), + sourceCounts: counts.sourceCounts, + vector: { enabled: true, available: true }, + batch: { + enabled: false, + failures: 0, + limit: 0, + wait: false, + concurrency: 0, + pollIntervalMs: 0, + timeoutMs: 0, + }, + custom: { + qmd: { + collections: this.qmd.collections.length, + lastUpdateAt: this.lastUpdateAt, + }, + }, + }; + } + + async probeVectorAvailability(): Promise { + return true; + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + if (this.updateTimer) { + clearInterval(this.updateTimer); + this.updateTimer = null; + } + await this.pendingUpdate?.catch(() => undefined); + if (this.db) { + this.db.close(); + this.db = null; + } + } + + private async runUpdate(reason: string, force?: boolean): Promise { + if (this.pendingUpdate && !force) return this.pendingUpdate; + const run = async () => { + if (this.sessionExporter) { + await this.exportSessions(); + } + await this.runQmd(["update"], { timeoutMs: 120_000 }); + try { + await this.runQmd(["embed"], { timeoutMs: 120_000 }); + } catch (err) { + log.warn(`qmd embed failed (${reason}): ${String(err)}`); + } + this.lastUpdateAt = Date.now(); + this.docPathCache.clear(); + }; + this.pendingUpdate = run().finally(() => { + this.pendingUpdate = null; + }); + await this.pendingUpdate; + } + + private async runQmd( + args: string[], + opts?: { timeoutMs?: number }, + ): Promise<{ stdout: string; stderr: string }> { + return await new Promise((resolve, reject) => { + const child = spawn(this.qmd.command, args, { + env: this.env, + cwd: this.workspaceDir, + }); + let stdout = ""; + let stderr = ""; + const timer = opts?.timeoutMs + ? setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`qmd ${args.join(" ")} timed out after ${opts.timeoutMs}ms`)); + }, opts.timeoutMs) + : null; + child.stdout.on("data", (data) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data) => { + stderr += data.toString(); + }); + child.on("error", (err) => { + if (timer) clearTimeout(timer); + reject(err); + }); + child.on("close", (code) => { + if (timer) clearTimeout(timer); + if (code === 0) { + resolve({ stdout, stderr }); + } else { + reject(new Error(`qmd ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`)); + } + }); + }); + } + + private ensureDb() { + if (this.db) return this.db; + const sqlite = requireNodeSqlite(); + this.db = sqlite.open(this.indexPath, { readonly: true }); + return this.db; + } + + private async exportSessions(): Promise { + if (!this.sessionExporter) return; + const exportDir = this.sessionExporter.dir; + await fs.mkdir(exportDir, { recursive: true }); + const files = await listSessionFilesForAgent(this.agentId); + const keep = new Set(); + const cutoff = this.sessionExporter.retentionMs + ? Date.now() - this.sessionExporter.retentionMs + : null; + for (const sessionFile of files) { + const entry = await buildSessionEntry(sessionFile); + if (!entry) continue; + if (cutoff && entry.mtimeMs < cutoff) continue; + const target = path.join(exportDir, `${path.basename(sessionFile, ".jsonl")}.md`); + await fs.writeFile(target, this.renderSessionMarkdown(entry), "utf-8"); + keep.add(target); + } + const exported = await fs.readdir(exportDir).catch(() => []); + for (const name of exported) { + if (!name.endsWith(".md")) continue; + const full = path.join(exportDir, name); + if (!keep.has(full)) { + await fs.rm(full, { force: true }); + } + } + } + + private renderSessionMarkdown(entry: SessionFileEntry): string { + const header = `# Session ${path.basename(entry.absPath, path.extname(entry.absPath))}`; + const body = entry.content?.trim().length ? entry.content.trim() : "(empty)"; + return `${header}\n\n${body}\n`; + } + + private pickSessionCollectionName(): string { + const existing = new Set(this.qmd.collections.map((collection) => collection.name)); + if (!existing.has("sessions")) return "sessions"; + let counter = 2; + let candidate = `sessions-${counter}`; + while (existing.has(candidate)) { + counter += 1; + candidate = `sessions-${counter}`; + } + return candidate; + } + + private async resolveDocLocation( + docid?: string, + ): Promise<{ rel: string; abs: string; source: MemorySource } | null> { + if (!docid) return null; + const normalized = docid.startsWith("#") ? docid.slice(1) : docid; + if (!normalized) return null; + const cached = this.docPathCache.get(normalized); + if (cached) return cached; + const db = this.ensureDb(); + const row = db + .prepare("SELECT collection, path FROM documents WHERE hash LIKE ? AND active = 1 LIMIT 1") + .get(`${normalized}%`) as { collection: string; path: string } | undefined; + if (!row) return null; + const location = this.toDocLocation(row.collection, row.path); + if (!location) return null; + this.docPathCache.set(normalized, location); + return location; + } + + private extractSnippetLines(snippet: string): { startLine: number; endLine: number } { + const match = SNIPPET_HEADER_RE.exec(snippet); + if (match) { + const start = Number(match[1]); + const count = Number(match[2]); + if (Number.isFinite(start) && Number.isFinite(count)) { + return { startLine: start, endLine: start + count - 1 }; + } + } + const lines = snippet.split("\n").length; + return { startLine: 1, endLine: lines }; + } + + private readCounts(): { + totalDocuments: number; + sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>; + } { + try { + const db = this.ensureDb(); + const rows = db + .prepare( + "SELECT collection, COUNT(*) as c FROM documents WHERE active = 1 GROUP BY collection", + ) + .all() as Array<{ collection: string; c: number }>; + const bySource = new Map(); + for (const source of this.sources) { + bySource.set(source, { files: 0, chunks: 0 }); + } + let total = 0; + for (const row of rows) { + const root = this.collectionRoots.get(row.collection); + const source = root?.kind ?? "memory"; + const entry = bySource.get(source) ?? { files: 0, chunks: 0 }; + entry.files += row.c ?? 0; + entry.chunks += row.c ?? 0; + bySource.set(source, entry); + total += row.c ?? 0; + } + return { + totalDocuments: total, + sourceCounts: Array.from(bySource.entries()).map(([source, value]) => ({ + source, + files: value.files, + chunks: value.chunks, + })), + }; + } catch (err) { + log.warn(`failed to read qmd index stats: ${String(err)}`); + return { + totalDocuments: 0, + sourceCounts: Array.from(this.sources).map((source) => ({ source, files: 0, chunks: 0 })), + }; + } + } + + private isScopeAllowed(sessionKey?: string): boolean { + const scope = this.qmd.scope; + if (!scope) return true; + const channel = this.deriveChannelFromKey(sessionKey); + const chatType = this.deriveChatTypeFromKey(sessionKey); + const normalizedKey = sessionKey ?? ""; + for (const rule of scope.rules ?? []) { + if (!rule) continue; + const match = rule.match ?? {}; + if (match.channel && match.channel !== channel) continue; + if (match.chatType && match.chatType !== chatType) continue; + if (match.keyPrefix && !normalizedKey.startsWith(match.keyPrefix)) continue; + return rule.action === "allow"; + } + const fallback = scope.default ?? "allow"; + return fallback === "allow"; + } + + private deriveChannelFromKey(key?: string) { + if (!key) return undefined; + const parts = key.split(":").filter(Boolean); + if (parts.length >= 3 && (parts[1] === "group" || parts[1] === "channel")) { + return parts[0]?.toLowerCase(); + } + return undefined; + } + + private deriveChatTypeFromKey(key?: string) { + if (!key) return undefined; + if (key.includes(":group:")) return "group"; + if (key.includes(":channel:")) return "channel"; + return "direct"; + } + + private toDocLocation( + collection: string, + collectionRelativePath: string, + ): { rel: string; abs: string; source: MemorySource } | null { + const root = this.collectionRoots.get(collection); + if (!root) return null; + const normalizedRelative = collectionRelativePath.replace(/\\/g, "/"); + const absPath = path.normalize(path.resolve(root.path, collectionRelativePath)); + const relativeToWorkspace = path.relative(this.workspaceDir, absPath); + const relPath = this.buildSearchPath( + collection, + normalizedRelative, + relativeToWorkspace, + absPath, + ); + return { rel: relPath, abs: absPath, source: root.kind }; + } + + private buildSearchPath( + collection: string, + collectionRelativePath: string, + relativeToWorkspace: string, + absPath: string, + ): string { + const insideWorkspace = this.isInsideWorkspace(relativeToWorkspace); + if (insideWorkspace) { + const normalized = relativeToWorkspace.replace(/\\/g, "/"); + if (!normalized) return path.basename(absPath); + return normalized; + } + const sanitized = collectionRelativePath.replace(/^\/+/, ""); + return `qmd/${collection}/${sanitized}`; + } + + private isInsideWorkspace(relativePath: string): boolean { + if (!relativePath) return true; + if (relativePath.startsWith("..")) return false; + if (relativePath.startsWith(`..${path.sep}`)) return false; + return !path.isAbsolute(relativePath); + } + + private resolveReadPath(relPath: string): string { + if (relPath.startsWith("qmd/")) { + const [, collection, ...rest] = relPath.split("/"); + if (!collection || rest.length === 0) { + throw new Error("invalid qmd path"); + } + const root = this.collectionRoots.get(collection); + if (!root) throw new Error(`unknown qmd collection: ${collection}`); + const joined = rest.join("/"); + const resolved = path.resolve(root.path, joined); + if (!this.isWithinRoot(root.path, resolved)) { + throw new Error("qmd path escapes collection"); + } + return resolved; + } + const absPath = path.resolve(this.workspaceDir, relPath); + if (!this.isWithinWorkspace(absPath)) { + throw new Error("path escapes workspace"); + } + return absPath; + } + + private isWithinWorkspace(absPath: string): boolean { + const normalizedWorkspace = this.workspaceDir.endsWith(path.sep) + ? this.workspaceDir + : `${this.workspaceDir}${path.sep}`; + if (absPath === this.workspaceDir) return true; + const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`; + return candidate.startsWith(normalizedWorkspace); + } + + private isWithinRoot(root: string, candidate: string): boolean { + const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`; + if (candidate === root) return true; + const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`; + return next.startsWith(normalizedRoot); + } +} diff --git a/src/memory/search-manager.test.ts b/src/memory/search-manager.test.ts new file mode 100644 index 000000000..7dd822fa5 --- /dev/null +++ b/src/memory/search-manager.test.ts @@ -0,0 +1,62 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mockPrimary = { + search: vi.fn(async () => []), + readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })), + status: vi.fn(() => ({ + backend: "qmd" as const, + provider: "qmd", + model: "qmd", + requestedProvider: "qmd", + files: 0, + chunks: 0, + dirty: false, + workspaceDir: "/tmp", + dbPath: "/tmp/index.sqlite", + sources: ["memory" as const], + sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }], + })), + sync: vi.fn(async () => {}), + probeVectorAvailability: vi.fn(async () => true), + close: vi.fn(async () => {}), +}; + +vi.mock("./qmd-manager.js", () => ({ + QmdMemoryManager: { + create: vi.fn(async () => mockPrimary), + }, +})); + +vi.mock("./manager.js", () => ({ + MemoryIndexManager: { + get: vi.fn(async () => null), + }, +})); + +import { QmdMemoryManager } from "./qmd-manager.js"; +import { getMemorySearchManager } from "./search-manager.js"; + +beforeEach(() => { + mockPrimary.search.mockClear(); + mockPrimary.readFile.mockClear(); + mockPrimary.status.mockClear(); + mockPrimary.sync.mockClear(); + mockPrimary.probeVectorAvailability.mockClear(); + mockPrimary.close.mockClear(); + QmdMemoryManager.create.mockClear(); +}); + +describe("getMemorySearchManager caching", () => { + it("reuses the same QMD manager instance for repeated calls", async () => { + const cfg = { + memory: { backend: "qmd", qmd: {} }, + agents: { list: [{ id: "main", default: true, workspace: "/tmp/workspace" }] }, + } as const; + + const first = await getMemorySearchManager({ cfg, agentId: "main" }); + const second = await getMemorySearchManager({ cfg, agentId: "main" }); + + expect(first.manager).toBe(second.manager); + expect(QmdMemoryManager.create).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index 9bcd529f3..f3a0f07af 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -1,8 +1,15 @@ +import { createSubsystemLogger } from "../logging/subsystem.js"; import type { MoltbotConfig } from "../config/config.js"; +import { resolveMemoryBackendConfig } from "./backend-config.js"; +import type { ResolvedQmdConfig } from "./backend-config.js"; import type { MemoryIndexManager } from "./manager.js"; +import type { MemorySearchManager, MemorySyncProgressUpdate } from "./types.js"; + +const log = createSubsystemLogger("memory"); +const QMD_MANAGER_CACHE = new Map(); export type MemorySearchManagerResult = { - manager: MemoryIndexManager | null; + manager: MemorySearchManager | null; error?: string; }; @@ -10,6 +17,40 @@ export async function getMemorySearchManager(params: { cfg: MoltbotConfig; agentId: string; }): Promise { + const resolved = resolveMemoryBackendConfig(params); + if (resolved.backend === "qmd" && resolved.qmd) { + const cacheKey = buildQmdCacheKey(params.agentId, resolved.qmd); + const cached = QMD_MANAGER_CACHE.get(cacheKey); + if (cached) { + return { manager: cached }; + } + try { + const { QmdMemoryManager } = await import("./qmd-manager.js"); + const primary = await QmdMemoryManager.create({ + cfg: params.cfg, + agentId: params.agentId, + resolved, + }); + if (primary) { + const wrapper = new FallbackMemoryManager( + { + primary, + fallbackFactory: async () => { + const { MemoryIndexManager } = await import("./manager.js"); + return await MemoryIndexManager.get(params); + }, + }, + () => QMD_MANAGER_CACHE.delete(cacheKey), + ); + QMD_MANAGER_CACHE.set(cacheKey, wrapper); + return { manager: wrapper }; + } + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + log.warn(`qmd memory unavailable; falling back to builtin: ${message}`); + } + } + try { const { MemoryIndexManager } = await import("./manager.js"); const manager = await MemoryIndexManager.get(params); @@ -19,3 +60,117 @@ export async function getMemorySearchManager(params: { return { manager: null, error: message }; } } + +class FallbackMemoryManager implements MemorySearchManager { + private fallback: MemorySearchManager | null = null; + private primaryFailed = false; + private lastError?: string; + + constructor( + private readonly deps: { + primary: MemorySearchManager; + fallbackFactory: () => Promise; + }, + private readonly onClose?: () => void, + ) {} + + async search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ) { + if (!this.primaryFailed) { + try { + return await this.deps.primary.search(query, opts); + } catch (err) { + this.primaryFailed = true; + this.lastError = err instanceof Error ? err.message : String(err); + log.warn(`qmd memory failed; switching to builtin index: ${this.lastError}`); + await this.deps.primary.close?.().catch(() => {}); + } + } + const fallback = await this.ensureFallback(); + if (fallback) { + return await fallback.search(query, opts); + } + throw new Error(this.lastError ?? "memory search unavailable"); + } + + async readFile(params: { relPath: string; from?: number; lines?: number }) { + if (!this.primaryFailed) { + return await this.deps.primary.readFile(params); + } + const fallback = await this.ensureFallback(); + if (fallback) { + return await fallback.readFile(params); + } + throw new Error(this.lastError ?? "memory read unavailable"); + } + + status() { + if (!this.primaryFailed) { + return this.deps.primary.status(); + } + const fallbackStatus = this.fallback?.status(); + if (fallbackStatus) { + const custom = fallbackStatus.custom ?? {}; + return { + ...fallbackStatus, + custom: { + ...custom, + fallback: { disabled: true, reason: this.lastError ?? "unknown" }, + }, + }; + } + const primaryStatus = this.deps.primary.status(); + const custom = primaryStatus.custom ?? {}; + return { + ...primaryStatus, + custom: { + ...custom, + fallback: { disabled: true, reason: this.lastError ?? "unknown" }, + }, + }; + } + + async sync(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }) { + if (!this.primaryFailed) { + await this.deps.primary.sync?.(params); + return; + } + const fallback = await this.ensureFallback(); + await fallback?.sync?.(params); + } + + async probeVectorAvailability() { + if (!this.primaryFailed) { + return await this.deps.primary.probeVectorAvailability(); + } + const fallback = await this.ensureFallback(); + return (await fallback?.probeVectorAvailability()) ?? false; + } + + async close() { + await this.deps.primary.close?.(); + await this.fallback?.close?.(); + this.onClose?.(); + } + + private async ensureFallback(): Promise { + if (this.fallback) return this.fallback; + const fallback = await this.deps.fallbackFactory(); + if (!fallback) { + log.warn("memory fallback requested but builtin index is unavailable"); + return null; + } + this.fallback = fallback; + return this.fallback; + } +} + +function buildQmdCacheKey(agentId: string, config: ResolvedQmdConfig): string { + return `${agentId}:${JSON.stringify(config)}`; +} diff --git a/src/memory/types.ts b/src/memory/types.ts new file mode 100644 index 000000000..c26940ede --- /dev/null +++ b/src/memory/types.ts @@ -0,0 +1,72 @@ +export type MemorySource = "memory" | "sessions"; + +export type MemorySearchResult = { + path: string; + startLine: number; + endLine: number; + score: number; + snippet: string; + source: MemorySource; + citation?: string; +}; + +export type MemorySyncProgressUpdate = { + completed: number; + total: number; + label?: string; +}; + +export type MemoryProviderStatus = { + backend: "builtin" | "qmd"; + provider: string; + model?: string; + requestedProvider?: string; + files?: number; + chunks?: number; + dirty?: boolean; + workspaceDir?: string; + dbPath?: string; + sources?: MemorySource[]; + cache?: { enabled: boolean; entries?: number; maxEntries?: number }; + fts?: { enabled: boolean; available: boolean; error?: string }; + fallback?: { from: string; reason?: string }; + vector?: { + enabled: boolean; + available?: boolean; + extensionPath?: string; + loadError?: string; + dims?: number; + }; + batch?: { + enabled: boolean; + failures: number; + limit: number; + wait: boolean; + concurrency: number; + pollIntervalMs: number; + timeoutMs: number; + lastError?: string; + lastProvider?: string; + }; + custom?: Record; +}; + +export interface MemorySearchManager { + search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ): Promise; + readFile(params: { + relPath: string; + from?: number; + lines?: number; + }): Promise<{ text: string; path: string }>; + status(): MemoryProviderStatus; + sync?(params?: { + reason?: string; + force?: boolean; + progress?: (update: MemorySyncProgressUpdate) => void; + }): Promise; + probeVectorAvailability(): Promise; + close?(): Promise; +}