feat (memory): Implement new (opt-in) QMD memory backend
This commit is contained in:
parent
9688454a30
commit
1cb4af074f
@ -35,6 +35,9 @@ Status: beta.
|
||||
- Docs: add GCP Compute Engine deployment guide. (#1848) Thanks @hougangdev.
|
||||
- Docs: add LINE channel guide. Thanks @thewilloftheshadow.
|
||||
- Docs: credit both contributors for Control UI refresh. (#1852) Thanks @EnzeD.
|
||||
- Memory: add optional QMD-backed memory backend with fallbacks to the existing Markdown index. Thanks @vgnsh.
|
||||
- Memory: surface QMD startup errors so we fall back to the legacy index when the CLI is missing, and allow extra QMD collections outside the workspace to return snippets via the `qmd/<collection>/…` path prefix.
|
||||
- Memory: export session transcripts into the QMD index when `memory.qmd.sessions.enabled` is set and honor `memory.citations` when formatting snippets.
|
||||
- Onboarding: add Venice API key to non-interactive flow. (#1893) Thanks @jonisjongithub.
|
||||
- Onboarding: strengthen security warning copy for beta + access control expectations.
|
||||
- Tlon: format thread reply IDs as @ud. (#1837) Thanks @wca4a.
|
||||
|
||||
@ -96,6 +96,84 @@ embeddings for memory search. For Gemini, use `GEMINI_API_KEY` or
|
||||
`models.providers.google.apiKey`. When using a custom OpenAI-compatible endpoint,
|
||||
set `memorySearch.remote.apiKey` (and optional `memorySearch.remote.headers`).
|
||||
|
||||
### QMD backend (experimental)
|
||||
|
||||
Set `memory.backend = "qmd"` to swap the built-in SQLite indexer for
|
||||
[QMD](https://github.com/tobi/qmd): a local-first search sidecar that combines
|
||||
BM25 + vectors + reranking. Markdown stays the source of truth; Moltbot shells
|
||||
out to QMD for retrieval. Key points:
|
||||
|
||||
**Prereqs**
|
||||
- Disabled by default. Opt in per-config (`memory.backend = "qmd"`).
|
||||
- Install the QMD CLI separately (`bun install -g github.com/tobi/qmd` or grab
|
||||
a release) and make sure the `qmd` binary is on the gateway’s `PATH`.
|
||||
- QMD needs an SQLite build that allows extensions (`brew install sqlite` on
|
||||
macOS). The gateway sets `INDEX_PATH`/`QMD_CONFIG_DIR` automatically.
|
||||
|
||||
**How the sidecar runs**
|
||||
- The gateway writes a self-contained QMD home under
|
||||
`~/.clawdbot/agents/<agentId>/qmd/` (config + cache + sqlite DB).
|
||||
- Collections are rewritten from `memory.qmd.paths` (plus default workspace
|
||||
memory files) into `index.yml`, then `qmd update` + `qmd embed` run on boot and
|
||||
on a configurable interval (`memory.qmd.update.interval`, default 5 m).
|
||||
- Searches run via `qmd query --json`. If QMD fails or the binary is missing,
|
||||
Moltbot automatically falls back to the builtin SQLite manager so memory tools
|
||||
keep working.
|
||||
|
||||
**Config surface (`memory.qmd.*`)**
|
||||
- `command` (default `qmd`): override the executable path.
|
||||
- `includeDefaultMemory` (default `true`): auto-index `MEMORY.md` + `memory/**/*.md`.
|
||||
- `paths[]`: add extra directories/files (`path`, optional `pattern`, optional
|
||||
stable `name`).
|
||||
- `sessions`: opt into session JSONL indexing (`enabled`, `retentionDays`,
|
||||
`exportDir`, `redactToolOutputs`—defaults to redacting tool payloads).
|
||||
- `update`: controls refresh cadence (`interval`, `debounceMs`, `onBoot`).
|
||||
- `limits`: clamp recall payload (`maxResults`, `maxSnippetChars`,
|
||||
`maxInjectedChars`, `timeoutMs`).
|
||||
- `scope`: same schema as [`session.sendPolicy`](/reference/configuration#session-sendpolicy).
|
||||
Default is DM-only (`deny` all, `allow` direct chats); loosen it to surface QMD
|
||||
hits in groups/channels.
|
||||
- Snippets sourced outside the workspace show up as
|
||||
`qmd/<collection>/<relative-path>` in `memory_search` results; `memory_get`
|
||||
understands that prefix and reads from the configured QMD collection root.
|
||||
- When `memory.qmd.sessions.enabled = true`, Moltbot exports sanitized session
|
||||
transcripts (User/Assistant turns) into a dedicated QMD collection under
|
||||
`~/.clawdbot/agents/<id>/qmd/sessions/`, so `memory_search` can recall recent
|
||||
conversations without touching the builtin SQLite index.
|
||||
- `memory_search` snippets now include a `Source: <path#line>` footer when
|
||||
`memory.citations` is `auto`/`on`; set `memory.citations = "off"` to keep
|
||||
the path metadata internal (the agent still receives the path for
|
||||
`memory_get`, but the snippet text omits the footer and the system prompt
|
||||
warns the agent not to cite it).
|
||||
|
||||
**Example**
|
||||
|
||||
```json5
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
citations: "auto",
|
||||
qmd: {
|
||||
includeDefaultMemory: true,
|
||||
update: { interval: "5m", debounceMs: 15000 },
|
||||
limits: { maxResults: 6, timeoutMs: 4000 },
|
||||
scope: {
|
||||
default: "deny",
|
||||
rules: [{ action: "allow", match: { chatType: "direct" } }]
|
||||
},
|
||||
paths: [
|
||||
{ name: "docs", path: "~/notes", pattern: "**/*.md" }
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Citations & fallback**
|
||||
- `memory.citations` applies regardless of backend (`auto`/`on`/`off`).
|
||||
- When `qmd` runs, we tag `status().backend = "qmd"` so diagnostics show which
|
||||
engine served the results. If the QMD subprocess exits or JSON output can’t be
|
||||
parsed, the search manager logs a warning and returns the builtin provider
|
||||
(existing Markdown embeddings) until QMD recovers.
|
||||
|
||||
### Gemini embeddings (native)
|
||||
|
||||
Set the provider to `gemini` to use the Gemini embeddings API directly:
|
||||
|
||||
@ -212,6 +212,7 @@ export function buildSystemPrompt(params: {
|
||||
userTimeFormat,
|
||||
contextFiles: params.contextFiles,
|
||||
ttsHint,
|
||||
memoryCitationsMode: params.config?.memory?.citations,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -346,6 +346,7 @@ export async function compactEmbeddedPiSessionDirect(
|
||||
userTime,
|
||||
userTimeFormat,
|
||||
contextFiles,
|
||||
memoryCitationsMode: params.config?.memory?.citations,
|
||||
});
|
||||
const systemPrompt = createSystemPromptOverride(appendPrompt);
|
||||
|
||||
|
||||
@ -359,6 +359,7 @@ export async function runEmbeddedAttempt(
|
||||
userTime,
|
||||
userTimeFormat,
|
||||
contextFiles,
|
||||
memoryCitationsMode: params.config?.memory?.citations,
|
||||
});
|
||||
const systemPromptReport = buildSystemPromptReport({
|
||||
source: "run",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import type { AgentTool } from "@mariozechner/pi-agent-core";
|
||||
import type { MemoryCitationsMode } from "../../config/types.memory.js";
|
||||
import type { ResolvedTimeFormat } from "../date-time.js";
|
||||
import type { EmbeddedContextFile } from "../pi-embedded-helpers.js";
|
||||
import { buildAgentSystemPrompt, type PromptMode } from "../system-prompt.js";
|
||||
@ -45,6 +46,7 @@ export function buildEmbeddedSystemPrompt(params: {
|
||||
userTime?: string;
|
||||
userTimeFormat?: ResolvedTimeFormat;
|
||||
contextFiles?: EmbeddedContextFile[];
|
||||
memoryCitationsMode?: MemoryCitationsMode;
|
||||
}): string {
|
||||
return buildAgentSystemPrompt({
|
||||
workspaceDir: params.workspaceDir,
|
||||
@ -70,6 +72,7 @@ export function buildEmbeddedSystemPrompt(params: {
|
||||
userTime: params.userTime,
|
||||
userTimeFormat: params.userTimeFormat,
|
||||
contextFiles: params.contextFiles,
|
||||
memoryCitationsMode: params.memoryCitationsMode,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import type { ReasoningLevel, ThinkLevel } from "../auto-reply/thinking.js";
|
||||
import { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
|
||||
import { listDeliverableMessageChannels } from "../utils/message-channel.js";
|
||||
import type { MemoryCitationsMode } from "../config/types.memory.js";
|
||||
import type { ResolvedTimeFormat } from "./date-time.js";
|
||||
import type { EmbeddedContextFile } from "./pi-embedded-helpers.js";
|
||||
|
||||
@ -32,16 +33,30 @@ function buildSkillsSection(params: {
|
||||
];
|
||||
}
|
||||
|
||||
function buildMemorySection(params: { isMinimal: boolean; availableTools: Set<string> }) {
|
||||
function buildMemorySection(params: {
|
||||
isMinimal: boolean;
|
||||
availableTools: Set<string>;
|
||||
citationsMode?: MemoryCitationsMode;
|
||||
}) {
|
||||
if (params.isMinimal) return [];
|
||||
if (!params.availableTools.has("memory_search") && !params.availableTools.has("memory_get")) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
const lines = [
|
||||
"## Memory Recall",
|
||||
"Before answering anything about prior work, decisions, dates, people, preferences, or todos: run memory_search on MEMORY.md + memory/*.md; then use memory_get to pull only the needed lines. If low confidence after search, say you checked.",
|
||||
"",
|
||||
];
|
||||
if (params.citationsMode === "off") {
|
||||
lines.push(
|
||||
"Citations are disabled: do not mention file paths or line numbers in replies unless the user explicitly asks.",
|
||||
);
|
||||
} else {
|
||||
lines.push(
|
||||
"Citations: include Source: <path#line> when it helps the user verify memory snippets.",
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
return lines;
|
||||
}
|
||||
|
||||
function buildUserIdentitySection(ownerLine: string | undefined, isMinimal: boolean) {
|
||||
@ -178,6 +193,7 @@ export function buildAgentSystemPrompt(params: {
|
||||
level: "minimal" | "extensive";
|
||||
channel: string;
|
||||
};
|
||||
memoryCitationsMode?: MemoryCitationsMode;
|
||||
}) {
|
||||
const coreToolSummaries: Record<string, string> = {
|
||||
read: "Read file contents",
|
||||
@ -314,7 +330,11 @@ export function buildAgentSystemPrompt(params: {
|
||||
isMinimal,
|
||||
readToolName,
|
||||
});
|
||||
const memorySection = buildMemorySection({ isMinimal, availableTools });
|
||||
const memorySection = buildMemorySection({
|
||||
isMinimal,
|
||||
availableTools,
|
||||
citationsMode: params.memoryCitationsMode,
|
||||
});
|
||||
const docsSection = buildDocsSection({
|
||||
docsPath: params.docsPath,
|
||||
isMinimal,
|
||||
|
||||
65
src/agents/tools/memory-tool.citations.test.ts
Normal file
65
src/agents/tools/memory-tool.citations.test.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const stubManager = {
|
||||
search: vi.fn(async () => [
|
||||
{
|
||||
path: "MEMORY.md",
|
||||
startLine: 5,
|
||||
endLine: 7,
|
||||
score: 0.9,
|
||||
snippet: "@@ -5,3 @@\nAssistant: noted",
|
||||
source: "memory" as const,
|
||||
},
|
||||
]),
|
||||
readFile: vi.fn(),
|
||||
status: () => ({
|
||||
backend: "builtin" as const,
|
||||
files: 1,
|
||||
chunks: 1,
|
||||
dirty: false,
|
||||
workspaceDir: "/workspace",
|
||||
dbPath: "/workspace/.memory/index.sqlite",
|
||||
provider: "builtin",
|
||||
model: "builtin",
|
||||
requestedProvider: "builtin",
|
||||
sources: ["memory" as const],
|
||||
sourceCounts: [{ source: "memory" as const, files: 1, chunks: 1 }],
|
||||
}),
|
||||
sync: vi.fn(),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
close: vi.fn(),
|
||||
};
|
||||
|
||||
vi.mock("../../memory/index.js", () => {
|
||||
return {
|
||||
getMemorySearchManager: async () => ({ manager: stubManager }),
|
||||
};
|
||||
});
|
||||
|
||||
import { createMemorySearchTool } from "./memory-tool.js";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("memory search citations", () => {
|
||||
it("appends source information when citations are enabled", async () => {
|
||||
const cfg = { memory: { citations: "on" }, agents: { list: [{ id: "main", default: true }] } };
|
||||
const tool = createMemorySearchTool({ config: cfg });
|
||||
if (!tool) throw new Error("tool missing");
|
||||
const result = await tool.execute("call_citations_on", { query: "notes" });
|
||||
const details = result.details as { results: Array<{ snippet: string; citation?: string }> };
|
||||
expect(details.results[0]?.snippet).toMatch(/Source: MEMORY.md#L5-L7/);
|
||||
expect(details.results[0]?.citation).toBe("MEMORY.md#L5-L7");
|
||||
});
|
||||
|
||||
it("leaves snippet untouched when citations are off", async () => {
|
||||
const cfg = { memory: { citations: "off" }, agents: { list: [{ id: "main", default: true }] } };
|
||||
const tool = createMemorySearchTool({ config: cfg });
|
||||
if (!tool) throw new Error("tool missing");
|
||||
const result = await tool.execute("call_citations_off", { query: "notes" });
|
||||
const details = result.details as { results: Array<{ snippet: string; citation?: string }> };
|
||||
expect(details.results[0]?.snippet).not.toMatch(/Source:/);
|
||||
expect(details.results[0]?.citation).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@ -1,7 +1,9 @@
|
||||
import { Type } from "@sinclair/typebox";
|
||||
|
||||
import type { MoltbotConfig } from "../../config/config.js";
|
||||
import type { MemoryCitationsMode } from "../../config/types.memory.js";
|
||||
import { getMemorySearchManager } from "../../memory/index.js";
|
||||
import type { MemorySearchResult } from "../../memory/types.js";
|
||||
import { resolveSessionAgentId } from "../agent-scope.js";
|
||||
import { resolveMemorySearchConfig } from "../memory-search.js";
|
||||
import type { AnyAgentTool } from "./common.js";
|
||||
@ -48,17 +50,21 @@ export function createMemorySearchTool(options: {
|
||||
return jsonResult({ results: [], disabled: true, error });
|
||||
}
|
||||
try {
|
||||
const results = await manager.search(query, {
|
||||
const citationsMode = resolveMemoryCitationsMode(cfg);
|
||||
const includeCitations = citationsMode !== "off";
|
||||
const rawResults = await manager.search(query, {
|
||||
maxResults,
|
||||
minScore,
|
||||
sessionKey: options.agentSessionKey,
|
||||
});
|
||||
const status = manager.status();
|
||||
const results = decorateCitations(rawResults, includeCitations);
|
||||
return jsonResult({
|
||||
results,
|
||||
provider: status.provider,
|
||||
model: status.model,
|
||||
fallback: status.fallback,
|
||||
citations: citationsMode,
|
||||
});
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
@ -110,3 +116,28 @@ export function createMemoryGetTool(options: {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function resolveMemoryCitationsMode(cfg: MoltbotConfig): MemoryCitationsMode {
|
||||
const mode = cfg.memory?.citations;
|
||||
if (mode === "on" || mode === "off" || mode === "auto") return mode;
|
||||
return "auto";
|
||||
}
|
||||
|
||||
function decorateCitations(results: MemorySearchResult[], include: boolean): MemorySearchResult[] {
|
||||
if (!include) {
|
||||
return results.map((entry) => ({ ...entry, citation: undefined }));
|
||||
}
|
||||
return results.map((entry) => {
|
||||
const citation = formatCitation(entry);
|
||||
const snippet = `${entry.snippet.trim()}\n\nSource: ${citation}`;
|
||||
return { ...entry, citation, snippet };
|
||||
});
|
||||
}
|
||||
|
||||
function formatCitation(entry: MemorySearchResult): string {
|
||||
const lineRange =
|
||||
entry.startLine === entry.endLine
|
||||
? `#L${entry.startLine}`
|
||||
: `#L${entry.startLine}-L${entry.endLine}`;
|
||||
return `${entry.path}${lineRange}`;
|
||||
}
|
||||
|
||||
@ -150,6 +150,7 @@ async function resolveContextReport(
|
||||
ttsHint,
|
||||
runtimeInfo,
|
||||
sandboxInfo,
|
||||
memoryCitationsMode: params.cfg?.memory?.citations,
|
||||
});
|
||||
|
||||
return buildSystemPromptReport({
|
||||
|
||||
@ -6,7 +6,8 @@ import { probeGateway } from "../gateway/probe.js";
|
||||
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
|
||||
import { resolveOsSummary } from "../infra/os-summary.js";
|
||||
import { getTailnetHostname } from "../infra/tailscale.js";
|
||||
import type { MemoryIndexManager } from "../memory/manager.js";
|
||||
import { getMemorySearchManager } from "../memory/index.js";
|
||||
import type { MemoryProviderStatus } from "../memory/types.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { getAgentLocalStatuses } from "./status.agent-local.js";
|
||||
@ -15,7 +16,7 @@ import { getStatusSummary } from "./status.summary.js";
|
||||
import { getUpdateCheckResult } from "./status.update.js";
|
||||
import { buildChannelsTable } from "./status-all/channels.js";
|
||||
|
||||
type MemoryStatusSnapshot = ReturnType<MemoryIndexManager["status"]> & {
|
||||
type MemoryStatusSnapshot = MemoryProviderStatus & {
|
||||
agentId: string;
|
||||
};
|
||||
|
||||
@ -151,14 +152,13 @@ export async function scanStatus(
|
||||
if (!memoryPlugin.enabled) return null;
|
||||
if (memoryPlugin.slot !== "memory-core") return null;
|
||||
const agentId = agentStatus.defaultId ?? "main";
|
||||
const { MemoryIndexManager } = await import("../memory/manager.js");
|
||||
const manager = await MemoryIndexManager.get({ cfg, agentId }).catch(() => null);
|
||||
const { manager } = await getMemorySearchManager({ cfg, agentId });
|
||||
if (!manager) return null;
|
||||
try {
|
||||
await manager.probeVectorAvailability();
|
||||
} catch {}
|
||||
const status = manager.status();
|
||||
await manager.close().catch(() => {});
|
||||
await manager.close?.().catch(() => {});
|
||||
return { agentId, ...status };
|
||||
})();
|
||||
progress.tick();
|
||||
|
||||
@ -252,6 +252,27 @@ const FIELD_LABELS: Record<string, string> = {
|
||||
"Memory Search Hybrid Candidate Multiplier",
|
||||
"agents.defaults.memorySearch.cache.enabled": "Memory Search Embedding Cache",
|
||||
"agents.defaults.memorySearch.cache.maxEntries": "Memory Search Embedding Cache Max Entries",
|
||||
memory: "Memory",
|
||||
"memory.backend": "Memory Backend",
|
||||
"memory.citations": "Memory Citations Mode",
|
||||
"memory.qmd.command": "QMD Binary",
|
||||
"memory.qmd.includeDefaultMemory": "QMD Include Default Memory",
|
||||
"memory.qmd.paths": "QMD Extra Paths",
|
||||
"memory.qmd.paths.path": "QMD Path",
|
||||
"memory.qmd.paths.pattern": "QMD Path Pattern",
|
||||
"memory.qmd.paths.name": "QMD Path Name",
|
||||
"memory.qmd.sessions.enabled": "QMD Session Indexing",
|
||||
"memory.qmd.sessions.exportDir": "QMD Session Export Directory",
|
||||
"memory.qmd.sessions.retentionDays": "QMD Session Retention (days)",
|
||||
"memory.qmd.sessions.redactToolOutputs": "QMD Session Tool Redaction",
|
||||
"memory.qmd.update.interval": "QMD Update Interval",
|
||||
"memory.qmd.update.debounceMs": "QMD Update Debounce (ms)",
|
||||
"memory.qmd.update.onBoot": "QMD Update on Startup",
|
||||
"memory.qmd.limits.maxResults": "QMD Max Results",
|
||||
"memory.qmd.limits.maxSnippetChars": "QMD Max Snippet Chars",
|
||||
"memory.qmd.limits.maxInjectedChars": "QMD Max Injected Chars",
|
||||
"memory.qmd.limits.timeoutMs": "QMD Search Timeout (ms)",
|
||||
"memory.qmd.scope": "QMD Surface Scope",
|
||||
"auth.profiles": "Auth Profiles",
|
||||
"auth.order": "Auth Profile Order",
|
||||
"auth.cooldowns.billingBackoffHours": "Billing Backoff (hours)",
|
||||
@ -537,6 +558,37 @@ const FIELD_HELP: Record<string, string> = {
|
||||
"Multiplier for candidate pool size (default: 4).",
|
||||
"agents.defaults.memorySearch.cache.enabled":
|
||||
"Cache chunk embeddings in SQLite to speed up reindexing and frequent updates (default: true).",
|
||||
memory: "Memory backend configuration (global).",
|
||||
"memory.backend": 'Memory backend ("builtin" for Moltbot embeddings, "qmd" for QMD sidecar).',
|
||||
"memory.citations": 'Default citation behavior ("auto", "on", or "off").',
|
||||
"memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).",
|
||||
"memory.qmd.includeDefaultMemory":
|
||||
"Whether to automatically index MEMORY.md + memory/**/*.md (default: true).",
|
||||
"memory.qmd.paths":
|
||||
"Additional directories/files to index with QMD (path + optional glob pattern).",
|
||||
"memory.qmd.paths.path": "Absolute or ~-relative path to index via QMD.",
|
||||
"memory.qmd.paths.pattern": "Glob pattern relative to the path root (default: **/*.md).",
|
||||
"memory.qmd.paths.name":
|
||||
"Optional stable name for the QMD collection (default derived from path).",
|
||||
"memory.qmd.sessions.enabled":
|
||||
"Enable QMD session transcript indexing (experimental, default: false).",
|
||||
"memory.qmd.sessions.exportDir":
|
||||
"Override directory for sanitized session exports before indexing.",
|
||||
"memory.qmd.sessions.retentionDays":
|
||||
"Retention window for exported sessions before pruning (default: unlimited).",
|
||||
"memory.qmd.sessions.redactToolOutputs":
|
||||
"Strip tool call payloads/results when exporting sessions (default: true).",
|
||||
"memory.qmd.update.interval":
|
||||
"How often the QMD sidecar refreshes indexes (duration string, default: 5m).",
|
||||
"memory.qmd.update.debounceMs":
|
||||
"Minimum delay between successive QMD refresh runs (default: 15000).",
|
||||
"memory.qmd.update.onBoot": "Run QMD update once on gateway startup (default: true).",
|
||||
"memory.qmd.limits.maxResults": "Max QMD results returned to the agent loop (default: 6).",
|
||||
"memory.qmd.limits.maxSnippetChars": "Max characters per snippet pulled from QMD (default: 700).",
|
||||
"memory.qmd.limits.maxInjectedChars": "Max total characters injected from QMD hits per turn.",
|
||||
"memory.qmd.limits.timeoutMs": "Per-query timeout for QMD searches (default: 4000).",
|
||||
"memory.qmd.scope":
|
||||
"Session/channel scope for QMD recall (same syntax as session.sendPolicy; default: direct-only).",
|
||||
"agents.defaults.memorySearch.cache.maxEntries":
|
||||
"Optional cap on cached embeddings (best-effort).",
|
||||
"agents.defaults.memorySearch.sync.onSearch":
|
||||
|
||||
@ -23,6 +23,7 @@ import type { NodeHostConfig } from "./types.node-host.js";
|
||||
import type { PluginsConfig } from "./types.plugins.js";
|
||||
import type { SkillsConfig } from "./types.skills.js";
|
||||
import type { ToolsConfig } from "./types.tools.js";
|
||||
import type { MemoryConfig } from "./types.memory.js";
|
||||
|
||||
export type MoltbotConfig = {
|
||||
meta?: {
|
||||
@ -95,6 +96,7 @@ export type MoltbotConfig = {
|
||||
canvasHost?: CanvasHostConfig;
|
||||
talk?: TalkConfig;
|
||||
gateway?: GatewayConfig;
|
||||
memory?: MemoryConfig;
|
||||
};
|
||||
|
||||
export type ConfigValidationIssue = {
|
||||
|
||||
46
src/config/types.memory.ts
Normal file
46
src/config/types.memory.ts
Normal file
@ -0,0 +1,46 @@
|
||||
import type { SessionSendPolicyConfig } from "./types.base.js";
|
||||
|
||||
export type MemoryBackend = "builtin" | "qmd";
|
||||
export type MemoryCitationsMode = "auto" | "on" | "off";
|
||||
|
||||
export type MemoryConfig = {
|
||||
backend?: MemoryBackend;
|
||||
citations?: MemoryCitationsMode;
|
||||
qmd?: MemoryQmdConfig;
|
||||
};
|
||||
|
||||
export type MemoryQmdConfig = {
|
||||
command?: string;
|
||||
includeDefaultMemory?: boolean;
|
||||
paths?: MemoryQmdIndexPath[];
|
||||
sessions?: MemoryQmdSessionConfig;
|
||||
update?: MemoryQmdUpdateConfig;
|
||||
limits?: MemoryQmdLimitsConfig;
|
||||
scope?: SessionSendPolicyConfig;
|
||||
};
|
||||
|
||||
export type MemoryQmdIndexPath = {
|
||||
path: string;
|
||||
name?: string;
|
||||
pattern?: string;
|
||||
};
|
||||
|
||||
export type MemoryQmdSessionConfig = {
|
||||
enabled?: boolean;
|
||||
exportDir?: string;
|
||||
retentionDays?: number;
|
||||
redactToolOutputs?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryQmdUpdateConfig = {
|
||||
interval?: string;
|
||||
debounceMs?: number;
|
||||
onBoot?: boolean;
|
||||
};
|
||||
|
||||
export type MemoryQmdLimitsConfig = {
|
||||
maxResults?: number;
|
||||
maxSnippetChars?: number;
|
||||
maxInjectedChars?: number;
|
||||
timeoutMs?: number;
|
||||
};
|
||||
@ -28,3 +28,4 @@ export * from "./types.telegram.js";
|
||||
export * from "./types.tts.js";
|
||||
export * from "./types.tools.js";
|
||||
export * from "./types.whatsapp.js";
|
||||
export * from "./types.memory.js";
|
||||
|
||||
@ -16,6 +16,31 @@ const SessionResetConfigSchema = z
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const SessionSendPolicySchema = z
|
||||
.object({
|
||||
default: z.union([z.literal("allow"), z.literal("deny")]).optional(),
|
||||
rules: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
action: z.union([z.literal("allow"), z.literal("deny")]),
|
||||
match: z
|
||||
.object({
|
||||
channel: z.string().optional(),
|
||||
chatType: z
|
||||
.union([z.literal("direct"), z.literal("group"), z.literal("channel")])
|
||||
.optional(),
|
||||
keyPrefix: z.string().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict(),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const SessionSchema = z
|
||||
.object({
|
||||
scope: z.union([z.literal("per-sender"), z.literal("global")]).optional(),
|
||||
@ -51,31 +76,7 @@ export const SessionSchema = z
|
||||
])
|
||||
.optional(),
|
||||
mainKey: z.string().optional(),
|
||||
sendPolicy: z
|
||||
.object({
|
||||
default: z.union([z.literal("allow"), z.literal("deny")]).optional(),
|
||||
rules: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
action: z.union([z.literal("allow"), z.literal("deny")]),
|
||||
match: z
|
||||
.object({
|
||||
channel: z.string().optional(),
|
||||
chatType: z
|
||||
.union([z.literal("direct"), z.literal("group"), z.literal("channel")])
|
||||
.optional(),
|
||||
keyPrefix: z.string().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
})
|
||||
.strict(),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
sendPolicy: SessionSendPolicySchema.optional(),
|
||||
agentToAgent: z
|
||||
.object({
|
||||
maxPingPongTurns: z.number().int().min(0).max(5).optional(),
|
||||
|
||||
@ -5,7 +5,12 @@ import { AgentsSchema, AudioSchema, BindingsSchema, BroadcastSchema } from "./zo
|
||||
import { HexColorSchema, ModelsConfigSchema } from "./zod-schema.core.js";
|
||||
import { HookMappingSchema, HooksGmailSchema, InternalHooksSchema } from "./zod-schema.hooks.js";
|
||||
import { ChannelsSchema } from "./zod-schema.providers.js";
|
||||
import { CommandsSchema, MessagesSchema, SessionSchema } from "./zod-schema.session.js";
|
||||
import {
|
||||
CommandsSchema,
|
||||
MessagesSchema,
|
||||
SessionSchema,
|
||||
SessionSendPolicySchema,
|
||||
} from "./zod-schema.session.js";
|
||||
|
||||
const BrowserSnapshotDefaultsSchema = z
|
||||
.object({
|
||||
@ -27,6 +32,61 @@ const NodeHostSchema = z
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
const MemoryQmdPathSchema = z
|
||||
.object({
|
||||
path: z.string(),
|
||||
name: z.string().optional(),
|
||||
pattern: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MemoryQmdSessionSchema = z
|
||||
.object({
|
||||
enabled: z.boolean().optional(),
|
||||
exportDir: z.string().optional(),
|
||||
retentionDays: z.number().int().nonnegative().optional(),
|
||||
redactToolOutputs: z.boolean().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MemoryQmdUpdateSchema = z
|
||||
.object({
|
||||
interval: z.string().optional(),
|
||||
debounceMs: z.number().int().nonnegative().optional(),
|
||||
onBoot: z.boolean().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MemoryQmdLimitsSchema = z
|
||||
.object({
|
||||
maxResults: z.number().int().positive().optional(),
|
||||
maxSnippetChars: z.number().int().positive().optional(),
|
||||
maxInjectedChars: z.number().int().positive().optional(),
|
||||
timeoutMs: z.number().int().nonnegative().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MemoryQmdSchema = z
|
||||
.object({
|
||||
command: z.string().optional(),
|
||||
includeDefaultMemory: z.boolean().optional(),
|
||||
paths: z.array(MemoryQmdPathSchema).optional(),
|
||||
sessions: MemoryQmdSessionSchema.optional(),
|
||||
update: MemoryQmdUpdateSchema.optional(),
|
||||
limits: MemoryQmdLimitsSchema.optional(),
|
||||
scope: SessionSendPolicySchema.optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MemorySchema = z
|
||||
.object({
|
||||
backend: z.union([z.literal("builtin"), z.literal("qmd")]).optional(),
|
||||
citations: z.union([z.literal("auto"), z.literal("on"), z.literal("off")]).optional(),
|
||||
qmd: MemoryQmdSchema.optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
export const MoltbotSchema = z
|
||||
.object({
|
||||
meta: z
|
||||
@ -445,6 +505,7 @@ export const MoltbotSchema = z
|
||||
})
|
||||
.strict()
|
||||
.optional(),
|
||||
memory: MemorySchema,
|
||||
skills: z
|
||||
.object({
|
||||
allowBundled: z.array(z.string()).optional(),
|
||||
|
||||
58
src/memory/backend-config.test.ts
Normal file
58
src/memory/backend-config.test.ts
Normal file
@ -0,0 +1,58 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import { resolveMemoryBackendConfig } from "./backend-config.js";
|
||||
|
||||
describe("resolveMemoryBackendConfig", () => {
|
||||
it("defaults to builtin backend when config missing", () => {
|
||||
const cfg = { agents: { defaults: { workspace: "/tmp/memory-test" } } } as MoltbotConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(resolved.backend).toBe("builtin");
|
||||
expect(resolved.citations).toBe("auto");
|
||||
expect(resolved.qmd).toBeUndefined();
|
||||
});
|
||||
|
||||
it("resolves qmd backend with default collections", () => {
|
||||
const cfg = {
|
||||
agents: { defaults: { workspace: "/tmp/memory-test" } },
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {},
|
||||
},
|
||||
} as MoltbotConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
expect(resolved.backend).toBe("qmd");
|
||||
expect(resolved.qmd?.collections.length).toBeGreaterThanOrEqual(3);
|
||||
expect(resolved.qmd?.command).toBe("qmd");
|
||||
expect(resolved.qmd?.update.intervalMs).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("resolves custom paths relative to workspace", () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
defaults: { workspace: "/workspace/root" },
|
||||
list: [{ id: "main", workspace: "/workspace/root" }],
|
||||
},
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
paths: [
|
||||
{
|
||||
path: "notes",
|
||||
name: "custom-notes",
|
||||
pattern: "**/*.md",
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
} as MoltbotConfig;
|
||||
const resolved = resolveMemoryBackendConfig({ cfg, agentId: "main" });
|
||||
const custom = resolved.qmd?.collections.find((c) => c.name.startsWith("custom-notes"));
|
||||
expect(custom).toBeDefined();
|
||||
const workspaceRoot = resolveAgentWorkspaceDir(cfg, "main");
|
||||
expect(custom?.path).toBe(path.resolve(workspaceRoot, "notes"));
|
||||
});
|
||||
});
|
||||
245
src/memory/backend-config.ts
Normal file
245
src/memory/backend-config.ts
Normal file
@ -0,0 +1,245 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { parseDurationMs } from "../cli/parse-duration.js";
|
||||
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import type {
|
||||
MemoryBackend,
|
||||
MemoryCitationsMode,
|
||||
MemoryQmdConfig,
|
||||
MemoryQmdIndexPath,
|
||||
} from "../config/types.memory.js";
|
||||
import type { SessionSendPolicyConfig } from "../config/types.base.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
|
||||
export type ResolvedMemoryBackendConfig = {
|
||||
backend: MemoryBackend;
|
||||
citations: MemoryCitationsMode;
|
||||
qmd?: ResolvedQmdConfig;
|
||||
};
|
||||
|
||||
export type ResolvedQmdCollection = {
|
||||
name: string;
|
||||
path: string;
|
||||
pattern: string;
|
||||
kind: "memory" | "custom" | "sessions";
|
||||
};
|
||||
|
||||
export type ResolvedQmdUpdateConfig = {
|
||||
intervalMs: number;
|
||||
debounceMs: number;
|
||||
onBoot: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedQmdLimitsConfig = {
|
||||
maxResults: number;
|
||||
maxSnippetChars: number;
|
||||
maxInjectedChars: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
export type ResolvedQmdSessionConfig = {
|
||||
enabled: boolean;
|
||||
exportDir?: string;
|
||||
retentionDays?: number;
|
||||
redactToolOutputs: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedQmdConfig = {
|
||||
command: string;
|
||||
collections: ResolvedQmdCollection[];
|
||||
sessions: ResolvedQmdSessionConfig;
|
||||
update: ResolvedQmdUpdateConfig;
|
||||
limits: ResolvedQmdLimitsConfig;
|
||||
includeDefaultMemory: boolean;
|
||||
scope?: SessionSendPolicyConfig;
|
||||
};
|
||||
|
||||
const DEFAULT_BACKEND: MemoryBackend = "builtin";
|
||||
const DEFAULT_CITATIONS: MemoryCitationsMode = "auto";
|
||||
const DEFAULT_QMD_INTERVAL = "5m";
|
||||
const DEFAULT_QMD_DEBOUNCE_MS = 15_000;
|
||||
const DEFAULT_QMD_TIMEOUT_MS = 4_000;
|
||||
const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = {
|
||||
maxResults: 6,
|
||||
maxSnippetChars: 700,
|
||||
maxInjectedChars: 4_000,
|
||||
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
|
||||
};
|
||||
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
|
||||
default: "deny",
|
||||
rules: [
|
||||
{
|
||||
action: "allow",
|
||||
match: { chatType: "direct" },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function sanitizeName(input: string): string {
|
||||
const lower = input.toLowerCase().replace(/[^a-z0-9-]+/g, "-");
|
||||
const trimmed = lower.replace(/^-+|-+$/g, "");
|
||||
return trimmed || "collection";
|
||||
}
|
||||
|
||||
function ensureUniqueName(base: string, existing: Set<string>): string {
|
||||
let name = sanitizeName(base);
|
||||
if (!existing.has(name)) {
|
||||
existing.add(name);
|
||||
return name;
|
||||
}
|
||||
let suffix = 2;
|
||||
while (existing.has(`${name}-${suffix}`)) {
|
||||
suffix += 1;
|
||||
}
|
||||
const unique = `${name}-${suffix}`;
|
||||
existing.add(unique);
|
||||
return unique;
|
||||
}
|
||||
|
||||
function resolvePath(raw: string, workspaceDir: string): string {
|
||||
const trimmed = raw.trim();
|
||||
if (!trimmed) throw new Error("path required");
|
||||
if (trimmed.startsWith("~") || path.isAbsolute(trimmed)) {
|
||||
return path.normalize(resolveUserPath(trimmed));
|
||||
}
|
||||
return path.normalize(path.resolve(workspaceDir, trimmed));
|
||||
}
|
||||
|
||||
function resolveIntervalMs(raw: string | undefined): number {
|
||||
const value = raw?.trim();
|
||||
if (!value) return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
|
||||
try {
|
||||
return parseDurationMs(value, { defaultUnit: "m" });
|
||||
} catch {
|
||||
return parseDurationMs(DEFAULT_QMD_INTERVAL, { defaultUnit: "m" });
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDebounceMs(raw: number | undefined): number {
|
||||
if (typeof raw === "number" && Number.isFinite(raw) && raw >= 0) {
|
||||
return Math.floor(raw);
|
||||
}
|
||||
return DEFAULT_QMD_DEBOUNCE_MS;
|
||||
}
|
||||
|
||||
function resolveLimits(raw?: MemoryQmdConfig["limits"]): ResolvedQmdLimitsConfig {
|
||||
const parsed: ResolvedQmdLimitsConfig = { ...DEFAULT_QMD_LIMITS };
|
||||
if (raw?.maxResults && raw.maxResults > 0) parsed.maxResults = Math.floor(raw.maxResults);
|
||||
if (raw?.maxSnippetChars && raw.maxSnippetChars > 0) {
|
||||
parsed.maxSnippetChars = Math.floor(raw.maxSnippetChars);
|
||||
}
|
||||
if (raw?.maxInjectedChars && raw.maxInjectedChars > 0) {
|
||||
parsed.maxInjectedChars = Math.floor(raw.maxInjectedChars);
|
||||
}
|
||||
if (raw?.timeoutMs && raw.timeoutMs > 0) {
|
||||
parsed.timeoutMs = Math.floor(raw.timeoutMs);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function resolveSessionConfig(
|
||||
cfg: MemoryQmdConfig["sessions"],
|
||||
workspaceDir: string,
|
||||
): ResolvedQmdSessionConfig {
|
||||
const enabled = Boolean(cfg?.enabled);
|
||||
const exportDirRaw = cfg?.exportDir?.trim();
|
||||
const exportDir = exportDirRaw ? resolvePath(exportDirRaw, workspaceDir) : undefined;
|
||||
const retentionDays =
|
||||
cfg?.retentionDays && cfg.retentionDays > 0 ? Math.floor(cfg.retentionDays) : undefined;
|
||||
const redactToolOutputs = cfg?.redactToolOutputs !== false;
|
||||
return {
|
||||
enabled,
|
||||
exportDir,
|
||||
retentionDays,
|
||||
redactToolOutputs,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCustomPaths(
|
||||
rawPaths: MemoryQmdIndexPath[] | undefined,
|
||||
workspaceDir: string,
|
||||
existing: Set<string>,
|
||||
): ResolvedQmdCollection[] {
|
||||
if (!rawPaths?.length) return [];
|
||||
const collections: ResolvedQmdCollection[] = [];
|
||||
rawPaths.forEach((entry, index) => {
|
||||
const trimmedPath = entry?.path?.trim();
|
||||
if (!trimmedPath) return;
|
||||
let resolved: string;
|
||||
try {
|
||||
resolved = resolvePath(trimmedPath, workspaceDir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const pattern = entry.pattern?.trim() || "**/*.md";
|
||||
const baseName = entry.name?.trim() || `custom-${index + 1}`;
|
||||
const name = ensureUniqueName(baseName, existing);
|
||||
collections.push({
|
||||
name,
|
||||
path: resolved,
|
||||
pattern,
|
||||
kind: "custom",
|
||||
});
|
||||
});
|
||||
return collections;
|
||||
}
|
||||
|
||||
function resolveDefaultCollections(
|
||||
include: boolean,
|
||||
workspaceDir: string,
|
||||
existing: Set<string>,
|
||||
): ResolvedQmdCollection[] {
|
||||
if (!include) return [];
|
||||
const entries: Array<{ path: string; pattern: string; base: string }> = [
|
||||
{ path: workspaceDir, pattern: "MEMORY.md", base: "memory-root" },
|
||||
{ path: workspaceDir, pattern: "memory.md", base: "memory-alt" },
|
||||
{ path: path.join(workspaceDir, "memory"), pattern: "**/*.md", base: "memory-dir" },
|
||||
];
|
||||
return entries.map((entry) => ({
|
||||
name: ensureUniqueName(entry.base, existing),
|
||||
path: entry.path,
|
||||
pattern: entry.pattern,
|
||||
kind: "memory",
|
||||
}));
|
||||
}
|
||||
|
||||
export function resolveMemoryBackendConfig(params: {
|
||||
cfg: MoltbotConfig;
|
||||
agentId: string;
|
||||
}): ResolvedMemoryBackendConfig {
|
||||
const backend = params.cfg.memory?.backend ?? DEFAULT_BACKEND;
|
||||
const citations = params.cfg.memory?.citations ?? DEFAULT_CITATIONS;
|
||||
if (backend !== "qmd") {
|
||||
return { backend: "builtin", citations };
|
||||
}
|
||||
|
||||
const workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId);
|
||||
const qmdCfg = params.cfg.memory?.qmd;
|
||||
const includeDefaultMemory = qmdCfg?.includeDefaultMemory !== false;
|
||||
const nameSet = new Set<string>();
|
||||
const collections = [
|
||||
...resolveDefaultCollections(includeDefaultMemory, workspaceDir, nameSet),
|
||||
...resolveCustomPaths(qmdCfg?.paths, workspaceDir, nameSet),
|
||||
];
|
||||
|
||||
const resolved: ResolvedQmdConfig = {
|
||||
command: qmdCfg?.command?.trim() || "qmd",
|
||||
collections,
|
||||
includeDefaultMemory,
|
||||
sessions: resolveSessionConfig(qmdCfg?.sessions, workspaceDir),
|
||||
update: {
|
||||
intervalMs: resolveIntervalMs(qmdCfg?.update?.interval),
|
||||
debounceMs: resolveDebounceMs(qmdCfg?.update?.debounceMs),
|
||||
onBoot: qmdCfg?.update?.onBoot !== false,
|
||||
},
|
||||
limits: resolveLimits(qmdCfg?.limits),
|
||||
scope: qmdCfg?.scope ?? DEFAULT_QMD_SCOPE,
|
||||
};
|
||||
|
||||
return {
|
||||
backend: "qmd",
|
||||
citations,
|
||||
qmd: resolved,
|
||||
};
|
||||
}
|
||||
@ -1,2 +1,3 @@
|
||||
export type { MemoryIndexManager, MemorySearchResult } from "./manager.js";
|
||||
export { MemoryIndexManager } from "./manager.js";
|
||||
export type { MemorySearchResult, MemorySearchManager } from "./types.js";
|
||||
export { getMemorySearchManager, type MemorySearchManagerResult } from "./search-manager.js";
|
||||
|
||||
@ -45,17 +45,13 @@ import { searchKeyword, searchVector } from "./manager-search.js";
|
||||
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
||||
import { requireNodeSqlite } from "./sqlite.js";
|
||||
import { loadSqliteVecExtension } from "./sqlite-vec.js";
|
||||
|
||||
type MemorySource = "memory" | "sessions";
|
||||
|
||||
export type MemorySearchResult = {
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: MemorySource;
|
||||
};
|
||||
import type {
|
||||
MemoryProviderStatus,
|
||||
MemorySearchManager,
|
||||
MemorySearchResult,
|
||||
MemorySource,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "./types.js";
|
||||
|
||||
type MemoryIndexMeta = {
|
||||
model: string;
|
||||
@ -75,12 +71,6 @@ type SessionFileEntry = {
|
||||
content: string;
|
||||
};
|
||||
|
||||
type MemorySyncProgressUpdate = {
|
||||
completed: number;
|
||||
total: number;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
type MemorySyncProgressState = {
|
||||
completed: number;
|
||||
total: number;
|
||||
@ -115,7 +105,7 @@ const INDEX_CACHE = new Map<string, MemoryIndexManager>();
|
||||
const vectorToBlob = (embedding: number[]): Buffer =>
|
||||
Buffer.from(new Float32Array(embedding).buffer);
|
||||
|
||||
export class MemoryIndexManager {
|
||||
export class MemoryIndexManager implements MemorySearchManager {
|
||||
private readonly cacheKey: string;
|
||||
private readonly cfg: MoltbotConfig;
|
||||
private readonly agentId: string;
|
||||
@ -415,39 +405,7 @@ export class MemoryIndexManager {
|
||||
return { text: slice.join("\n"), path: relPath };
|
||||
}
|
||||
|
||||
status(): {
|
||||
files: number;
|
||||
chunks: number;
|
||||
dirty: boolean;
|
||||
workspaceDir: string;
|
||||
dbPath: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
requestedProvider: string;
|
||||
sources: MemorySource[];
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
|
||||
fts?: { enabled: boolean; available: boolean; error?: string };
|
||||
fallback?: { from: string; reason?: string };
|
||||
vector?: {
|
||||
enabled: boolean;
|
||||
available?: boolean;
|
||||
extensionPath?: string;
|
||||
loadError?: string;
|
||||
dims?: number;
|
||||
};
|
||||
batch?: {
|
||||
enabled: boolean;
|
||||
failures: number;
|
||||
limit: number;
|
||||
wait: boolean;
|
||||
concurrency: number;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
lastError?: string;
|
||||
lastProvider?: string;
|
||||
};
|
||||
} {
|
||||
status(): MemoryProviderStatus {
|
||||
const sourceFilter = this.buildSourceFilter();
|
||||
const files = this.db
|
||||
.prepare(`SELECT COUNT(*) as c FROM files WHERE 1=1${sourceFilter.sql}`)
|
||||
@ -489,9 +447,10 @@ export class MemoryIndexManager {
|
||||
return sources.map((source) => ({ source, ...bySource.get(source)! }));
|
||||
})();
|
||||
return {
|
||||
backend: "builtin",
|
||||
files: files?.c ?? 0,
|
||||
chunks: chunks?.c ?? 0,
|
||||
dirty: this.dirty,
|
||||
dirty: this.dirty || this.sessionsDirty,
|
||||
workspaceDir: this.workspaceDir,
|
||||
dbPath: this.settings.store.path,
|
||||
provider: this.provider.id,
|
||||
|
||||
612
src/memory/qmd-manager.ts
Normal file
612
src/memory/qmd-manager.ts
Normal file
@ -0,0 +1,612 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import YAML from "yaml";
|
||||
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
listSessionFilesForAgent,
|
||||
buildSessionEntry,
|
||||
type SessionFileEntry,
|
||||
} from "./session-files.js";
|
||||
import { requireNodeSqlite } from "./sqlite.js";
|
||||
import type {
|
||||
MemoryProviderStatus,
|
||||
MemorySearchManager,
|
||||
MemorySearchResult,
|
||||
MemorySource,
|
||||
MemorySyncProgressUpdate,
|
||||
} from "./types.js";
|
||||
import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
|
||||
const SNIPPET_HEADER_RE = /@@\s*-([0-9]+),([0-9]+)/;
|
||||
|
||||
type QmdQueryResult = {
|
||||
docid?: string;
|
||||
score?: number;
|
||||
file?: string;
|
||||
snippet?: string;
|
||||
body?: string;
|
||||
};
|
||||
|
||||
type CollectionRoot = {
|
||||
path: string;
|
||||
kind: MemorySource;
|
||||
};
|
||||
|
||||
type SessionExporterConfig = {
|
||||
dir: string;
|
||||
retentionMs?: number;
|
||||
collectionName: string;
|
||||
};
|
||||
|
||||
export class QmdMemoryManager implements MemorySearchManager {
|
||||
static async create(params: {
|
||||
cfg: MoltbotConfig;
|
||||
agentId: string;
|
||||
resolved: ResolvedMemoryBackendConfig;
|
||||
}): Promise<QmdMemoryManager | null> {
|
||||
const resolved = params.resolved.qmd;
|
||||
if (!resolved) return null;
|
||||
const manager = new QmdMemoryManager({ cfg: params.cfg, agentId: params.agentId, resolved });
|
||||
await manager.initialize();
|
||||
return manager;
|
||||
}
|
||||
|
||||
private readonly cfg: MoltbotConfig;
|
||||
private readonly agentId: string;
|
||||
private readonly qmd: ResolvedQmdConfig;
|
||||
private readonly workspaceDir: string;
|
||||
private readonly stateDir: string;
|
||||
private readonly agentStateDir: string;
|
||||
private readonly qmdDir: string;
|
||||
private readonly cacheDir: string;
|
||||
private readonly configDir: string;
|
||||
private readonly xdgConfigHome: string;
|
||||
private readonly xdgCacheHome: string;
|
||||
private readonly collectionsFile: string;
|
||||
private readonly indexPath: string;
|
||||
private readonly env: NodeJS.ProcessEnv;
|
||||
private readonly collectionRoots = new Map<string, CollectionRoot>();
|
||||
private readonly sources = new Set<MemorySource>();
|
||||
private readonly docPathCache = new Map<
|
||||
string,
|
||||
{ rel: string; abs: string; source: MemorySource }
|
||||
>();
|
||||
private readonly sessionExporter: SessionExporterConfig | null;
|
||||
private updateTimer: NodeJS.Timeout | null = null;
|
||||
private pendingUpdate: Promise<void> | null = null;
|
||||
private closed = false;
|
||||
private db: import("node:sqlite").DatabaseSync | null = null;
|
||||
private lastUpdateAt: number | null = null;
|
||||
|
||||
private constructor(params: {
|
||||
cfg: MoltbotConfig;
|
||||
agentId: string;
|
||||
resolved: ResolvedQmdConfig;
|
||||
}) {
|
||||
this.cfg = params.cfg;
|
||||
this.agentId = params.agentId;
|
||||
this.qmd = params.resolved;
|
||||
this.workspaceDir = resolveAgentWorkspaceDir(params.cfg, params.agentId);
|
||||
this.stateDir = resolveStateDir(process.env, os.homedir);
|
||||
this.agentStateDir = path.join(this.stateDir, "agents", this.agentId);
|
||||
this.qmdDir = path.join(this.agentStateDir, "qmd");
|
||||
this.cacheDir = path.join(this.qmdDir, "cache");
|
||||
this.configDir = path.join(this.qmdDir, "config");
|
||||
this.xdgConfigHome = path.join(this.qmdDir, "xdg-config");
|
||||
this.xdgCacheHome = path.join(this.qmdDir, "xdg-cache");
|
||||
this.collectionsFile = path.join(this.configDir, "index.yml");
|
||||
this.indexPath = path.join(this.cacheDir, "index.sqlite");
|
||||
this.env = {
|
||||
...process.env,
|
||||
QMD_CONFIG_DIR: this.configDir,
|
||||
XDG_CONFIG_HOME: this.xdgConfigHome,
|
||||
XDG_CACHE_HOME: this.xdgCacheHome,
|
||||
INDEX_PATH: this.indexPath,
|
||||
NO_COLOR: "1",
|
||||
};
|
||||
this.sessionExporter = this.qmd.sessions.enabled
|
||||
? {
|
||||
dir: this.qmd.sessions.exportDir ?? path.join(this.qmdDir, "sessions"),
|
||||
retentionMs: this.qmd.sessions.retentionDays
|
||||
? this.qmd.sessions.retentionDays * 24 * 60 * 60 * 1000
|
||||
: undefined,
|
||||
collectionName: this.pickSessionCollectionName(),
|
||||
}
|
||||
: null;
|
||||
if (this.sessionExporter) {
|
||||
this.qmd.collections = [
|
||||
...this.qmd.collections,
|
||||
{
|
||||
name: this.sessionExporter.collectionName,
|
||||
path: this.sessionExporter.dir,
|
||||
pattern: "**/*.md",
|
||||
kind: "sessions",
|
||||
},
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
private async initialize(): Promise<void> {
|
||||
await fs.mkdir(this.cacheDir, { recursive: true });
|
||||
await fs.mkdir(this.configDir, { recursive: true });
|
||||
await fs.mkdir(this.xdgConfigHome, { recursive: true });
|
||||
await fs.mkdir(this.xdgCacheHome, { recursive: true });
|
||||
|
||||
this.bootstrapCollections();
|
||||
await this.writeCollectionsConfig();
|
||||
|
||||
if (this.qmd.update.onBoot) {
|
||||
await this.runUpdate("boot", true);
|
||||
}
|
||||
if (this.qmd.update.intervalMs > 0) {
|
||||
this.updateTimer = setInterval(() => {
|
||||
void this.runUpdate("interval").catch((err) => {
|
||||
log.warn(`qmd update failed (${String(err)})`);
|
||||
});
|
||||
}, this.qmd.update.intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
private bootstrapCollections(): void {
|
||||
this.collectionRoots.clear();
|
||||
this.sources.clear();
|
||||
for (const collection of this.qmd.collections) {
|
||||
const kind: MemorySource = collection.kind === "sessions" ? "sessions" : "memory";
|
||||
this.collectionRoots.set(collection.name, { path: collection.path, kind });
|
||||
this.sources.add(kind);
|
||||
}
|
||||
}
|
||||
|
||||
private async writeCollectionsConfig(): Promise<void> {
|
||||
const collections: Record<string, { path: string; pattern: string }> = {};
|
||||
for (const collection of this.qmd.collections) {
|
||||
collections[collection.name] = {
|
||||
path: collection.path,
|
||||
pattern: collection.pattern,
|
||||
};
|
||||
}
|
||||
const yaml = YAML.stringify({ collections }, { indent: 2, lineWidth: 0 });
|
||||
await fs.writeFile(this.collectionsFile, yaml, "utf-8");
|
||||
}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
|
||||
): Promise<MemorySearchResult[]> {
|
||||
if (!this.isScopeAllowed(opts?.sessionKey)) return [];
|
||||
const trimmed = query.trim();
|
||||
if (!trimmed) return [];
|
||||
await this.pendingUpdate?.catch(() => undefined);
|
||||
const limit = Math.min(
|
||||
this.qmd.limits.maxResults,
|
||||
opts?.maxResults ?? this.qmd.limits.maxResults,
|
||||
);
|
||||
const args = ["query", trimmed, "--json", "-n", String(limit)];
|
||||
let stdout: string;
|
||||
try {
|
||||
const result = await this.runQmd(args, { timeoutMs: this.qmd.limits.timeoutMs });
|
||||
stdout = result.stdout;
|
||||
} catch (err) {
|
||||
log.warn(`qmd query failed: ${String(err)}`);
|
||||
throw err instanceof Error ? err : new Error(String(err));
|
||||
}
|
||||
let parsed: QmdQueryResult[] = [];
|
||||
try {
|
||||
parsed = JSON.parse(stdout);
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`qmd query returned invalid JSON: ${message}`);
|
||||
throw new Error(`qmd query returned invalid JSON: ${message}`);
|
||||
}
|
||||
const results: MemorySearchResult[] = [];
|
||||
for (const entry of parsed) {
|
||||
const doc = await this.resolveDocLocation(entry.docid);
|
||||
if (!doc) continue;
|
||||
const snippet = entry.snippet?.slice(0, this.qmd.limits.maxSnippetChars) ?? "";
|
||||
const lines = this.extractSnippetLines(snippet);
|
||||
const score = typeof entry.score === "number" ? entry.score : 0;
|
||||
const minScore = opts?.minScore ?? 0;
|
||||
if (score < minScore) continue;
|
||||
results.push({
|
||||
path: doc.rel,
|
||||
startLine: lines.startLine,
|
||||
endLine: lines.endLine,
|
||||
score,
|
||||
snippet,
|
||||
source: doc.source,
|
||||
});
|
||||
}
|
||||
return results.slice(0, limit);
|
||||
}
|
||||
|
||||
async sync(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}): Promise<void> {
|
||||
if (params?.progress) {
|
||||
params.progress({ completed: 0, total: 1, label: "Updating QMD index…" });
|
||||
}
|
||||
await this.runUpdate(params?.reason ?? "manual", params?.force);
|
||||
if (params?.progress) {
|
||||
params.progress({ completed: 1, total: 1, label: "QMD index updated" });
|
||||
}
|
||||
}
|
||||
|
||||
async readFile(params: {
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }> {
|
||||
const relPath = params.relPath?.trim();
|
||||
if (!relPath) throw new Error("path required");
|
||||
const absPath = this.resolveReadPath(relPath);
|
||||
const content = await fs.readFile(absPath, "utf-8");
|
||||
if (!params.from && !params.lines) {
|
||||
return { text: content, path: relPath };
|
||||
}
|
||||
const lines = content.split("\n");
|
||||
const start = Math.max(1, params.from ?? 1);
|
||||
const count = Math.max(1, params.lines ?? lines.length);
|
||||
const slice = lines.slice(start - 1, start - 1 + count);
|
||||
return { text: slice.join("\n"), path: relPath };
|
||||
}
|
||||
|
||||
status(): MemoryProviderStatus {
|
||||
const counts = this.readCounts();
|
||||
return {
|
||||
backend: "qmd",
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
files: counts.totalDocuments,
|
||||
chunks: counts.totalDocuments,
|
||||
dirty: false,
|
||||
workspaceDir: this.workspaceDir,
|
||||
dbPath: this.indexPath,
|
||||
sources: Array.from(this.sources),
|
||||
sourceCounts: counts.sourceCounts,
|
||||
vector: { enabled: true, available: true },
|
||||
batch: {
|
||||
enabled: false,
|
||||
failures: 0,
|
||||
limit: 0,
|
||||
wait: false,
|
||||
concurrency: 0,
|
||||
pollIntervalMs: 0,
|
||||
timeoutMs: 0,
|
||||
},
|
||||
custom: {
|
||||
qmd: {
|
||||
collections: this.qmd.collections.length,
|
||||
lastUpdateAt: this.lastUpdateAt,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async probeVectorAvailability(): Promise<boolean> {
|
||||
return true;
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
if (this.closed) return;
|
||||
this.closed = true;
|
||||
if (this.updateTimer) {
|
||||
clearInterval(this.updateTimer);
|
||||
this.updateTimer = null;
|
||||
}
|
||||
await this.pendingUpdate?.catch(() => undefined);
|
||||
if (this.db) {
|
||||
this.db.close();
|
||||
this.db = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async runUpdate(reason: string, force?: boolean): Promise<void> {
|
||||
if (this.pendingUpdate && !force) return this.pendingUpdate;
|
||||
const run = async () => {
|
||||
if (this.sessionExporter) {
|
||||
await this.exportSessions();
|
||||
}
|
||||
await this.runQmd(["update"], { timeoutMs: 120_000 });
|
||||
try {
|
||||
await this.runQmd(["embed"], { timeoutMs: 120_000 });
|
||||
} catch (err) {
|
||||
log.warn(`qmd embed failed (${reason}): ${String(err)}`);
|
||||
}
|
||||
this.lastUpdateAt = Date.now();
|
||||
this.docPathCache.clear();
|
||||
};
|
||||
this.pendingUpdate = run().finally(() => {
|
||||
this.pendingUpdate = null;
|
||||
});
|
||||
await this.pendingUpdate;
|
||||
}
|
||||
|
||||
private async runQmd(
|
||||
args: string[],
|
||||
opts?: { timeoutMs?: number },
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const child = spawn(this.qmd.command, args, {
|
||||
env: this.env,
|
||||
cwd: this.workspaceDir,
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const timer = opts?.timeoutMs
|
||||
? setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
reject(new Error(`qmd ${args.join(" ")} timed out after ${opts.timeoutMs}ms`));
|
||||
}, opts.timeoutMs)
|
||||
: null;
|
||||
child.stdout.on("data", (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
child.on("error", (err) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
reject(err);
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (code === 0) {
|
||||
resolve({ stdout, stderr });
|
||||
} else {
|
||||
reject(new Error(`qmd ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private ensureDb() {
|
||||
if (this.db) return this.db;
|
||||
const sqlite = requireNodeSqlite();
|
||||
this.db = sqlite.open(this.indexPath, { readonly: true });
|
||||
return this.db;
|
||||
}
|
||||
|
||||
private async exportSessions(): Promise<void> {
|
||||
if (!this.sessionExporter) return;
|
||||
const exportDir = this.sessionExporter.dir;
|
||||
await fs.mkdir(exportDir, { recursive: true });
|
||||
const files = await listSessionFilesForAgent(this.agentId);
|
||||
const keep = new Set<string>();
|
||||
const cutoff = this.sessionExporter.retentionMs
|
||||
? Date.now() - this.sessionExporter.retentionMs
|
||||
: null;
|
||||
for (const sessionFile of files) {
|
||||
const entry = await buildSessionEntry(sessionFile);
|
||||
if (!entry) continue;
|
||||
if (cutoff && entry.mtimeMs < cutoff) continue;
|
||||
const target = path.join(exportDir, `${path.basename(sessionFile, ".jsonl")}.md`);
|
||||
await fs.writeFile(target, this.renderSessionMarkdown(entry), "utf-8");
|
||||
keep.add(target);
|
||||
}
|
||||
const exported = await fs.readdir(exportDir).catch(() => []);
|
||||
for (const name of exported) {
|
||||
if (!name.endsWith(".md")) continue;
|
||||
const full = path.join(exportDir, name);
|
||||
if (!keep.has(full)) {
|
||||
await fs.rm(full, { force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private renderSessionMarkdown(entry: SessionFileEntry): string {
|
||||
const header = `# Session ${path.basename(entry.absPath, path.extname(entry.absPath))}`;
|
||||
const body = entry.content?.trim().length ? entry.content.trim() : "(empty)";
|
||||
return `${header}\n\n${body}\n`;
|
||||
}
|
||||
|
||||
private pickSessionCollectionName(): string {
|
||||
const existing = new Set(this.qmd.collections.map((collection) => collection.name));
|
||||
if (!existing.has("sessions")) return "sessions";
|
||||
let counter = 2;
|
||||
let candidate = `sessions-${counter}`;
|
||||
while (existing.has(candidate)) {
|
||||
counter += 1;
|
||||
candidate = `sessions-${counter}`;
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
|
||||
private async resolveDocLocation(
|
||||
docid?: string,
|
||||
): Promise<{ rel: string; abs: string; source: MemorySource } | null> {
|
||||
if (!docid) return null;
|
||||
const normalized = docid.startsWith("#") ? docid.slice(1) : docid;
|
||||
if (!normalized) return null;
|
||||
const cached = this.docPathCache.get(normalized);
|
||||
if (cached) return cached;
|
||||
const db = this.ensureDb();
|
||||
const row = db
|
||||
.prepare("SELECT collection, path FROM documents WHERE hash LIKE ? AND active = 1 LIMIT 1")
|
||||
.get(`${normalized}%`) as { collection: string; path: string } | undefined;
|
||||
if (!row) return null;
|
||||
const location = this.toDocLocation(row.collection, row.path);
|
||||
if (!location) return null;
|
||||
this.docPathCache.set(normalized, location);
|
||||
return location;
|
||||
}
|
||||
|
||||
private extractSnippetLines(snippet: string): { startLine: number; endLine: number } {
|
||||
const match = SNIPPET_HEADER_RE.exec(snippet);
|
||||
if (match) {
|
||||
const start = Number(match[1]);
|
||||
const count = Number(match[2]);
|
||||
if (Number.isFinite(start) && Number.isFinite(count)) {
|
||||
return { startLine: start, endLine: start + count - 1 };
|
||||
}
|
||||
}
|
||||
const lines = snippet.split("\n").length;
|
||||
return { startLine: 1, endLine: lines };
|
||||
}
|
||||
|
||||
private readCounts(): {
|
||||
totalDocuments: number;
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
} {
|
||||
try {
|
||||
const db = this.ensureDb();
|
||||
const rows = db
|
||||
.prepare(
|
||||
"SELECT collection, COUNT(*) as c FROM documents WHERE active = 1 GROUP BY collection",
|
||||
)
|
||||
.all() as Array<{ collection: string; c: number }>;
|
||||
const bySource = new Map<MemorySource, { files: number; chunks: number }>();
|
||||
for (const source of this.sources) {
|
||||
bySource.set(source, { files: 0, chunks: 0 });
|
||||
}
|
||||
let total = 0;
|
||||
for (const row of rows) {
|
||||
const root = this.collectionRoots.get(row.collection);
|
||||
const source = root?.kind ?? "memory";
|
||||
const entry = bySource.get(source) ?? { files: 0, chunks: 0 };
|
||||
entry.files += row.c ?? 0;
|
||||
entry.chunks += row.c ?? 0;
|
||||
bySource.set(source, entry);
|
||||
total += row.c ?? 0;
|
||||
}
|
||||
return {
|
||||
totalDocuments: total,
|
||||
sourceCounts: Array.from(bySource.entries()).map(([source, value]) => ({
|
||||
source,
|
||||
files: value.files,
|
||||
chunks: value.chunks,
|
||||
})),
|
||||
};
|
||||
} catch (err) {
|
||||
log.warn(`failed to read qmd index stats: ${String(err)}`);
|
||||
return {
|
||||
totalDocuments: 0,
|
||||
sourceCounts: Array.from(this.sources).map((source) => ({ source, files: 0, chunks: 0 })),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private isScopeAllowed(sessionKey?: string): boolean {
|
||||
const scope = this.qmd.scope;
|
||||
if (!scope) return true;
|
||||
const channel = this.deriveChannelFromKey(sessionKey);
|
||||
const chatType = this.deriveChatTypeFromKey(sessionKey);
|
||||
const normalizedKey = sessionKey ?? "";
|
||||
for (const rule of scope.rules ?? []) {
|
||||
if (!rule) continue;
|
||||
const match = rule.match ?? {};
|
||||
if (match.channel && match.channel !== channel) continue;
|
||||
if (match.chatType && match.chatType !== chatType) continue;
|
||||
if (match.keyPrefix && !normalizedKey.startsWith(match.keyPrefix)) continue;
|
||||
return rule.action === "allow";
|
||||
}
|
||||
const fallback = scope.default ?? "allow";
|
||||
return fallback === "allow";
|
||||
}
|
||||
|
||||
private deriveChannelFromKey(key?: string) {
|
||||
if (!key) return undefined;
|
||||
const parts = key.split(":").filter(Boolean);
|
||||
if (parts.length >= 3 && (parts[1] === "group" || parts[1] === "channel")) {
|
||||
return parts[0]?.toLowerCase();
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
private deriveChatTypeFromKey(key?: string) {
|
||||
if (!key) return undefined;
|
||||
if (key.includes(":group:")) return "group";
|
||||
if (key.includes(":channel:")) return "channel";
|
||||
return "direct";
|
||||
}
|
||||
|
||||
private toDocLocation(
|
||||
collection: string,
|
||||
collectionRelativePath: string,
|
||||
): { rel: string; abs: string; source: MemorySource } | null {
|
||||
const root = this.collectionRoots.get(collection);
|
||||
if (!root) return null;
|
||||
const normalizedRelative = collectionRelativePath.replace(/\\/g, "/");
|
||||
const absPath = path.normalize(path.resolve(root.path, collectionRelativePath));
|
||||
const relativeToWorkspace = path.relative(this.workspaceDir, absPath);
|
||||
const relPath = this.buildSearchPath(
|
||||
collection,
|
||||
normalizedRelative,
|
||||
relativeToWorkspace,
|
||||
absPath,
|
||||
);
|
||||
return { rel: relPath, abs: absPath, source: root.kind };
|
||||
}
|
||||
|
||||
private buildSearchPath(
|
||||
collection: string,
|
||||
collectionRelativePath: string,
|
||||
relativeToWorkspace: string,
|
||||
absPath: string,
|
||||
): string {
|
||||
const insideWorkspace = this.isInsideWorkspace(relativeToWorkspace);
|
||||
if (insideWorkspace) {
|
||||
const normalized = relativeToWorkspace.replace(/\\/g, "/");
|
||||
if (!normalized) return path.basename(absPath);
|
||||
return normalized;
|
||||
}
|
||||
const sanitized = collectionRelativePath.replace(/^\/+/, "");
|
||||
return `qmd/${collection}/${sanitized}`;
|
||||
}
|
||||
|
||||
private isInsideWorkspace(relativePath: string): boolean {
|
||||
if (!relativePath) return true;
|
||||
if (relativePath.startsWith("..")) return false;
|
||||
if (relativePath.startsWith(`..${path.sep}`)) return false;
|
||||
return !path.isAbsolute(relativePath);
|
||||
}
|
||||
|
||||
private resolveReadPath(relPath: string): string {
|
||||
if (relPath.startsWith("qmd/")) {
|
||||
const [, collection, ...rest] = relPath.split("/");
|
||||
if (!collection || rest.length === 0) {
|
||||
throw new Error("invalid qmd path");
|
||||
}
|
||||
const root = this.collectionRoots.get(collection);
|
||||
if (!root) throw new Error(`unknown qmd collection: ${collection}`);
|
||||
const joined = rest.join("/");
|
||||
const resolved = path.resolve(root.path, joined);
|
||||
if (!this.isWithinRoot(root.path, resolved)) {
|
||||
throw new Error("qmd path escapes collection");
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
const absPath = path.resolve(this.workspaceDir, relPath);
|
||||
if (!this.isWithinWorkspace(absPath)) {
|
||||
throw new Error("path escapes workspace");
|
||||
}
|
||||
return absPath;
|
||||
}
|
||||
|
||||
private isWithinWorkspace(absPath: string): boolean {
|
||||
const normalizedWorkspace = this.workspaceDir.endsWith(path.sep)
|
||||
? this.workspaceDir
|
||||
: `${this.workspaceDir}${path.sep}`;
|
||||
if (absPath === this.workspaceDir) return true;
|
||||
const candidate = absPath.endsWith(path.sep) ? absPath : `${absPath}${path.sep}`;
|
||||
return candidate.startsWith(normalizedWorkspace);
|
||||
}
|
||||
|
||||
private isWithinRoot(root: string, candidate: string): boolean {
|
||||
const normalizedRoot = root.endsWith(path.sep) ? root : `${root}${path.sep}`;
|
||||
if (candidate === root) return true;
|
||||
const next = candidate.endsWith(path.sep) ? candidate : `${candidate}${path.sep}`;
|
||||
return next.startsWith(normalizedRoot);
|
||||
}
|
||||
}
|
||||
62
src/memory/search-manager.test.ts
Normal file
62
src/memory/search-manager.test.ts
Normal file
@ -0,0 +1,62 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mockPrimary = {
|
||||
search: vi.fn(async () => []),
|
||||
readFile: vi.fn(async () => ({ text: "", path: "MEMORY.md" })),
|
||||
status: vi.fn(() => ({
|
||||
backend: "qmd" as const,
|
||||
provider: "qmd",
|
||||
model: "qmd",
|
||||
requestedProvider: "qmd",
|
||||
files: 0,
|
||||
chunks: 0,
|
||||
dirty: false,
|
||||
workspaceDir: "/tmp",
|
||||
dbPath: "/tmp/index.sqlite",
|
||||
sources: ["memory" as const],
|
||||
sourceCounts: [{ source: "memory" as const, files: 0, chunks: 0 }],
|
||||
})),
|
||||
sync: vi.fn(async () => {}),
|
||||
probeVectorAvailability: vi.fn(async () => true),
|
||||
close: vi.fn(async () => {}),
|
||||
};
|
||||
|
||||
vi.mock("./qmd-manager.js", () => ({
|
||||
QmdMemoryManager: {
|
||||
create: vi.fn(async () => mockPrimary),
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("./manager.js", () => ({
|
||||
MemoryIndexManager: {
|
||||
get: vi.fn(async () => null),
|
||||
},
|
||||
}));
|
||||
|
||||
import { QmdMemoryManager } from "./qmd-manager.js";
|
||||
import { getMemorySearchManager } from "./search-manager.js";
|
||||
|
||||
beforeEach(() => {
|
||||
mockPrimary.search.mockClear();
|
||||
mockPrimary.readFile.mockClear();
|
||||
mockPrimary.status.mockClear();
|
||||
mockPrimary.sync.mockClear();
|
||||
mockPrimary.probeVectorAvailability.mockClear();
|
||||
mockPrimary.close.mockClear();
|
||||
QmdMemoryManager.create.mockClear();
|
||||
});
|
||||
|
||||
describe("getMemorySearchManager caching", () => {
|
||||
it("reuses the same QMD manager instance for repeated calls", async () => {
|
||||
const cfg = {
|
||||
memory: { backend: "qmd", qmd: {} },
|
||||
agents: { list: [{ id: "main", default: true, workspace: "/tmp/workspace" }] },
|
||||
} as const;
|
||||
|
||||
const first = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
const second = await getMemorySearchManager({ cfg, agentId: "main" });
|
||||
|
||||
expect(first.manager).toBe(second.manager);
|
||||
expect(QmdMemoryManager.create).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@ -1,8 +1,15 @@
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import type { MoltbotConfig } from "../config/config.js";
|
||||
import { resolveMemoryBackendConfig } from "./backend-config.js";
|
||||
import type { ResolvedQmdConfig } from "./backend-config.js";
|
||||
import type { MemoryIndexManager } from "./manager.js";
|
||||
import type { MemorySearchManager, MemorySyncProgressUpdate } from "./types.js";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
const QMD_MANAGER_CACHE = new Map<string, MemorySearchManager>();
|
||||
|
||||
export type MemorySearchManagerResult = {
|
||||
manager: MemoryIndexManager | null;
|
||||
manager: MemorySearchManager | null;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
@ -10,6 +17,40 @@ export async function getMemorySearchManager(params: {
|
||||
cfg: MoltbotConfig;
|
||||
agentId: string;
|
||||
}): Promise<MemorySearchManagerResult> {
|
||||
const resolved = resolveMemoryBackendConfig(params);
|
||||
if (resolved.backend === "qmd" && resolved.qmd) {
|
||||
const cacheKey = buildQmdCacheKey(params.agentId, resolved.qmd);
|
||||
const cached = QMD_MANAGER_CACHE.get(cacheKey);
|
||||
if (cached) {
|
||||
return { manager: cached };
|
||||
}
|
||||
try {
|
||||
const { QmdMemoryManager } = await import("./qmd-manager.js");
|
||||
const primary = await QmdMemoryManager.create({
|
||||
cfg: params.cfg,
|
||||
agentId: params.agentId,
|
||||
resolved,
|
||||
});
|
||||
if (primary) {
|
||||
const wrapper = new FallbackMemoryManager(
|
||||
{
|
||||
primary,
|
||||
fallbackFactory: async () => {
|
||||
const { MemoryIndexManager } = await import("./manager.js");
|
||||
return await MemoryIndexManager.get(params);
|
||||
},
|
||||
},
|
||||
() => QMD_MANAGER_CACHE.delete(cacheKey),
|
||||
);
|
||||
QMD_MANAGER_CACHE.set(cacheKey, wrapper);
|
||||
return { manager: wrapper };
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`qmd memory unavailable; falling back to builtin: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const { MemoryIndexManager } = await import("./manager.js");
|
||||
const manager = await MemoryIndexManager.get(params);
|
||||
@ -19,3 +60,117 @@ export async function getMemorySearchManager(params: {
|
||||
return { manager: null, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
class FallbackMemoryManager implements MemorySearchManager {
|
||||
private fallback: MemorySearchManager | null = null;
|
||||
private primaryFailed = false;
|
||||
private lastError?: string;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
primary: MemorySearchManager;
|
||||
fallbackFactory: () => Promise<MemorySearchManager | null>;
|
||||
},
|
||||
private readonly onClose?: () => void,
|
||||
) {}
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
|
||||
) {
|
||||
if (!this.primaryFailed) {
|
||||
try {
|
||||
return await this.deps.primary.search(query, opts);
|
||||
} catch (err) {
|
||||
this.primaryFailed = true;
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
log.warn(`qmd memory failed; switching to builtin index: ${this.lastError}`);
|
||||
await this.deps.primary.close?.().catch(() => {});
|
||||
}
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
if (fallback) {
|
||||
return await fallback.search(query, opts);
|
||||
}
|
||||
throw new Error(this.lastError ?? "memory search unavailable");
|
||||
}
|
||||
|
||||
async readFile(params: { relPath: string; from?: number; lines?: number }) {
|
||||
if (!this.primaryFailed) {
|
||||
return await this.deps.primary.readFile(params);
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
if (fallback) {
|
||||
return await fallback.readFile(params);
|
||||
}
|
||||
throw new Error(this.lastError ?? "memory read unavailable");
|
||||
}
|
||||
|
||||
status() {
|
||||
if (!this.primaryFailed) {
|
||||
return this.deps.primary.status();
|
||||
}
|
||||
const fallbackStatus = this.fallback?.status();
|
||||
if (fallbackStatus) {
|
||||
const custom = fallbackStatus.custom ?? {};
|
||||
return {
|
||||
...fallbackStatus,
|
||||
custom: {
|
||||
...custom,
|
||||
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
|
||||
},
|
||||
};
|
||||
}
|
||||
const primaryStatus = this.deps.primary.status();
|
||||
const custom = primaryStatus.custom ?? {};
|
||||
return {
|
||||
...primaryStatus,
|
||||
custom: {
|
||||
...custom,
|
||||
fallback: { disabled: true, reason: this.lastError ?? "unknown" },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async sync(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}) {
|
||||
if (!this.primaryFailed) {
|
||||
await this.deps.primary.sync?.(params);
|
||||
return;
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
await fallback?.sync?.(params);
|
||||
}
|
||||
|
||||
async probeVectorAvailability() {
|
||||
if (!this.primaryFailed) {
|
||||
return await this.deps.primary.probeVectorAvailability();
|
||||
}
|
||||
const fallback = await this.ensureFallback();
|
||||
return (await fallback?.probeVectorAvailability()) ?? false;
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.deps.primary.close?.();
|
||||
await this.fallback?.close?.();
|
||||
this.onClose?.();
|
||||
}
|
||||
|
||||
private async ensureFallback(): Promise<MemorySearchManager | null> {
|
||||
if (this.fallback) return this.fallback;
|
||||
const fallback = await this.deps.fallbackFactory();
|
||||
if (!fallback) {
|
||||
log.warn("memory fallback requested but builtin index is unavailable");
|
||||
return null;
|
||||
}
|
||||
this.fallback = fallback;
|
||||
return this.fallback;
|
||||
}
|
||||
}
|
||||
|
||||
function buildQmdCacheKey(agentId: string, config: ResolvedQmdConfig): string {
|
||||
return `${agentId}:${JSON.stringify(config)}`;
|
||||
}
|
||||
|
||||
72
src/memory/types.ts
Normal file
72
src/memory/types.ts
Normal file
@ -0,0 +1,72 @@
|
||||
export type MemorySource = "memory" | "sessions";
|
||||
|
||||
export type MemorySearchResult = {
|
||||
path: string;
|
||||
startLine: number;
|
||||
endLine: number;
|
||||
score: number;
|
||||
snippet: string;
|
||||
source: MemorySource;
|
||||
citation?: string;
|
||||
};
|
||||
|
||||
export type MemorySyncProgressUpdate = {
|
||||
completed: number;
|
||||
total: number;
|
||||
label?: string;
|
||||
};
|
||||
|
||||
export type MemoryProviderStatus = {
|
||||
backend: "builtin" | "qmd";
|
||||
provider: string;
|
||||
model?: string;
|
||||
requestedProvider?: string;
|
||||
files?: number;
|
||||
chunks?: number;
|
||||
dirty?: boolean;
|
||||
workspaceDir?: string;
|
||||
dbPath?: string;
|
||||
sources?: MemorySource[];
|
||||
cache?: { enabled: boolean; entries?: number; maxEntries?: number };
|
||||
fts?: { enabled: boolean; available: boolean; error?: string };
|
||||
fallback?: { from: string; reason?: string };
|
||||
vector?: {
|
||||
enabled: boolean;
|
||||
available?: boolean;
|
||||
extensionPath?: string;
|
||||
loadError?: string;
|
||||
dims?: number;
|
||||
};
|
||||
batch?: {
|
||||
enabled: boolean;
|
||||
failures: number;
|
||||
limit: number;
|
||||
wait: boolean;
|
||||
concurrency: number;
|
||||
pollIntervalMs: number;
|
||||
timeoutMs: number;
|
||||
lastError?: string;
|
||||
lastProvider?: string;
|
||||
};
|
||||
custom?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
export interface MemorySearchManager {
|
||||
search(
|
||||
query: string,
|
||||
opts?: { maxResults?: number; minScore?: number; sessionKey?: string },
|
||||
): Promise<MemorySearchResult[]>;
|
||||
readFile(params: {
|
||||
relPath: string;
|
||||
from?: number;
|
||||
lines?: number;
|
||||
}): Promise<{ text: string; path: string }>;
|
||||
status(): MemoryProviderStatus;
|
||||
sync?(params?: {
|
||||
reason?: string;
|
||||
force?: boolean;
|
||||
progress?: (update: MemorySyncProgressUpdate) => void;
|
||||
}): Promise<void>;
|
||||
probeVectorAvailability(): Promise<boolean>;
|
||||
close?(): Promise<void>;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user