fix: update tests for hierarchical memory structure

This commit is contained in:
Nickolai Yegorov 2026-01-28 00:49:05 +03:00
parent d91522d4a6
commit 225ed0c794
4 changed files with 117 additions and 57 deletions

View File

@ -84,7 +84,7 @@ describe("memory index", () => {
await result.manager.sync({ force: true }); await result.manager.sync({ force: true });
const results = await result.manager.search("alpha"); const results = await result.manager.search("alpha");
expect(results.length).toBeGreaterThan(0); expect(results.length).toBeGreaterThan(0);
expect(results[0]?.path).toContain("memory/2026-01-12.md"); expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md");
const status = result.manager.status(); const status = result.manager.status();
expect(status.sourceCounts).toEqual( expect(status.sourceCounts).toEqual(
expect.arrayContaining([ expect.arrayContaining([
@ -254,7 +254,7 @@ describe("memory index", () => {
await manager.sync({ force: true }); await manager.sync({ force: true });
const results = await manager.search("zebra"); const results = await manager.search("zebra");
expect(results.length).toBeGreaterThan(0); expect(results.length).toBeGreaterThan(0);
expect(results[0]?.path).toContain("memory/2026-01-12.md"); expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md");
}); });
it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { it("hybrid weights can favor vector-only matches over keyword-only matches", async () => {

View File

@ -8,7 +8,6 @@ import {
isOldMemoryFormat, isOldMemoryFormat,
migrateMemoryFile, migrateMemoryFile,
migrateAllMemoryFiles, migrateAllMemoryFiles,
type MigrationResult,
} from "./internal.js"; } from "./internal.js";
describe("chunkMarkdown", () => { describe("chunkMarkdown", () => {
@ -93,7 +92,10 @@ describe("migrateMemoryFile", () => {
expect(newContent).toBe("# Test content\n"); expect(newContent).toBe("# Test content\n");
// Verify old file still exists (backup) // Verify old file still exists (backup)
const oldExists = await fs.access(oldPath).then(() => true).catch(() => false); const oldExists = await fs
.access(oldPath)
.then(() => true)
.catch(() => false);
expect(oldExists).toBe(true); expect(oldExists).toBe(true);
}); });
@ -104,7 +106,9 @@ describe("migrateMemoryFile", () => {
const result = await migrateMemoryFile(oldPath, tempDir, logger); const result = await migrateMemoryFile(oldPath, tempDir, logger);
expect(result.status).toBe("migrated"); expect(result.status).toBe("migrated");
expect(result.path).toBe(path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md")); expect(result.path).toBe(
path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md"),
);
}); });
it("skips migration if new file already exists", async () => { it("skips migration if new file already exists", async () => {
@ -185,7 +189,11 @@ describe("migrateAllMemoryFiles", () => {
// Create old-format files // Create old-format files
await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Day 1\n", "utf-8"); await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Day 1\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Day 2\n", "utf-8"); await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Day 2\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "2025-01-29-discussion.md"), "# Discussion\n", "utf-8"); await fs.writeFile(
path.join(tempDir, "memory", "2025-01-29-discussion.md"),
"# Discussion\n",
"utf-8",
);
const result = await migrateAllMemoryFiles(tempDir, { logger }); const result = await migrateAllMemoryFiles(tempDir, { logger });
@ -197,9 +205,18 @@ describe("migrateAllMemoryFiles", () => {
expect(result.durationMs).toBeGreaterThanOrEqual(0); expect(result.durationMs).toBeGreaterThanOrEqual(0);
// Verify new files exist // Verify new files exist
const file1Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false); const file1Exists = await fs
const file2Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md")).then(() => true).catch(() => false); .access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"))
const file3Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md")).then(() => true).catch(() => false); .then(() => true)
.catch(() => false);
const file2Exists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md"))
.then(() => true)
.catch(() => false);
const file3Exists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md"))
.then(() => true)
.catch(() => false);
expect(file1Exists).toBe(true); expect(file1Exists).toBe(true);
expect(file2Exists).toBe(true); expect(file2Exists).toBe(true);
expect(file3Exists).toBe(true); expect(file3Exists).toBe(true);
@ -209,7 +226,11 @@ describe("migrateAllMemoryFiles", () => {
// Create files: one in memory/ root (old format), one in subdirectory (user custom, should not migrate) // Create files: one in memory/ root (old format), one in subdirectory (user custom, should not migrate)
await fs.mkdir(path.join(tempDir, "memory", "archive"), { recursive: true }); await fs.mkdir(path.join(tempDir, "memory", "archive"), { recursive: true });
await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Root\n", "utf-8"); await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Root\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "archive", "2024-12-31.md"), "# Archived\n", "utf-8"); await fs.writeFile(
path.join(tempDir, "memory", "archive", "2024-12-31.md"),
"# Archived\n",
"utf-8",
);
const result = await migrateAllMemoryFiles(tempDir, { logger }); const result = await migrateAllMemoryFiles(tempDir, { logger });
@ -219,14 +240,21 @@ describe("migrateAllMemoryFiles", () => {
expect(result.migratedFiles).not.toContain("memory/2024/12/2024-12-31.md"); expect(result.migratedFiles).not.toContain("memory/2024/12/2024-12-31.md");
// Archived file should still exist in original location // Archived file should still exist in original location
const archivedExists = await fs.access(path.join(tempDir, "memory", "archive", "2024-12-31.md")).then(() => true).catch(() => false); const archivedExists = await fs
.access(path.join(tempDir, "memory", "archive", "2024-12-31.md"))
.then(() => true)
.catch(() => false);
expect(archivedExists).toBe(true); expect(archivedExists).toBe(true);
}); });
it("skips year directories (new format)", async () => { it("skips year directories (new format)", async () => {
// Create new-format files (should be skipped) // Create new-format files (should be skipped)
await fs.mkdir(path.join(tempDir, "memory", "2025", "01"), { recursive: true }); await fs.mkdir(path.join(tempDir, "memory", "2025", "01"), { recursive: true });
await fs.writeFile(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"), "# New format\n", "utf-8"); await fs.writeFile(
path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"),
"# New format\n",
"utf-8",
);
// Create old-format file // Create old-format file
await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Old format\n", "utf-8"); await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Old format\n", "utf-8");
@ -246,7 +274,10 @@ describe("migrateAllMemoryFiles", () => {
expect(result.skipped).toBe(0); expect(result.skipped).toBe(0);
// Verify file was NOT migrated // Verify file was NOT migrated
const newExists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false); const newExists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"))
.then(() => true)
.catch(() => false);
expect(newExists).toBe(false); expect(newExists).toBe(false);
}); });
@ -283,7 +314,8 @@ describe("migrateAllMemoryFiles", () => {
const result = await migrateAllMemoryFiles(tempDir, { logger }); const result = await migrateAllMemoryFiles(tempDir, { logger });
const expectedBytes = Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8"); const expectedBytes =
Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8");
expect(result.totalBytes).toBe(expectedBytes); expect(result.totalBytes).toBe(expectedBytes);
}); });
}); });

View File

@ -50,7 +50,11 @@ async function walkDir(
dir: string, dir: string,
files: string[], files: string[],
workspaceDir: string, workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
) { ) {
const entries = await fs.readdir(dir, { withFileTypes: true }); const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) { for (const entry of entries) {
@ -67,7 +71,7 @@ async function walkDir(
if (isOldMemoryFormat(relPath)) { if (isOldMemoryFormat(relPath)) {
// Auto-migrate to new format // Auto-migrate to new format
const result = await migrateMemoryFile(full, workspaceDir, logger); const result = await migrateMemoryFile(full, workspaceDir, logger);
if (result.status !== 'failed') { if (result.status !== "failed") {
files.push(result.path); files.push(result.path);
} }
continue; continue;
@ -85,7 +89,7 @@ export function isOldMemoryFormat(relPath: string): boolean {
// Matches: memory/2025-01-27.md or memory/2025-01-27-slug.md // Matches: memory/2025-01-27.md or memory/2025-01-27-slug.md
// But NOT: memory/2025/01/2025-01-27.md (new hierarchical format) // But NOT: memory/2025/01/2025-01-27.md (new hierarchical format)
// But NOT: memory/notes/2025-01-27.md (user subdirectory) // But NOT: memory/notes/2025-01-27.md (user subdirectory)
const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^.\/]+)?\.md$/; const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^./]+)?\.md$/;
return oldFormatRegex.test(relPath); return oldFormatRegex.test(relPath);
} }
@ -93,9 +97,9 @@ export function isOldMemoryFormat(relPath: string): boolean {
* Result of migrating a memory file * Result of migrating a memory file
*/ */
export type MigrationResult = export type MigrationResult =
| { status: 'migrated'; path: string } | { status: "migrated"; path: string }
| { status: 'skipped'; path: string } | { status: "skipped"; path: string }
| { status: 'failed' }; | { status: "failed" };
/** /**
* Migrate an old-format memory file to new hierarchical structure * Migrate an old-format memory file to new hierarchical structure
@ -107,7 +111,11 @@ export type MigrationResult =
export async function migrateMemoryFile( export async function migrateMemoryFile(
oldPath: string, oldPath: string,
workspaceDir: string, workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
): Promise<MigrationResult> { ): Promise<MigrationResult> {
const log = logger || { info: console.log, warn: console.warn, error: console.error }; const log = logger || { info: console.log, warn: console.warn, error: console.error };
@ -117,7 +125,7 @@ export async function migrateMemoryFile(
if (!match) { if (!match) {
log.error(`[memory] Invalid filename format for migration: ${filename}`); log.error(`[memory] Invalid filename format for migration: ${filename}`);
return { status: 'failed' }; return { status: "failed" };
} }
const [, year, month, day] = match; const [, year, month, day] = match;
@ -125,9 +133,9 @@ export async function migrateMemoryFile(
// Validate date (prevent invalid dates like 2025-99-99) // Validate date (prevent invalid dates like 2025-99-99)
const dateStr = `${year}-${month}-${day}`; const dateStr = `${year}-${month}-${day}`;
const parsedDate = new Date(dateStr); const parsedDate = new Date(dateStr);
if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split('T')[0] !== dateStr) { if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split("T")[0] !== dateStr) {
log.error(`[memory] Invalid date in filename: ${filename} (parsed: ${dateStr})`); log.error(`[memory] Invalid date in filename: ${filename} (parsed: ${dateStr})`);
return { status: 'failed' }; return { status: "failed" };
} }
const newDir = path.join(workspaceDir, "memory", year, month); const newDir = path.join(workspaceDir, "memory", year, month);
@ -141,20 +149,20 @@ export async function migrateMemoryFile(
if (await exists(newPath)) { if (await exists(newPath)) {
// New file exists, keep it and skip migration // New file exists, keep it and skip migration
log.warn(`[memory] New format file exists, skipping migration: ${filename}`); log.warn(`[memory] New format file exists, skipping migration: ${filename}`);
return { status: 'skipped', path: newPath }; return { status: "skipped", path: newPath };
} }
// Atomic file creation with exclusive flag to prevent race conditions // Atomic file creation with exclusive flag to prevent race conditions
let fileHandle; let fileHandle;
try { try {
// Try to open file exclusively (fails if exists) // Try to open file exclusively (fails if exists)
fileHandle = await fs.open(newPath, 'wx'); fileHandle = await fs.open(newPath, "wx");
await fileHandle.close(); await fileHandle.close();
} catch (err: any) { } catch (err: any) {
if (err.code === 'EEXIST') { if (err.code === "EEXIST") {
// Another process created it first // Another process created it first
log.warn(`[memory] File created by another process, skipping: ${filename}`); log.warn(`[memory] File created by another process, skipping: ${filename}`);
return { status: 'skipped', path: newPath }; return { status: "skipped", path: newPath };
} }
throw err; throw err;
} }
@ -164,18 +172,24 @@ export async function migrateMemoryFile(
// Optional: Remove old file after successful migration // Optional: Remove old file after successful migration
// For now, keep both as backup // For now, keep both as backup
log.info(`[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`); log.info(
`[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`,
);
return { status: 'migrated', path: newPath }; return { status: "migrated", path: newPath };
} catch (err) { } catch (err) {
log.error(`[memory] Failed to migrate memory file: ${path.basename(oldPath)}`, err); log.error(`[memory] Failed to migrate memory file: ${path.basename(oldPath)}`, err);
return { status: 'failed' }; return { status: "failed" };
} }
} }
export async function listMemoryFiles( export async function listMemoryFiles(
workspaceDir: string, workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
): Promise<string[]> { ): Promise<string[]> {
const result: string[] = []; const result: string[] = [];
const memoryFile = path.join(workspaceDir, "MEMORY.md"); const memoryFile = path.join(workspaceDir, "MEMORY.md");
@ -324,7 +338,11 @@ export type MigrateAllOptions = {
/** If true, only simulate migration without actual file operations */ /** If true, only simulate migration without actual file operations */
dryRun?: boolean; dryRun?: boolean;
/** Optional logger for migration messages */ /** Optional logger for migration messages */
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }; logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
};
}; };
export type MigrateAllResult = { export type MigrateAllResult = {
@ -374,7 +392,7 @@ async function findOldFormatFiles(
} }
} }
} }
} catch (err) { } catch {
// Ignore permission errors and continue // Ignore permission errors and continue
} }
return results; return results;
@ -450,7 +468,9 @@ export async function migrateAllMemoryFiles(
log.info(`[memory] [DRY RUN] Would skip (exists): ${relPath}`); log.info(`[memory] [DRY RUN] Would skip (exists): ${relPath}`);
skipped++; skipped++;
} else { } else {
log.info(`[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`); log.info(
`[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`,
);
migrated++; migrated++;
try { try {
const stat = await fs.stat(fullPath); const stat = await fs.stat(fullPath);
@ -466,15 +486,15 @@ export async function migrateAllMemoryFiles(
const stat = await fs.stat(fullPath); const stat = await fs.stat(fullPath);
const result = await migrateMemoryFile(fullPath, workspaceDir, logger); const result = await migrateMemoryFile(fullPath, workspaceDir, logger);
if (result.status === 'skipped') { if (result.status === "skipped") {
skipped++; skipped++;
} else if (result.status === 'migrated') { } else if (result.status === "migrated") {
migrated++; migrated++;
totalBytes += stat.size; totalBytes += stat.size;
migratedFiles.push(path.relative(workspaceDir, result.path).replace(/\\/g, "/")); migratedFiles.push(path.relative(workspaceDir, result.path).replace(/\\/g, "/"));
} else { } else {
failed++; failed++;
failedFiles.push({ path: relPath, error: 'Migration failed' }); failedFiles.push({ path: relPath, error: "Migration failed" });
} }
} catch (err) { } catch (err) {
failed++; failed++;
@ -486,7 +506,9 @@ export async function migrateAllMemoryFiles(
const durationMs = Date.now() - startTime; const durationMs = Date.now() - startTime;
log.info(`[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`); log.info(
`[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`,
);
return { return {
migrated, migrated,

View File

@ -266,7 +266,9 @@ describe("memory indexing with OpenAI batches", () => {
it("falls back to non-batch on failure and resets failures after success", async () => { it("falls back to non-batch on failure and resets failures after success", async () => {
const content = ["flaky", "batch"].join("\n\n"); const content = ["flaky", "batch"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content); // Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"), content);
let uploadedRequests: Array<{ custom_id?: string }> = []; let uploadedRequests: Array<{ custom_id?: string }> = [];
let mode: "fail" | "ok" = "fail"; let mode: "fail" | "ok" = "fail";
@ -363,8 +365,10 @@ describe("memory indexing with OpenAI batches", () => {
embedBatch.mockClear(); embedBatch.mockClear();
mode = "ok"; mode = "ok";
// Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile( await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"), path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"),
["flaky", "batch", "recovery"].join("\n\n"), ["flaky", "batch", "recovery"].join("\n\n"),
); );
await manager.sync({ force: true }); await manager.sync({ force: true });
@ -376,7 +380,9 @@ describe("memory indexing with OpenAI batches", () => {
it("disables batch after repeated failures and skips batch thereafter", async () => { it("disables batch after repeated failures and skips batch thereafter", async () => {
const content = ["repeat", "failures"].join("\n\n"); const content = ["repeat", "failures"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-10.md"), content); // Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"), content);
let uploadedRequests: Array<{ custom_id?: string }> = []; let uploadedRequests: Array<{ custom_id?: string }> = [];
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
@ -459,7 +465,7 @@ describe("memory indexing with OpenAI batches", () => {
embedBatch.mockClear(); embedBatch.mockClear();
await fs.writeFile( await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"), path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"),
["repeat", "failures", "again"].join("\n\n"), ["repeat", "failures", "again"].join("\n\n"),
); );
await manager.sync({ force: true }); await manager.sync({ force: true });
@ -470,7 +476,7 @@ describe("memory indexing with OpenAI batches", () => {
const fetchCalls = fetchMock.mock.calls.length; const fetchCalls = fetchMock.mock.calls.length;
embedBatch.mockClear(); embedBatch.mockClear();
await fs.writeFile( await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"), path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"),
["repeat", "failures", "fallback"].join("\n\n"), ["repeat", "failures", "fallback"].join("\n\n"),
); );
await manager.sync({ force: true }); await manager.sync({ force: true });