diff --git a/src/memory/index.test.ts b/src/memory/index.test.ts index 58a98e580..a239ccfa9 100644 --- a/src/memory/index.test.ts +++ b/src/memory/index.test.ts @@ -84,7 +84,7 @@ describe("memory index", () => { await result.manager.sync({ force: true }); const results = await result.manager.search("alpha"); expect(results.length).toBeGreaterThan(0); - expect(results[0]?.path).toContain("memory/2026-01-12.md"); + expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md"); const status = result.manager.status(); expect(status.sourceCounts).toEqual( expect.arrayContaining([ @@ -254,7 +254,7 @@ describe("memory index", () => { await manager.sync({ force: true }); const results = await manager.search("zebra"); expect(results.length).toBeGreaterThan(0); - expect(results[0]?.path).toContain("memory/2026-01-12.md"); + expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md"); }); it("hybrid weights can favor vector-only matches over keyword-only matches", async () => { diff --git a/src/memory/internal.test.ts b/src/memory/internal.test.ts index 31fe4c940..7b654ea2f 100644 --- a/src/memory/internal.test.ts +++ b/src/memory/internal.test.ts @@ -8,7 +8,6 @@ import { isOldMemoryFormat, migrateMemoryFile, migrateAllMemoryFiles, - type MigrationResult, } from "./internal.js"; describe("chunkMarkdown", () => { @@ -93,7 +92,10 @@ describe("migrateMemoryFile", () => { expect(newContent).toBe("# Test content\n"); // Verify old file still exists (backup) - const oldExists = await fs.access(oldPath).then(() => true).catch(() => false); + const oldExists = await fs + .access(oldPath) + .then(() => true) + .catch(() => false); expect(oldExists).toBe(true); }); @@ -104,7 +106,9 @@ describe("migrateMemoryFile", () => { const result = await migrateMemoryFile(oldPath, tempDir, logger); expect(result.status).toBe("migrated"); - expect(result.path).toBe(path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md")); + expect(result.path).toBe( + path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md"), + ); }); it("skips migration if new file already exists", async () => { @@ -185,7 +189,11 @@ describe("migrateAllMemoryFiles", () => { // Create old-format files await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Day 1\n", "utf-8"); await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Day 2\n", "utf-8"); - await fs.writeFile(path.join(tempDir, "memory", "2025-01-29-discussion.md"), "# Discussion\n", "utf-8"); + await fs.writeFile( + path.join(tempDir, "memory", "2025-01-29-discussion.md"), + "# Discussion\n", + "utf-8", + ); const result = await migrateAllMemoryFiles(tempDir, { logger }); @@ -197,9 +205,18 @@ describe("migrateAllMemoryFiles", () => { expect(result.durationMs).toBeGreaterThanOrEqual(0); // Verify new files exist - const file1Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false); - const file2Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md")).then(() => true).catch(() => false); - const file3Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md")).then(() => true).catch(() => false); + const file1Exists = await fs + .access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")) + .then(() => true) + .catch(() => false); + const file2Exists = await fs + .access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md")) + .then(() => true) + .catch(() => false); + const file3Exists = await fs + .access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md")) + .then(() => true) + .catch(() => false); expect(file1Exists).toBe(true); expect(file2Exists).toBe(true); expect(file3Exists).toBe(true); @@ -209,7 +226,11 @@ describe("migrateAllMemoryFiles", () => { // Create files: one in memory/ root (old format), one in subdirectory (user custom, should not migrate) await fs.mkdir(path.join(tempDir, "memory", "archive"), { recursive: true }); await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Root\n", "utf-8"); - await fs.writeFile(path.join(tempDir, "memory", "archive", "2024-12-31.md"), "# Archived\n", "utf-8"); + await fs.writeFile( + path.join(tempDir, "memory", "archive", "2024-12-31.md"), + "# Archived\n", + "utf-8", + ); const result = await migrateAllMemoryFiles(tempDir, { logger }); @@ -217,16 +238,23 @@ describe("migrateAllMemoryFiles", () => { expect(result.migrated).toBe(1); expect(result.migratedFiles).toContain("memory/2025/01/2025-01-27.md"); expect(result.migratedFiles).not.toContain("memory/2024/12/2024-12-31.md"); - + // Archived file should still exist in original location - const archivedExists = await fs.access(path.join(tempDir, "memory", "archive", "2024-12-31.md")).then(() => true).catch(() => false); + const archivedExists = await fs + .access(path.join(tempDir, "memory", "archive", "2024-12-31.md")) + .then(() => true) + .catch(() => false); expect(archivedExists).toBe(true); }); it("skips year directories (new format)", async () => { // Create new-format files (should be skipped) await fs.mkdir(path.join(tempDir, "memory", "2025", "01"), { recursive: true }); - await fs.writeFile(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"), "# New format\n", "utf-8"); + await fs.writeFile( + path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"), + "# New format\n", + "utf-8", + ); // Create old-format file await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Old format\n", "utf-8"); @@ -246,13 +274,16 @@ describe("migrateAllMemoryFiles", () => { expect(result.skipped).toBe(0); // Verify file was NOT migrated - const newExists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false); + const newExists = await fs + .access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")) + .then(() => true) + .catch(() => false); expect(newExists).toBe(false); }); it("returns empty result if no memory directory", async () => { const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), "empty-")); - + const result = await migrateAllMemoryFiles(emptyDir, { logger }); expect(result.migrated).toBe(0); @@ -283,7 +314,8 @@ describe("migrateAllMemoryFiles", () => { const result = await migrateAllMemoryFiles(tempDir, { logger }); - const expectedBytes = Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8"); + const expectedBytes = + Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8"); expect(result.totalBytes).toBe(expectedBytes); }); }); diff --git a/src/memory/internal.ts b/src/memory/internal.ts index 13016bc05..95d50a565 100644 --- a/src/memory/internal.ts +++ b/src/memory/internal.ts @@ -50,7 +50,11 @@ async function walkDir( dir: string, files: string[], workspaceDir: string, - logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, + logger?: { + info: (msg: string) => void; + warn: (msg: string) => void; + error: (msg: string, err?: unknown) => void; + }, ) { const entries = await fs.readdir(dir, { withFileTypes: true }); for (const entry of entries) { @@ -67,7 +71,7 @@ async function walkDir( if (isOldMemoryFormat(relPath)) { // Auto-migrate to new format const result = await migrateMemoryFile(full, workspaceDir, logger); - if (result.status !== 'failed') { + if (result.status !== "failed") { files.push(result.path); } continue; @@ -85,7 +89,7 @@ export function isOldMemoryFormat(relPath: string): boolean { // Matches: memory/2025-01-27.md or memory/2025-01-27-slug.md // But NOT: memory/2025/01/2025-01-27.md (new hierarchical format) // But NOT: memory/notes/2025-01-27.md (user subdirectory) - const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^.\/]+)?\.md$/; + const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^./]+)?\.md$/; return oldFormatRegex.test(relPath); } @@ -93,9 +97,9 @@ export function isOldMemoryFormat(relPath: string): boolean { * Result of migrating a memory file */ export type MigrationResult = - | { status: 'migrated'; path: string } - | { status: 'skipped'; path: string } - | { status: 'failed' }; + | { status: "migrated"; path: string } + | { status: "skipped"; path: string } + | { status: "failed" }; /** * Migrate an old-format memory file to new hierarchical structure @@ -107,27 +111,31 @@ export type MigrationResult = export async function migrateMemoryFile( oldPath: string, workspaceDir: string, - logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, + logger?: { + info: (msg: string) => void; + warn: (msg: string) => void; + error: (msg: string, err?: unknown) => void; + }, ): Promise { const log = logger || { info: console.log, warn: console.warn, error: console.error }; - + try { const filename = path.basename(oldPath); const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})(?:-.*)?\.md$/); if (!match) { log.error(`[memory] Invalid filename format for migration: ${filename}`); - return { status: 'failed' }; + return { status: "failed" }; } const [, year, month, day] = match; - + // Validate date (prevent invalid dates like 2025-99-99) const dateStr = `${year}-${month}-${day}`; const parsedDate = new Date(dateStr); - if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split('T')[0] !== dateStr) { + if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split("T")[0] !== dateStr) { log.error(`[memory] Invalid date in filename: ${filename} (parsed: ${dateStr})`); - return { status: 'failed' }; + return { status: "failed" }; } const newDir = path.join(workspaceDir, "memory", year, month); @@ -141,20 +149,20 @@ export async function migrateMemoryFile( if (await exists(newPath)) { // New file exists, keep it and skip migration log.warn(`[memory] New format file exists, skipping migration: ${filename}`); - return { status: 'skipped', path: newPath }; + return { status: "skipped", path: newPath }; } // Atomic file creation with exclusive flag to prevent race conditions let fileHandle; try { // Try to open file exclusively (fails if exists) - fileHandle = await fs.open(newPath, 'wx'); + fileHandle = await fs.open(newPath, "wx"); await fileHandle.close(); } catch (err: any) { - if (err.code === 'EEXIST') { + if (err.code === "EEXIST") { // Another process created it first log.warn(`[memory] File created by another process, skipping: ${filename}`); - return { status: 'skipped', path: newPath }; + return { status: "skipped", path: newPath }; } throw err; } @@ -164,18 +172,24 @@ export async function migrateMemoryFile( // Optional: Remove old file after successful migration // For now, keep both as backup - log.info(`[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`); + log.info( + `[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`, + ); - return { status: 'migrated', path: newPath }; + return { status: "migrated", path: newPath }; } catch (err) { log.error(`[memory] Failed to migrate memory file: ${path.basename(oldPath)}`, err); - return { status: 'failed' }; + return { status: "failed" }; } } export async function listMemoryFiles( workspaceDir: string, - logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }, + logger?: { + info: (msg: string) => void; + warn: (msg: string) => void; + error: (msg: string, err?: unknown) => void; + }, ): Promise { const result: string[] = []; const memoryFile = path.join(workspaceDir, "MEMORY.md"); @@ -324,7 +338,11 @@ export type MigrateAllOptions = { /** If true, only simulate migration without actual file operations */ dryRun?: boolean; /** Optional logger for migration messages */ - logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void }; + logger?: { + info: (msg: string) => void; + warn: (msg: string) => void; + error: (msg: string, err?: unknown) => void; + }; }; export type MigrateAllResult = { @@ -352,10 +370,10 @@ async function findOldFormatFiles( try { const entries = await fs.readdir(dir, { withFileTypes: true }); const memoryDir = path.join(workspaceDir, "memory"); - + for (const entry of entries) { const fullPath = path.join(dir, entry.name); - + if (entry.isDirectory()) { // Skip year directories (already new format) only in memory/ root const isMemoryRoot = path.normalize(dir) === path.normalize(memoryDir); @@ -374,7 +392,7 @@ async function findOldFormatFiles( } } } - } catch (err) { + } catch { // Ignore permission errors and continue } return results; @@ -392,10 +410,10 @@ export async function migrateAllMemoryFiles( ): Promise { const { dryRun = false, logger } = options; const log = logger || { info: console.log, warn: console.warn, error: console.error }; - + const startTime = Date.now(); const memoryDir = path.join(workspaceDir, "memory"); - + if (!(await exists(memoryDir))) { return { migrated: 0, @@ -410,7 +428,7 @@ export async function migrateAllMemoryFiles( // Find all old-format files recursively const oldFiles = await findOldFormatFiles(memoryDir, workspaceDir); - + if (oldFiles.length === 0) { log.info(`[memory] No old-format files found in ${memoryDir}`); return { @@ -438,7 +456,7 @@ export async function migrateAllMemoryFiles( for (const fullPath of oldFiles) { const relPath = path.relative(workspaceDir, fullPath).replace(/\\/g, "/"); - + if (dryRun) { // In dry-run mode, just check if target exists const filename = path.basename(fullPath); @@ -450,7 +468,9 @@ export async function migrateAllMemoryFiles( log.info(`[memory] [DRY RUN] Would skip (exists): ${relPath}`); skipped++; } else { - log.info(`[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`); + log.info( + `[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`, + ); migrated++; try { const stat = await fs.stat(fullPath); @@ -465,16 +485,16 @@ export async function migrateAllMemoryFiles( try { const stat = await fs.stat(fullPath); const result = await migrateMemoryFile(fullPath, workspaceDir, logger); - - if (result.status === 'skipped') { + + if (result.status === "skipped") { skipped++; - } else if (result.status === 'migrated') { + } else if (result.status === "migrated") { migrated++; totalBytes += stat.size; migratedFiles.push(path.relative(workspaceDir, result.path).replace(/\\/g, "/")); } else { failed++; - failedFiles.push({ path: relPath, error: 'Migration failed' }); + failedFiles.push({ path: relPath, error: "Migration failed" }); } } catch (err) { failed++; @@ -485,9 +505,11 @@ export async function migrateAllMemoryFiles( } const durationMs = Date.now() - startTime; - - log.info(`[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`); - + + log.info( + `[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`, + ); + return { migrated, skipped, diff --git a/src/memory/manager.batch.test.ts b/src/memory/manager.batch.test.ts index 31327cbc8..f3103e91c 100644 --- a/src/memory/manager.batch.test.ts +++ b/src/memory/manager.batch.test.ts @@ -266,7 +266,9 @@ describe("memory indexing with OpenAI batches", () => { it("falls back to non-batch on failure and resets failures after success", async () => { const content = ["flaky", "batch"].join("\n\n"); - await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content); + // Create file in new format to avoid migration during sync + await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true }); + await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"), content); let uploadedRequests: Array<{ custom_id?: string }> = []; let mode: "fail" | "ok" = "fail"; @@ -363,8 +365,10 @@ describe("memory indexing with OpenAI batches", () => { embedBatch.mockClear(); mode = "ok"; + // Create file in new format to avoid migration during sync + await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true }); await fs.writeFile( - path.join(workspaceDir, "memory", "2026-01-09.md"), + path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"), ["flaky", "batch", "recovery"].join("\n\n"), ); await manager.sync({ force: true }); @@ -376,7 +380,9 @@ describe("memory indexing with OpenAI batches", () => { it("disables batch after repeated failures and skips batch thereafter", async () => { const content = ["repeat", "failures"].join("\n\n"); - await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-10.md"), content); + // Create file in new format to avoid migration during sync + await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true }); + await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"), content); let uploadedRequests: Array<{ custom_id?: string }> = []; const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { @@ -459,7 +465,7 @@ describe("memory indexing with OpenAI batches", () => { embedBatch.mockClear(); await fs.writeFile( - path.join(workspaceDir, "memory", "2026-01-10.md"), + path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"), ["repeat", "failures", "again"].join("\n\n"), ); await manager.sync({ force: true }); @@ -470,7 +476,7 @@ describe("memory indexing with OpenAI batches", () => { const fetchCalls = fetchMock.mock.calls.length; embedBatch.mockClear(); await fs.writeFile( - path.join(workspaceDir, "memory", "2026-01-10.md"), + path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"), ["repeat", "failures", "fallback"].join("\n\n"), ); await manager.sync({ force: true });