romtuck/openclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: update tests for hierarchical memory structure

Browse Source
This commit is contained in:
Nickolai Yegorov 2026-01-28 00:49:05 +03:00
parent d91522d4a6
commit 225ed0c794
4 changed files with 117 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/memory/index.test.ts
View File

@ -84,7 +84,7 @@ describe("memory index", () => {
await result.manager.sync({ force: true });
const results = await result.manager.search("alpha");
expect(results.length).toBeGreaterThan(0);
expect(results[0]?.path).toContain("memory/2026-01-12.md");
expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md");
const status = result.manager.status();
expect(status.sourceCounts).toEqual(
expect.arrayContaining([
@ -254,7 +254,7 @@ describe("memory index", () => {
await manager.sync({ force: true });
const results = await manager.search("zebra");
expect(results.length).toBeGreaterThan(0);
expect(results[0]?.path).toContain("memory/2026-01-12.md");
expect(results[0]?.path).toContain("memory/2026/01/2026-01-12.md");
});
it("hybrid weights can favor vector-only matches over keyword-only matches", async () => {

60
src/memory/internal.test.ts
View File

@ -8,7 +8,6 @@ import {
isOldMemoryFormat,
migrateMemoryFile,
migrateAllMemoryFiles,
type MigrationResult,
} from "./internal.js";
describe("chunkMarkdown", () => {
@ -93,7 +92,10 @@ describe("migrateMemoryFile", () => {
expect(newContent).toBe("# Test content\n");
// Verify old file still exists (backup)
const oldExists = await fs.access(oldPath).then(() => true).catch(() => false);
const oldExists = await fs
.access(oldPath)
.then(() => true)
.catch(() => false);
expect(oldExists).toBe(true);
});
@ -104,7 +106,9 @@ describe("migrateMemoryFile", () => {
const result = await migrateMemoryFile(oldPath, tempDir, logger);
expect(result.status).toBe("migrated");
expect(result.path).toBe(path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md"));
expect(result.path).toBe(
path.join(tempDir, "memory", "2025", "01", "2025-01-27-discussion.md"),
);
});
it("skips migration if new file already exists", async () => {
@ -185,7 +189,11 @@ describe("migrateAllMemoryFiles", () => {
// Create old-format files
await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Day 1\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Day 2\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "2025-01-29-discussion.md"), "# Discussion\n", "utf-8");
await fs.writeFile(
path.join(tempDir, "memory", "2025-01-29-discussion.md"),
"# Discussion\n",
"utf-8",
);
const result = await migrateAllMemoryFiles(tempDir, { logger });
@ -197,9 +205,18 @@ describe("migrateAllMemoryFiles", () => {
expect(result.durationMs).toBeGreaterThanOrEqual(0);
// Verify new files exist
const file1Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false);
const file2Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md")).then(() => true).catch(() => false);
const file3Exists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md")).then(() => true).catch(() => false);
const file1Exists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"))
.then(() => true)
.catch(() => false);
const file2Exists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-28.md"))
.then(() => true)
.catch(() => false);
const file3Exists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-29-discussion.md"))
.then(() => true)
.catch(() => false);
expect(file1Exists).toBe(true);
expect(file2Exists).toBe(true);
expect(file3Exists).toBe(true);
@ -209,7 +226,11 @@ describe("migrateAllMemoryFiles", () => {
// Create files: one in memory/ root (old format), one in subdirectory (user custom, should not migrate)
await fs.mkdir(path.join(tempDir, "memory", "archive"), { recursive: true });
await fs.writeFile(path.join(tempDir, "memory", "2025-01-27.md"), "# Root\n", "utf-8");
await fs.writeFile(path.join(tempDir, "memory", "archive", "2024-12-31.md"), "# Archived\n", "utf-8");
await fs.writeFile(
path.join(tempDir, "memory", "archive", "2024-12-31.md"),
"# Archived\n",
"utf-8",
);
const result = await migrateAllMemoryFiles(tempDir, { logger });
@ -217,16 +238,23 @@ describe("migrateAllMemoryFiles", () => {
expect(result.migrated).toBe(1);
expect(result.migratedFiles).toContain("memory/2025/01/2025-01-27.md");
expect(result.migratedFiles).not.toContain("memory/2024/12/2024-12-31.md");
// Archived file should still exist in original location
const archivedExists = await fs.access(path.join(tempDir, "memory", "archive", "2024-12-31.md")).then(() => true).catch(() => false);
const archivedExists = await fs
.access(path.join(tempDir, "memory", "archive", "2024-12-31.md"))
.then(() => true)
.catch(() => false);
expect(archivedExists).toBe(true);
});
it("skips year directories (new format)", async () => {
// Create new-format files (should be skipped)
await fs.mkdir(path.join(tempDir, "memory", "2025", "01"), { recursive: true });
await fs.writeFile(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"), "# New format\n", "utf-8");
await fs.writeFile(
path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"),
"# New format\n",
"utf-8",
);
// Create old-format file
await fs.writeFile(path.join(tempDir, "memory", "2025-01-28.md"), "# Old format\n", "utf-8");
@ -246,13 +274,16 @@ describe("migrateAllMemoryFiles", () => {
expect(result.skipped).toBe(0);
// Verify file was NOT migrated
const newExists = await fs.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md")).then(() => true).catch(() => false);
const newExists = await fs
.access(path.join(tempDir, "memory", "2025", "01", "2025-01-27.md"))
.then(() => true)
.catch(() => false);
expect(newExists).toBe(false);
});
it("returns empty result if no memory directory", async () => {
const emptyDir = await fs.mkdtemp(path.join(os.tmpdir(), "empty-"));
const result = await migrateAllMemoryFiles(emptyDir, { logger });
expect(result.migrated).toBe(0);
@ -283,7 +314,8 @@ describe("migrateAllMemoryFiles", () => {
const result = await migrateAllMemoryFiles(tempDir, { logger });
const expectedBytes = Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8");
const expectedBytes =
Buffer.byteLength(content1, "utf-8") + Buffer.byteLength(content2, "utf-8");
expect(result.totalBytes).toBe(expectedBytes);
});
});

94
src/memory/internal.ts
View File

@ -50,7 +50,11 @@ async function walkDir(
dir: string,
files: string[],
workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void },
logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
) {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
@ -67,7 +71,7 @@ async function walkDir(
if (isOldMemoryFormat(relPath)) {
// Auto-migrate to new format
const result = await migrateMemoryFile(full, workspaceDir, logger);
if (result.status !== 'failed') {
if (result.status !== "failed") {
files.push(result.path);
}
continue;
@ -85,7 +89,7 @@ export function isOldMemoryFormat(relPath: string): boolean {
// Matches: memory/2025-01-27.md or memory/2025-01-27-slug.md
// But NOT: memory/2025/01/2025-01-27.md (new hierarchical format)
// But NOT: memory/notes/2025-01-27.md (user subdirectory)
const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^.\/]+)?\.md$/;
const oldFormatRegex = /^memory\/\d{4}-\d{2}-\d{2}(?:-[^./]+)?\.md$/;
return oldFormatRegex.test(relPath);
}
@ -93,9 +97,9 @@ export function isOldMemoryFormat(relPath: string): boolean {
* Result of migrating a memory file
*/
export type MigrationResult =
| { status: 'migrated'; path: string }
| { status: 'skipped'; path: string }
| { status: 'failed' };
| { status: "migrated"; path: string }
| { status: "skipped"; path: string }
| { status: "failed" };
/**
* Migrate an old-format memory file to new hierarchical structure
@ -107,27 +111,31 @@ export type MigrationResult =
export async function migrateMemoryFile(
oldPath: string,
workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void },
logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
): Promise<MigrationResult> {
const log = logger || { info: console.log, warn: console.warn, error: console.error };
try {
const filename = path.basename(oldPath);
const match = filename.match(/^(\d{4})-(\d{2})-(\d{2})(?:-.*)?\.md$/);
if (!match) {
log.error(`[memory] Invalid filename format for migration: ${filename}`);
return { status: 'failed' };
return { status: "failed" };
}
const [, year, month, day] = match;
// Validate date (prevent invalid dates like 2025-99-99)
const dateStr = `${year}-${month}-${day}`;
const parsedDate = new Date(dateStr);
if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split('T')[0] !== dateStr) {
if (isNaN(parsedDate.getTime()) || parsedDate.toISOString().split("T")[0] !== dateStr) {
log.error(`[memory] Invalid date in filename: ${filename} (parsed: ${dateStr})`);
return { status: 'failed' };
return { status: "failed" };
}
const newDir = path.join(workspaceDir, "memory", year, month);
@ -141,20 +149,20 @@ export async function migrateMemoryFile(
if (await exists(newPath)) {
// New file exists, keep it and skip migration
log.warn(`[memory] New format file exists, skipping migration: ${filename}`);
return { status: 'skipped', path: newPath };
return { status: "skipped", path: newPath };
}
// Atomic file creation with exclusive flag to prevent race conditions
let fileHandle;
try {
// Try to open file exclusively (fails if exists)
fileHandle = await fs.open(newPath, 'wx');
fileHandle = await fs.open(newPath, "wx");
await fileHandle.close();
} catch (err: any) {
if (err.code === 'EEXIST') {
if (err.code === "EEXIST") {
// Another process created it first
log.warn(`[memory] File created by another process, skipping: ${filename}`);
return { status: 'skipped', path: newPath };
return { status: "skipped", path: newPath };
}
throw err;
}
@ -164,18 +172,24 @@ export async function migrateMemoryFile(
// Optional: Remove old file after successful migration
// For now, keep both as backup
log.info(`[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`);
log.info(
`[memory] Migrated old-format memory file: ${filename} -> memory/${year}/${month}/${filename}`,
);
return { status: 'migrated', path: newPath };
return { status: "migrated", path: newPath };
} catch (err) {
log.error(`[memory] Failed to migrate memory file: ${path.basename(oldPath)}`, err);
return { status: 'failed' };
return { status: "failed" };
}
}
export async function listMemoryFiles(
workspaceDir: string,
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void },
logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
},
): Promise<string[]> {
const result: string[] = [];
const memoryFile = path.join(workspaceDir, "MEMORY.md");
@ -324,7 +338,11 @@ export type MigrateAllOptions = {
/** If true, only simulate migration without actual file operations */
dryRun?: boolean;
/** Optional logger for migration messages */
logger?: { info: (msg: string) => void; warn: (msg: string) => void; error: (msg: string, err?: unknown) => void };
logger?: {
info: (msg: string) => void;
warn: (msg: string) => void;
error: (msg: string, err?: unknown) => void;
};
};
export type MigrateAllResult = {
@ -352,10 +370,10 @@ async function findOldFormatFiles(
try {
const entries = await fs.readdir(dir, { withFileTypes: true });
const memoryDir = path.join(workspaceDir, "memory");
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
// Skip year directories (already new format) only in memory/ root
const isMemoryRoot = path.normalize(dir) === path.normalize(memoryDir);
@ -374,7 +392,7 @@ async function findOldFormatFiles(
}
}
}
} catch (err) {
} catch {
// Ignore permission errors and continue
}
return results;
@ -392,10 +410,10 @@ export async function migrateAllMemoryFiles(
): Promise<MigrateAllResult> {
const { dryRun = false, logger } = options;
const log = logger || { info: console.log, warn: console.warn, error: console.error };
const startTime = Date.now();
const memoryDir = path.join(workspaceDir, "memory");
if (!(await exists(memoryDir))) {
return {
migrated: 0,
@ -410,7 +428,7 @@ export async function migrateAllMemoryFiles(
// Find all old-format files recursively
const oldFiles = await findOldFormatFiles(memoryDir, workspaceDir);
if (oldFiles.length === 0) {
log.info(`[memory] No old-format files found in ${memoryDir}`);
return {
@ -438,7 +456,7 @@ export async function migrateAllMemoryFiles(
for (const fullPath of oldFiles) {
const relPath = path.relative(workspaceDir, fullPath).replace(/\\/g, "/");
if (dryRun) {
// In dry-run mode, just check if target exists
const filename = path.basename(fullPath);
@ -450,7 +468,9 @@ export async function migrateAllMemoryFiles(
log.info(`[memory] [DRY RUN] Would skip (exists): ${relPath}`);
skipped++;
} else {
log.info(`[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`);
log.info(
`[memory] [DRY RUN] Would migrate: ${relPath} -> memory/${year}/${month}/${filename}`,
);
migrated++;
try {
const stat = await fs.stat(fullPath);
@ -465,16 +485,16 @@ export async function migrateAllMemoryFiles(
try {
const stat = await fs.stat(fullPath);
const result = await migrateMemoryFile(fullPath, workspaceDir, logger);
if (result.status === 'skipped') {
if (result.status === "skipped") {
skipped++;
} else if (result.status === 'migrated') {
} else if (result.status === "migrated") {
migrated++;
totalBytes += stat.size;
migratedFiles.push(path.relative(workspaceDir, result.path).replace(/\\/g, "/"));
} else {
failed++;
failedFiles.push({ path: relPath, error: 'Migration failed' });
failedFiles.push({ path: relPath, error: "Migration failed" });
}
} catch (err) {
failed++;
@ -485,9 +505,11 @@ export async function migrateAllMemoryFiles(
}
const durationMs = Date.now() - startTime;
log.info(`[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`);
log.info(
`[memory] Migration complete: ${migrated} migrated, ${skipped} skipped, ${failed} failed (${durationMs}ms, ${totalBytes} bytes)`,
);
return {
migrated,
skipped,

16
src/memory/manager.batch.test.ts
View File

@ -266,7 +266,9 @@ describe("memory indexing with OpenAI batches", () => {
it("falls back to non-batch on failure and resets failures after success", async () => {
const content = ["flaky", "batch"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-09.md"), content);
// Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"), content);
let uploadedRequests: Array<{ custom_id?: string }> = [];
let mode: "fail" | "ok" = "fail";
@ -363,8 +365,10 @@ describe("memory indexing with OpenAI batches", () => {
embedBatch.mockClear();
mode = "ok";
// Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-09.md"),
path.join(workspaceDir, "memory", "2026", "01", "2026-01-09.md"),
["flaky", "batch", "recovery"].join("\n\n"),
);
await manager.sync({ force: true });
@ -376,7 +380,9 @@ describe("memory indexing with OpenAI batches", () => {
it("disables batch after repeated failures and skips batch thereafter", async () => {
const content = ["repeat", "failures"].join("\n\n");
await fs.writeFile(path.join(workspaceDir, "memory", "2026-01-10.md"), content);
// Create file in new format to avoid migration during sync
await fs.mkdir(path.join(workspaceDir, "memory", "2026", "01"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"), content);
let uploadedRequests: Array<{ custom_id?: string }> = [];
const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => {
@ -459,7 +465,7 @@ describe("memory indexing with OpenAI batches", () => {
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"),
path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"),
["repeat", "failures", "again"].join("\n\n"),
);
await manager.sync({ force: true });
@ -470,7 +476,7 @@ describe("memory indexing with OpenAI batches", () => {
const fetchCalls = fetchMock.mock.calls.length;
embedBatch.mockClear();
await fs.writeFile(
path.join(workspaceDir, "memory", "2026-01-10.md"),
path.join(workspaceDir, "memory", "2026", "01", "2026-01-10.md"),
["repeat", "failures", "fallback"].join("\n\n"),
);
await manager.sync({ force: true });