refactor(memory): extract vector management to dedicated module
- Extract sqlite-vec loading and vector table lifecycle to VectorManager class - Move ~70 lines of vector logic from manager.ts to vector/vector-manager.ts - manager.ts delegates vector operations to VectorManager instance - Zero breaking changes to public API - All existing tests should pass Part of memory modularization effort to reduce manager.ts complexity (2179 → 2107 lines)
This commit is contained in:
parent
da421b9ef7
commit
37a1c361e0
@ -44,7 +44,7 @@ import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js"
|
|||||||
import { searchKeyword, searchVector } from "./manager-search.js";
|
import { searchKeyword, searchVector } from "./manager-search.js";
|
||||||
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
import { ensureMemoryIndexSchema } from "./memory-schema.js";
|
||||||
import { requireNodeSqlite } from "./sqlite.js";
|
import { requireNodeSqlite } from "./sqlite.js";
|
||||||
import { loadSqliteVecExtension } from "./sqlite-vec.js";
|
import { VectorManager } from "./vector/vector-manager.js";
|
||||||
|
|
||||||
type MemorySource = "memory" | "sessions";
|
type MemorySource = "memory" | "sessions";
|
||||||
|
|
||||||
@ -102,7 +102,6 @@ const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|||||||
const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
|
const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
|
||||||
const BATCH_FAILURE_LIMIT = 2;
|
const BATCH_FAILURE_LIMIT = 2;
|
||||||
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
|
||||||
const VECTOR_LOAD_TIMEOUT_MS = 30_000;
|
|
||||||
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
|
const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
|
||||||
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
|
const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
|
||||||
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 60_000;
|
const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 60_000;
|
||||||
@ -142,19 +141,12 @@ export class MemoryIndexManager {
|
|||||||
private readonly sources: Set<MemorySource>;
|
private readonly sources: Set<MemorySource>;
|
||||||
private providerKey: string;
|
private providerKey: string;
|
||||||
private readonly cache: { enabled: boolean; maxEntries?: number };
|
private readonly cache: { enabled: boolean; maxEntries?: number };
|
||||||
private readonly vector: {
|
private vectorManager: VectorManager;
|
||||||
enabled: boolean;
|
|
||||||
available: boolean | null;
|
|
||||||
extensionPath?: string;
|
|
||||||
loadError?: string;
|
|
||||||
dims?: number;
|
|
||||||
};
|
|
||||||
private readonly fts: {
|
private readonly fts: {
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
available: boolean;
|
available: boolean;
|
||||||
loadError?: string;
|
loadError?: string;
|
||||||
};
|
};
|
||||||
private vectorReady: Promise<boolean> | null = null;
|
|
||||||
private watcher: FSWatcher | null = null;
|
private watcher: FSWatcher | null = null;
|
||||||
private watchTimer: NodeJS.Timeout | null = null;
|
private watchTimer: NodeJS.Timeout | null = null;
|
||||||
private sessionWatchTimer: NodeJS.Timeout | null = null;
|
private sessionWatchTimer: NodeJS.Timeout | null = null;
|
||||||
@ -232,15 +224,17 @@ export class MemoryIndexManager {
|
|||||||
};
|
};
|
||||||
this.fts = { enabled: params.settings.query.hybrid.enabled, available: false };
|
this.fts = { enabled: params.settings.query.hybrid.enabled, available: false };
|
||||||
this.ensureSchema();
|
this.ensureSchema();
|
||||||
this.vector = {
|
|
||||||
enabled: params.settings.store.vector.enabled,
|
|
||||||
available: null,
|
|
||||||
extensionPath: params.settings.store.vector.extensionPath,
|
|
||||||
};
|
|
||||||
const meta = this.readMeta();
|
const meta = this.readMeta();
|
||||||
if (meta?.vectorDims) {
|
this.vectorManager = new VectorManager(
|
||||||
this.vector.dims = meta.vectorDims;
|
this.db,
|
||||||
}
|
{
|
||||||
|
enabled: params.settings.store.vector.enabled,
|
||||||
|
extensionPath: params.settings.store.vector.extensionPath,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dims: meta?.vectorDims,
|
||||||
|
},
|
||||||
|
);
|
||||||
this.ensureWatcher();
|
this.ensureWatcher();
|
||||||
this.ensureSessionListener();
|
this.ensureSessionListener();
|
||||||
this.ensureIntervalSync();
|
this.ensureIntervalSync();
|
||||||
@ -519,13 +513,16 @@ export class MemoryIndexManager {
|
|||||||
fallback: this.fallbackReason
|
fallback: this.fallbackReason
|
||||||
? { from: this.fallbackFrom ?? "local", reason: this.fallbackReason }
|
? { from: this.fallbackFrom ?? "local", reason: this.fallbackReason }
|
||||||
: undefined,
|
: undefined,
|
||||||
vector: {
|
vector: (() => {
|
||||||
enabled: this.vector.enabled,
|
const state = this.vectorManager.getState();
|
||||||
available: this.vector.available ?? undefined,
|
return {
|
||||||
extensionPath: this.vector.extensionPath,
|
enabled: this.settings.store.vector.enabled,
|
||||||
loadError: this.vector.loadError,
|
available: state.available ?? undefined,
|
||||||
dims: this.vector.dims,
|
extensionPath: state.extensionPath,
|
||||||
},
|
loadError: state.loadError,
|
||||||
|
dims: state.dims,
|
||||||
|
};
|
||||||
|
})(),
|
||||||
batch: {
|
batch: {
|
||||||
enabled: this.batch.enabled,
|
enabled: this.batch.enabled,
|
||||||
failures: this.batchFailureCount,
|
failures: this.batchFailureCount,
|
||||||
@ -541,8 +538,7 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async probeVectorAvailability(): Promise<boolean> {
|
async probeVectorAvailability(): Promise<boolean> {
|
||||||
if (!this.vector.enabled) return false;
|
return this.vectorManager.ensureReady();
|
||||||
return this.ensureVectorReady();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async probeEmbeddingAvailability(): Promise<{ ok: boolean; error?: string }> {
|
async probeEmbeddingAvailability(): Promise<{ ok: boolean; error?: string }> {
|
||||||
@ -583,76 +579,7 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
private async ensureVectorReady(dimensions?: number): Promise<boolean> {
|
||||||
if (!this.vector.enabled) return false;
|
return this.vectorManager.ensureReady(dimensions);
|
||||||
if (!this.vectorReady) {
|
|
||||||
this.vectorReady = this.withTimeout(
|
|
||||||
this.loadVectorExtension(),
|
|
||||||
VECTOR_LOAD_TIMEOUT_MS,
|
|
||||||
`sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1000)}s`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let ready = false;
|
|
||||||
try {
|
|
||||||
ready = await this.vectorReady;
|
|
||||||
} catch (err) {
|
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
|
||||||
this.vector.available = false;
|
|
||||||
this.vector.loadError = message;
|
|
||||||
this.vectorReady = null;
|
|
||||||
log.warn(`sqlite-vec unavailable: ${message}`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (ready && typeof dimensions === "number" && dimensions > 0) {
|
|
||||||
this.ensureVectorTable(dimensions);
|
|
||||||
}
|
|
||||||
return ready;
|
|
||||||
}
|
|
||||||
|
|
||||||
private async loadVectorExtension(): Promise<boolean> {
|
|
||||||
if (this.vector.available !== null) return this.vector.available;
|
|
||||||
if (!this.vector.enabled) {
|
|
||||||
this.vector.available = false;
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
const resolvedPath = this.vector.extensionPath?.trim()
|
|
||||||
? resolveUserPath(this.vector.extensionPath)
|
|
||||||
: undefined;
|
|
||||||
const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath });
|
|
||||||
if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
|
||||||
this.vector.extensionPath = loaded.extensionPath;
|
|
||||||
this.vector.available = true;
|
|
||||||
return true;
|
|
||||||
} catch (err) {
|
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
|
||||||
this.vector.available = false;
|
|
||||||
this.vector.loadError = message;
|
|
||||||
log.warn(`sqlite-vec unavailable: ${message}`);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private ensureVectorTable(dimensions: number): void {
|
|
||||||
if (this.vector.dims === dimensions) return;
|
|
||||||
if (this.vector.dims && this.vector.dims !== dimensions) {
|
|
||||||
this.dropVectorTable();
|
|
||||||
}
|
|
||||||
this.db.exec(
|
|
||||||
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
|
|
||||||
` id TEXT PRIMARY KEY,\n` +
|
|
||||||
` embedding FLOAT[${dimensions}]\n` +
|
|
||||||
`)`,
|
|
||||||
);
|
|
||||||
this.vector.dims = dimensions;
|
|
||||||
}
|
|
||||||
|
|
||||||
private dropVectorTable(): void {
|
|
||||||
try {
|
|
||||||
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
|
|
||||||
} catch (err) {
|
|
||||||
const message = err instanceof Error ? err.message : String(err);
|
|
||||||
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
|
private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
|
||||||
@ -1308,10 +1235,8 @@ export class MemoryIndexManager {
|
|||||||
const originalState = {
|
const originalState = {
|
||||||
ftsAvailable: this.fts.available,
|
ftsAvailable: this.fts.available,
|
||||||
ftsError: this.fts.loadError,
|
ftsError: this.fts.loadError,
|
||||||
vectorAvailable: this.vector.available,
|
db: this.db,
|
||||||
vectorLoadError: this.vector.loadError,
|
providerKey: this.providerKey,
|
||||||
vectorDims: this.vector.dims,
|
|
||||||
vectorReady: this.vectorReady,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const restoreOriginalState = () => {
|
const restoreOriginalState = () => {
|
||||||
@ -1322,17 +1247,19 @@ export class MemoryIndexManager {
|
|||||||
}
|
}
|
||||||
this.fts.available = originalState.ftsAvailable;
|
this.fts.available = originalState.ftsAvailable;
|
||||||
this.fts.loadError = originalState.ftsError;
|
this.fts.loadError = originalState.ftsError;
|
||||||
this.vector.available = originalDbClosed ? null : originalState.vectorAvailable;
|
this.providerKey = originalState.providerKey;
|
||||||
this.vector.loadError = originalState.vectorLoadError;
|
// VectorManager will be recreated with the restored db connection
|
||||||
this.vector.dims = originalState.vectorDims;
|
|
||||||
this.vectorReady = originalDbClosed ? null : originalState.vectorReady;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
this.db = tempDb;
|
this.db = tempDb;
|
||||||
this.vectorReady = null;
|
// Reset VectorManager on meta change
|
||||||
this.vector.available = null;
|
this.vectorManager = new VectorManager(
|
||||||
this.vector.loadError = undefined;
|
this.db,
|
||||||
this.vector.dims = undefined;
|
{
|
||||||
|
enabled: this.settings.store.vector.enabled,
|
||||||
|
extensionPath: this.settings.store.vector.extensionPath,
|
||||||
|
},
|
||||||
|
);
|
||||||
this.fts.available = false;
|
this.fts.available = false;
|
||||||
this.fts.loadError = undefined;
|
this.fts.loadError = undefined;
|
||||||
this.ensureSchema();
|
this.ensureSchema();
|
||||||
@ -1369,8 +1296,9 @@ export class MemoryIndexManager {
|
|||||||
chunkTokens: this.settings.chunking.tokens,
|
chunkTokens: this.settings.chunking.tokens,
|
||||||
chunkOverlap: this.settings.chunking.overlap,
|
chunkOverlap: this.settings.chunking.overlap,
|
||||||
};
|
};
|
||||||
if (this.vector.available && this.vector.dims) {
|
const vectorState = this.vectorManager.getState();
|
||||||
nextMeta.vectorDims = this.vector.dims;
|
if (vectorState.available && vectorState.dims) {
|
||||||
|
nextMeta.vectorDims = vectorState.dims;
|
||||||
}
|
}
|
||||||
|
|
||||||
this.writeMeta(nextMeta);
|
this.writeMeta(nextMeta);
|
||||||
@ -1383,11 +1311,18 @@ export class MemoryIndexManager {
|
|||||||
await this.swapIndexFiles(dbPath, tempDbPath);
|
await this.swapIndexFiles(dbPath, tempDbPath);
|
||||||
|
|
||||||
this.db = this.openDatabaseAtPath(dbPath);
|
this.db = this.openDatabaseAtPath(dbPath);
|
||||||
this.vectorReady = null;
|
|
||||||
this.vector.available = null;
|
|
||||||
this.vector.loadError = undefined;
|
|
||||||
this.ensureSchema();
|
this.ensureSchema();
|
||||||
this.vector.dims = nextMeta.vectorDims;
|
// Recreate VectorManager with new database connection
|
||||||
|
this.vectorManager = new VectorManager(
|
||||||
|
this.db,
|
||||||
|
{
|
||||||
|
enabled: this.settings.store.vector.enabled,
|
||||||
|
extensionPath: this.settings.store.vector.extensionPath,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
dims: nextMeta.vectorDims,
|
||||||
|
},
|
||||||
|
);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
try {
|
try {
|
||||||
this.db.close();
|
this.db.close();
|
||||||
@ -1406,8 +1341,7 @@ export class MemoryIndexManager {
|
|||||||
this.db.exec(`DELETE FROM ${FTS_TABLE}`);
|
this.db.exec(`DELETE FROM ${FTS_TABLE}`);
|
||||||
} catch { }
|
} catch { }
|
||||||
}
|
}
|
||||||
this.dropVectorTable();
|
this.vectorManager.dropVectorTable();
|
||||||
this.vector.dims = undefined;
|
|
||||||
this.sessionsDirtyFiles.clear();
|
this.sessionsDirtyFiles.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
170
src/memory/vector/vector-manager.ts
Normal file
170
src/memory/vector/vector-manager.ts
Normal file
@ -0,0 +1,170 @@
|
|||||||
|
import type { DatabaseSync } from "node:sqlite";
|
||||||
|
|
||||||
|
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||||
|
import { resolveUserPath } from "../../utils.js";
|
||||||
|
import { loadSqliteVecExtension } from "../sqlite-vec.js";
|
||||||
|
|
||||||
|
const log = createSubsystemLogger("memory:vector");
|
||||||
|
|
||||||
|
const VECTOR_LOAD_TIMEOUT_MS = 30_000;
|
||||||
|
const VECTOR_TABLE = "chunks_vec";
|
||||||
|
|
||||||
|
export type VectorConfig = {
|
||||||
|
enabled: boolean;
|
||||||
|
extensionPath?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type VectorState = {
|
||||||
|
available: boolean | null;
|
||||||
|
extensionPath?: string;
|
||||||
|
loadError?: string;
|
||||||
|
dims?: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Manages sqlite-vec extension loading and vector table lifecycle.
|
||||||
|
*/
|
||||||
|
export class VectorManager {
|
||||||
|
private state: VectorState;
|
||||||
|
private loadPromise: Promise<boolean> | null = null;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
private readonly db: DatabaseSync,
|
||||||
|
private readonly config: VectorConfig,
|
||||||
|
initialState?: Partial<VectorState>,
|
||||||
|
) {
|
||||||
|
this.state = {
|
||||||
|
available: null,
|
||||||
|
extensionPath: config.extensionPath,
|
||||||
|
...initialState,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures the vector extension is loaded and ready.
|
||||||
|
* Optionally creates/recreates the vector table if dimensions are provided.
|
||||||
|
*/
|
||||||
|
async ensureReady(dimensions?: number): Promise<boolean> {
|
||||||
|
if (!this.config.enabled) return false;
|
||||||
|
|
||||||
|
if (!this.loadPromise) {
|
||||||
|
this.loadPromise = this.withTimeout(
|
||||||
|
this.loadExtension(),
|
||||||
|
VECTOR_LOAD_TIMEOUT_MS,
|
||||||
|
`sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1000)}s`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let ready = false;
|
||||||
|
try {
|
||||||
|
ready = await this.loadPromise;
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
this.state.available = false;
|
||||||
|
this.state.loadError = message;
|
||||||
|
this.loadPromise = null;
|
||||||
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ready && typeof dimensions === "number" && dimensions > 0) {
|
||||||
|
this.ensureVectorTable(dimensions);
|
||||||
|
}
|
||||||
|
|
||||||
|
return ready;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Loads the sqlite-vec extension.
|
||||||
|
*/
|
||||||
|
private async loadExtension(): Promise<boolean> {
|
||||||
|
if (this.state.available !== null) return this.state.available;
|
||||||
|
|
||||||
|
if (!this.config.enabled) {
|
||||||
|
this.state.available = false;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const resolvedPath = this.config.extensionPath?.trim()
|
||||||
|
? resolveUserPath(this.config.extensionPath)
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
const loaded = await loadSqliteVecExtension({
|
||||||
|
db: this.db,
|
||||||
|
extensionPath: resolvedPath,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!loaded.ok) {
|
||||||
|
throw new Error(loaded.error ?? "unknown sqlite-vec load error");
|
||||||
|
}
|
||||||
|
|
||||||
|
this.state.extensionPath = loaded.extensionPath;
|
||||||
|
this.state.available = true;
|
||||||
|
return true;
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
this.state.available = false;
|
||||||
|
this.state.loadError = message;
|
||||||
|
log.warn(`sqlite-vec unavailable: ${message}`);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Ensures the vector table exists with the specified dimensions.
|
||||||
|
* Drops and recreates if dimensions change.
|
||||||
|
*/
|
||||||
|
private ensureVectorTable(dimensions: number): void {
|
||||||
|
if (this.state.dims === dimensions) return;
|
||||||
|
|
||||||
|
if (this.state.dims && this.state.dims !== dimensions) {
|
||||||
|
this.dropVectorTable();
|
||||||
|
}
|
||||||
|
|
||||||
|
this.db.exec(
|
||||||
|
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
|
||||||
|
` id TEXT PRIMARY KEY,\n` +
|
||||||
|
` embedding FLOAT[${dimensions}]\n` +
|
||||||
|
`)`,
|
||||||
|
);
|
||||||
|
|
||||||
|
this.state.dims = dimensions;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drops the vector table.
|
||||||
|
*/
|
||||||
|
dropVectorTable(): void {
|
||||||
|
try {
|
||||||
|
this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
|
||||||
|
this.state.dims = undefined;
|
||||||
|
} catch (err) {
|
||||||
|
const message = err instanceof Error ? err.message : String(err);
|
||||||
|
log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the current state of the vector manager.
|
||||||
|
*/
|
||||||
|
getState(): Readonly<VectorState> {
|
||||||
|
return { ...this.state };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wraps a promise with a timeout.
|
||||||
|
*/
|
||||||
|
private async withTimeout<T>(
|
||||||
|
promise: Promise<T>,
|
||||||
|
timeoutMs: number,
|
||||||
|
message: string,
|
||||||
|
): Promise<T> {
|
||||||
|
return Promise.race([
|
||||||
|
promise,
|
||||||
|
new Promise<T>((_, reject) =>
|
||||||
|
setTimeout(() => reject(new Error(message)), timeoutMs),
|
||||||
|
),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue
Block a user