Merge 04df8f3a83 into 4583f88626

src/memory/manager.ts

@@ -45,7 +45,7 @@ import { bm25RankToScore, buildFtsQuery, mergeHybridResults } from "./hybrid.js"
 import { searchKeyword, searchVector } from "./manager-search.js";
 import { ensureMemoryIndexSchema } from "./memory-schema.js";
 import { requireNodeSqlite } from "./sqlite.js";
-import { loadSqliteVecExtension } from "./sqlite-vec.js";
+import { VectorManager } from "./vector/vector-manager.js";
 
 type MemorySource = "memory" | "sessions";
 
@@ -103,7 +103,6 @@ const EMBEDDING_RETRY_BASE_DELAY_MS = 500;
 const EMBEDDING_RETRY_MAX_DELAY_MS = 8000;
 const BATCH_FAILURE_LIMIT = 2;
 const SESSION_DELTA_READ_CHUNK_BYTES = 64 * 1024;
-const VECTOR_LOAD_TIMEOUT_MS = 30_000;
 const EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 60_000;
 const EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 60_000;
 const EMBEDDING_BATCH_TIMEOUT_REMOTE_MS = 2 * 60_000;
@@ -143,19 +142,12 @@ export class MemoryIndexManager {
   private readonly sources: Set<MemorySource>;
   private providerKey: string;
   private readonly cache: { enabled: boolean; maxEntries?: number };
-  private readonly vector: {
+  private vectorManager: VectorManager;
-    enabled: boolean;
-    available: boolean | null;
-    extensionPath?: string;
-    loadError?: string;
-    dims?: number;
-  };
   private readonly fts: {
     enabled: boolean;
     available: boolean;
     loadError?: string;
   };
-  private vectorReady: Promise<boolean> | null = null;
   private watcher: FSWatcher | null = null;
   private watchTimer: NodeJS.Timeout | null = null;
   private sessionWatchTimer: NodeJS.Timeout | null = null;
@@ -233,15 +225,17 @@ export class MemoryIndexManager {
     };
     this.fts = { enabled: params.settings.query.hybrid.enabled, available: false };
     this.ensureSchema();
-    this.vector = {
-      enabled: params.settings.store.vector.enabled,
-      available: null,
-      extensionPath: params.settings.store.vector.extensionPath,
-    };
     const meta = this.readMeta();
-    if (meta?.vectorDims) {
+    this.vectorManager = new VectorManager(
-      this.vector.dims = meta.vectorDims;
+      this.db,
-    }
+      {
+        enabled: params.settings.store.vector.enabled,
+        extensionPath: params.settings.store.vector.extensionPath,
+      },
+      {
+        dims: meta?.vectorDims,
+      },
+    );
     this.ensureWatcher();
     this.ensureSessionListener();
     this.ensureIntervalSync();
@@ -561,13 +555,16 @@ export class MemoryIndexManager {
       fallback: this.fallbackReason
         ? { from: this.fallbackFrom ?? "local", reason: this.fallbackReason }
         : undefined,
-      vector: {
+      vector: (() => {
-        enabled: this.vector.enabled,
+        const state = this.vectorManager.getState();
-        available: this.vector.available ?? undefined,
+        return {
-        extensionPath: this.vector.extensionPath,
+          enabled: this.settings.store.vector.enabled,
-        loadError: this.vector.loadError,
+          available: state.available ?? undefined,
-        dims: this.vector.dims,
+          extensionPath: state.extensionPath,
-      },
+          loadError: state.loadError,
+          dims: state.dims,
+        };
+      })(),
       batch: {
         enabled: this.batch.enabled,
         failures: this.batchFailureCount,
@@ -583,8 +580,7 @@ export class MemoryIndexManager {
   }
 
   async probeVectorAvailability(): Promise<boolean> {
-    if (!this.vector.enabled) return false;
+    return this.vectorManager.ensureReady();
-    return this.ensureVectorReady();
   }
 
   async probeEmbeddingAvailability(): Promise<{ ok: boolean; error?: string }> {
@@ -625,76 +621,7 @@ export class MemoryIndexManager {
   }
 
   private async ensureVectorReady(dimensions?: number): Promise<boolean> {
-    if (!this.vector.enabled) return false;
+    return this.vectorManager.ensureReady(dimensions);
-    if (!this.vectorReady) {
-      this.vectorReady = this.withTimeout(
-        this.loadVectorExtension(),
-        VECTOR_LOAD_TIMEOUT_MS,
-        `sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1000)}s`,
-      );
-    }
-    let ready = false;
-    try {
-      ready = await this.vectorReady;
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      this.vector.available = false;
-      this.vector.loadError = message;
-      this.vectorReady = null;
-      log.warn(`sqlite-vec unavailable: ${message}`);
-      return false;
-    }
-    if (ready && typeof dimensions === "number" && dimensions > 0) {
-      this.ensureVectorTable(dimensions);
-    }
-    return ready;
-  }
-
-  private async loadVectorExtension(): Promise<boolean> {
-    if (this.vector.available !== null) return this.vector.available;
-    if (!this.vector.enabled) {
-      this.vector.available = false;
-      return false;
-    }
-    try {
-      const resolvedPath = this.vector.extensionPath?.trim()
-        ? resolveUserPath(this.vector.extensionPath)
-        : undefined;
-      const loaded = await loadSqliteVecExtension({ db: this.db, extensionPath: resolvedPath });
-      if (!loaded.ok) throw new Error(loaded.error ?? "unknown sqlite-vec load error");
-      this.vector.extensionPath = loaded.extensionPath;
-      this.vector.available = true;
-      return true;
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      this.vector.available = false;
-      this.vector.loadError = message;
-      log.warn(`sqlite-vec unavailable: ${message}`);
-      return false;
-    }
-  }
-
-  private ensureVectorTable(dimensions: number): void {
-    if (this.vector.dims === dimensions) return;
-    if (this.vector.dims && this.vector.dims !== dimensions) {
-      this.dropVectorTable();
-    }
-    this.db.exec(
-      `CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
-        `  id TEXT PRIMARY KEY,\n` +
-        `  embedding FLOAT[${dimensions}]\n` +
-        `)`,
-    );
-    this.vector.dims = dimensions;
-  }
-
-  private dropVectorTable(): void {
-    try {
-      this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
-    } catch (err) {
-      const message = err instanceof Error ? err.message : String(err);
-      log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
-    }
   }
 
   private buildSourceFilter(alias?: string): { sql: string; params: MemorySource[] } {
@@ -1362,10 +1289,8 @@ export class MemoryIndexManager {
     const originalState = {
       ftsAvailable: this.fts.available,
       ftsError: this.fts.loadError,
-      vectorAvailable: this.vector.available,
+      db: this.db,
-      vectorLoadError: this.vector.loadError,
+      providerKey: this.providerKey,
-      vectorDims: this.vector.dims,
-      vectorReady: this.vectorReady,
     };
 
     const restoreOriginalState = () => {
@@ -1376,17 +1301,19 @@ export class MemoryIndexManager {
       }
       this.fts.available = originalState.ftsAvailable;
       this.fts.loadError = originalState.ftsError;
-      this.vector.available = originalDbClosed ? null : originalState.vectorAvailable;
+      this.providerKey = originalState.providerKey;
-      this.vector.loadError = originalState.vectorLoadError;
+      // VectorManager will be recreated with the restored db connection
-      this.vector.dims = originalState.vectorDims;
-      this.vectorReady = originalDbClosed ? null : originalState.vectorReady;
     };
 
     this.db = tempDb;
-    this.vectorReady = null;
+    // Reset VectorManager on meta change
-    this.vector.available = null;
+    this.vectorManager = new VectorManager(
-    this.vector.loadError = undefined;
+      this.db,
-    this.vector.dims = undefined;
+      {
+        enabled: this.settings.store.vector.enabled,
+        extensionPath: this.settings.store.vector.extensionPath,
+      },
+    );
     this.fts.available = false;
     this.fts.loadError = undefined;
     this.ensureSchema();
@@ -1423,8 +1350,9 @@ export class MemoryIndexManager {
         chunkTokens: this.settings.chunking.tokens,
         chunkOverlap: this.settings.chunking.overlap,
       };
-      if (this.vector.available && this.vector.dims) {
+      const vectorState = this.vectorManager.getState();
-        nextMeta.vectorDims = this.vector.dims;
+      if (vectorState.available && vectorState.dims) {
+        nextMeta.vectorDims = vectorState.dims;
       }
 
       this.writeMeta(nextMeta);
@@ -1437,11 +1365,18 @@ export class MemoryIndexManager {
       await this.swapIndexFiles(dbPath, tempDbPath);
 
       this.db = this.openDatabaseAtPath(dbPath);
-      this.vectorReady = null;
-      this.vector.available = null;
-      this.vector.loadError = undefined;
       this.ensureSchema();
-      this.vector.dims = nextMeta.vectorDims;
+      // Recreate VectorManager with new database connection
+      this.vectorManager = new VectorManager(
+        this.db,
+        {
+          enabled: this.settings.store.vector.enabled,
+          extensionPath: this.settings.store.vector.extensionPath,
+        },
+        {
+          dims: nextMeta.vectorDims,
+        },
+      );
     } catch (err) {
       try {
         this.db.close();
@@ -1460,8 +1395,7 @@ export class MemoryIndexManager {
         this.db.exec(`DELETE FROM ${FTS_TABLE}`);
       } catch { }
     }
-    this.dropVectorTable();
+    this.vectorManager.dropVectorTable();
-    this.vector.dims = undefined;
     this.sessionsDirtyFiles.clear();
   }
 

src/memory/vector/vector-manager.ts

@@ -0,0 +1,170 @@
+import type { DatabaseSync } from "node:sqlite";
+
+import { createSubsystemLogger } from "../../logging/subsystem.js";
+import { resolveUserPath } from "../../utils.js";
+import { loadSqliteVecExtension } from "../sqlite-vec.js";
+
+const log = createSubsystemLogger("memory:vector");
+
+const VECTOR_LOAD_TIMEOUT_MS = 30_000;
+const VECTOR_TABLE = "chunks_vec";
+
+export type VectorConfig = {
+  enabled: boolean;
+  extensionPath?: string;
+};
+
+export type VectorState = {
+  available: boolean | null;
+  extensionPath?: string;
+  loadError?: string;
+  dims?: number;
+};
+
+/**
+ * Manages sqlite-vec extension loading and vector table lifecycle.
+ */
+export class VectorManager {
+  private state: VectorState;
+  private loadPromise: Promise<boolean> | null = null;
+
+  constructor(
+    private readonly db: DatabaseSync,
+    private readonly config: VectorConfig,
+    initialState?: Partial<VectorState>,
+  ) {
+    this.state = {
+      available: null,
+      extensionPath: config.extensionPath,
+      ...initialState,
+    };
+  }
+
+  /**
+   * Ensures the vector extension is loaded and ready.
+   * Optionally creates/recreates the vector table if dimensions are provided.
+   */
+  async ensureReady(dimensions?: number): Promise<boolean> {
+    if (!this.config.enabled) return false;
+
+    if (!this.loadPromise) {
+      this.loadPromise = this.withTimeout(
+        this.loadExtension(),
+        VECTOR_LOAD_TIMEOUT_MS,
+        `sqlite-vec load timed out after ${Math.round(VECTOR_LOAD_TIMEOUT_MS / 1000)}s`,
+      );
+    }
+
+    let ready = false;
+    try {
+      ready = await this.loadPromise;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      this.state.available = false;
+      this.state.loadError = message;
+      this.loadPromise = null;
+      log.warn(`sqlite-vec unavailable: ${message}`);
+      return false;
+    }
+
+    if (ready && typeof dimensions === "number" && dimensions > 0) {
+      this.ensureVectorTable(dimensions);
+    }
+
+    return ready;
+  }
+
+  /**
+   * Loads the sqlite-vec extension.
+   */
+  private async loadExtension(): Promise<boolean> {
+    if (this.state.available !== null) return this.state.available;
+
+    if (!this.config.enabled) {
+      this.state.available = false;
+      return false;
+    }
+
+    try {
+      const resolvedPath = this.config.extensionPath?.trim()
+        ? resolveUserPath(this.config.extensionPath)
+        : undefined;
+
+      const loaded = await loadSqliteVecExtension({
+        db: this.db,
+        extensionPath: resolvedPath,
+      });
+
+      if (!loaded.ok) {
+        throw new Error(loaded.error ?? "unknown sqlite-vec load error");
+      }
+
+      this.state.extensionPath = loaded.extensionPath;
+      this.state.available = true;
+      return true;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      this.state.available = false;
+      this.state.loadError = message;
+      log.warn(`sqlite-vec unavailable: ${message}`);
+      return false;
+    }
+  }
+
+  /**
+   * Ensures the vector table exists with the specified dimensions.
+   * Drops and recreates if dimensions change.
+   */
+  private ensureVectorTable(dimensions: number): void {
+    if (this.state.dims === dimensions) return;
+
+    if (this.state.dims && this.state.dims !== dimensions) {
+      this.dropVectorTable();
+    }
+
+    this.db.exec(
+      `CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE} USING vec0(\n` +
+        `  id TEXT PRIMARY KEY,\n` +
+        `  embedding FLOAT[${dimensions}]\n` +
+        `)`,
+    );
+
+    this.state.dims = dimensions;
+  }
+
+  /**
+   * Drops the vector table.
+   */
+  dropVectorTable(): void {
+    try {
+      this.db.exec(`DROP TABLE IF EXISTS ${VECTOR_TABLE}`);
+      this.state.dims = undefined;
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      log.debug(`Failed to drop ${VECTOR_TABLE}: ${message}`);
+    }
+  }
+
+  /**
+   * Gets the current state of the vector manager.
+   */
+  getState(): Readonly<VectorState> {
+    return { ...this.state };
+  }
+
+  /**
+   * Wraps a promise with a timeout.
+   */
+  private async withTimeout<T>(
+    promise: Promise<T>,
+    timeoutMs: number,
+    message: string,
+  ): Promise<T> {
+    return Promise.race([
+      promise,
+      new Promise<T>((_, reject) =>
+        setTimeout(() => reject(new Error(message)), timeoutMs),
+      ),
+    ]);
+  }
+}