From c1aabbb7ee227fec51832c8a1fb6100aed16aab5 Mon Sep 17 00:00:00 2001 From: vasilije Date: Mon, 26 Jan 2026 22:09:00 +0100 Subject: [PATCH 1/8] added cognee draft --- docs/memory-cognee.md | 340 ++++++++++++++++++++++++++++ examples/cognee-config.yaml | 47 ++++ examples/cognee-docker-compose.yaml | 77 +++++++ src/agents/memory-search.ts | 29 ++- src/config/types.tools.ts | 23 +- src/memory/cognee-client.test.ts | 239 +++++++++++++++++++ src/memory/cognee-client.ts | 299 ++++++++++++++++++++++++ src/memory/cognee-provider.test.ts | 153 +++++++++++++ src/memory/cognee-provider.ts | 331 +++++++++++++++++++++++++++ src/memory/search-manager.ts | 22 +- 10 files changed, 1555 insertions(+), 5 deletions(-) create mode 100644 docs/memory-cognee.md create mode 100644 examples/cognee-config.yaml create mode 100644 examples/cognee-docker-compose.yaml create mode 100644 src/memory/cognee-client.test.ts create mode 100644 src/memory/cognee-client.ts create mode 100644 src/memory/cognee-provider.test.ts create mode 100644 src/memory/cognee-provider.ts diff --git a/docs/memory-cognee.md b/docs/memory-cognee.md new file mode 100644 index 000000000..6980513f5 --- /dev/null +++ b/docs/memory-cognee.md @@ -0,0 +1,340 @@ +--- +summary: "Cognee knowledge graph memory: setup, Docker config, and usage" +read_when: + - Setting up Cognee memory provider + - Configuring knowledge graph memory + - Running Cognee with Docker +--- + +# Cognee Memory Provider + +Clawdbot supports **Cognee** as an optional memory provider. Unlike the default SQLite-based vector memory, Cognee builds a knowledge graph with entity extraction and semantic relationships, providing richer contextual memory for your AI agent. + +## What is Cognee? + +Cognee is an AI memory framework that: +- Extracts entities (people, places, concepts) from documents +- Builds a knowledge graph of relationships +- Enables semantic search with LLM-powered reasoning +- Supports multiple search modes (insights, chunks, summaries) + +Learn more at [docs.cognee.ai](https://docs.cognee.ai/). + +## Setup Options + +### Option 1: Local Docker (Recommended) + +Run Cognee locally using Docker Compose: + +**Step 1: Create docker-compose.yml** + +```yaml +version: '3.8' + +services: + cognee: + image: topoteretes/cognee:latest + container_name: cognee + ports: + - "8000:8000" + environment: + # Optional: Set API key for authentication + - COGNEE_API_KEY=your-local-api-key + # Database configuration + - DATABASE_URL=postgresql://cognee:cognee@postgres:5432/cognee + volumes: + - cognee_data:/app/data + depends_on: + - postgres + restart: unless-stopped + + postgres: + image: postgres:15-alpine + container_name: cognee-postgres + environment: + - POSTGRES_USER=cognee + - POSTGRES_PASSWORD=cognee + - POSTGRES_DB=cognee + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + +volumes: + cognee_data: + postgres_data: +``` + +**Step 2: Start Cognee** + +```bash +docker-compose up -d +``` + +**Step 3: Verify** + +```bash +curl http://localhost:8000/status +# Should return: {"status":"healthy"} +``` + +### Option 2: Cognee Cloud + +Use the hosted Cognee service: + +1. Sign up at [platform.cognee.ai](https://platform.cognee.ai/) +2. Get your API key from the dashboard +3. Use base URL: `https://cognee--cognee-saas-backend-serve.modal.run` + +## Configuration + +Add Cognee memory configuration to your `~/.clawdbot/config.yaml`: + +### Basic Configuration (Docker Local) + +```yaml +agents: + defaults: + memorySearch: + enabled: true + provider: cognee + sources: [memory] # or [memory, sessions] + cognee: + baseUrl: http://localhost:8000 + datasetName: clawdbot + searchType: insights # or "chunks", "summaries" + maxResults: 6 + autoCognify: true +``` + +### Cloud Configuration + +```yaml +agents: + defaults: + memorySearch: + enabled: true + provider: cognee + sources: [memory, sessions] + cognee: + baseUrl: https://cognee--cognee-saas-backend-serve.modal.run + apiKey: your-api-key-here # Required for cloud + datasetName: clawdbot + searchType: insights + maxResults: 8 + autoCognify: true + timeoutSeconds: 60 +``` + +## Configuration Options + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `baseUrl` | string | `http://localhost:8000` | Cognee API endpoint | +| `apiKey` | string | - | API key (required for cloud, optional for local) | +| `datasetName` | string | `"clawdbot"` | Dataset name for organizing memories | +| `searchType` | string | `"insights"` | Search mode: `insights`, `chunks`, or `summaries` | +| `maxResults` | number | `6` | Maximum search results returned | +| `autoCognify` | boolean | `true` | Auto-process documents after adding | +| `cognifyBatchSize` | number | `100` | Batch size for processing | +| `timeoutSeconds` | number | `30` | Request timeout in seconds | + +## Search Types + +Cognee offers three search modes: + +### Insights (Recommended) +Best for: **High-level understanding and reasoning** +- Returns AI-generated insights from knowledge graph +- Combines multiple related facts +- Good for: "What projects am I working on?" or "Summarize my notes about X" + +### Chunks +Best for: **Specific text matching** +- Returns raw document chunks +- Similar to traditional vector search +- Good for: Finding exact quotes or specific information + +### Summaries +Best for: **Document overviews** +- Returns condensed summaries +- Good for: Quick scanning of content + +## Usage + +### Memory Files + +Cognee automatically syncs your memory files: +- `MEMORY.md` or `memory.md` in workspace root +- All `*.md` files in `memory/` directory + +### Session Transcripts (Optional) + +Enable session memory to index conversation history: + +```yaml +agents: + defaults: + memorySearch: + provider: cognee + sources: [memory, sessions] # Include sessions + experimental: + sessionMemory: true +``` + +### Manual Sync + +Force a memory sync: + +```bash +# Not yet implemented - coming soon +clawdbot memory sync --provider cognee +``` + +### Check Status + +```bash +# Not yet implemented - coming soon +clawdbot memory status --provider cognee +``` + +## How It Works + +1. **Add**: Memory files are sent to Cognee with metadata +2. **Cognify**: Cognee processes documents: + - Extracts entities (people, places, concepts) + - Identifies relationships + - Builds knowledge graph +3. **Search**: Agent queries use semantic search: + - Searches knowledge graph + - Returns relevant insights/chunks/summaries + - Includes metadata and scores + +## Comparison: Cognee vs SQLite Memory + +| Feature | Cognee | SQLite (Default) | +|---------|--------|------------------| +| **Setup** | Requires Docker/cloud | Built-in, no setup | +| **Offline** | No (needs service) | Yes (fully local) | +| **Search** | Knowledge graph + LLM | Vector + BM25 hybrid | +| **Entities** | Extracted automatically | Not available | +| **Relationships** | Yes (graph-based) | No | +| **Speed** | Slower (API calls) | Faster (local DB) | +| **Memory** | Stored externally | SQLite file | +| **Best for** | Rich context, reasoning | Fast lookup, privacy | + +## Troubleshooting + +### Connection Failed + +**Error**: `Failed to connect to Cognee at http://localhost:8000` + +**Solutions**: +1. Verify Docker is running: `docker ps | grep cognee` +2. Check Cognee logs: `docker logs cognee` +3. Test manually: `curl http://localhost:8000/status` +4. Ensure port 8000 is not blocked + +### Slow Performance + +**Solutions**: +1. Reduce `maxResults` (try 3-5 instead of 10+) +2. Use `searchType: "chunks"` for faster results +3. Set `autoCognify: false` and cognify manually +4. Check Docker resource limits + +### Out of Memory + +**Solutions**: +1. Increase Docker memory limit (Docker Desktop settings) +2. Reduce `cognifyBatchSize` (try 50 instead of 100) +3. Process fewer files at once +4. Clear old datasets via Cognee API + +## Advanced Configuration + +### Per-Agent Override + +```yaml +agents: + defaults: + memorySearch: + provider: openai # Default for all agents + + agents: + research-bot: + memorySearch: + provider: cognee # Override for this agent + cognee: + searchType: insights + maxResults: 10 +``` + +### Hybrid Setup (Not Yet Supported) + +Future versions may support using both Cognee and SQLite: +- Cognee for semantic understanding +- SQLite for fast local lookup + +## Docker Production Tips + +### Health Checks + +Add health checks to docker-compose.yml: + +```yaml +services: + cognee: + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s +``` + +### Resource Limits + +```yaml +services: + cognee: + deploy: + resources: + limits: + cpus: '2' + memory: 4G + reservations: + cpus: '1' + memory: 2G +``` + +### Persistent Storage + +Mount volumes for persistence: + +```yaml +volumes: + - ./cognee_data:/app/data + - ./cognee_logs:/app/logs +``` + +## Roadmap + +Planned features: +- [ ] `clawdbot memory status --provider cognee` command +- [ ] `clawdbot memory sync --provider cognee` command +- [ ] Hybrid mode (Cognee + SQLite) +- [ ] Graph visualization export +- [ ] Manual entity management + +## Resources + +- [Cognee Documentation](https://docs.cognee.ai/) +- [Cognee GitHub](https://github.com/topoteretes/cognee) +- [Clawdbot Memory Guide](/memory) +- [Docker Setup Guide](/install/docker) + +## Feedback + +Cognee integration is new. Report issues at: +- Clawdbot: [github.com/clawdbot/clawdbot/issues](https://github.com/clawdbot/clawdbot/issues) +- Cognee: [github.com/topoteretes/cognee/issues](https://github.com/topoteretes/cognee/issues) diff --git a/examples/cognee-config.yaml b/examples/cognee-config.yaml new file mode 100644 index 000000000..b8abc922b --- /dev/null +++ b/examples/cognee-config.yaml @@ -0,0 +1,47 @@ +# Example Clawdbot configuration with Cognee memory provider +# Copy to ~/.clawdbot/config.yaml and customize + +agents: + defaults: + # Use Cognee for knowledge graph memory + memorySearch: + enabled: true + provider: cognee + sources: [memory, sessions] # Index both memory files and conversation history + + # Cognee-specific configuration + cognee: + # Local Docker setup (default) + baseUrl: http://localhost:8000 + + # For Cognee Cloud, use: + # baseUrl: https://cognee--cognee-saas-backend-serve.modal.run + # apiKey: your-api-key-here + + # Dataset name for organizing memories + datasetName: clawdbot + + # Search mode: "insights" (recommended), "chunks", or "summaries" + searchType: insights + + # Maximum search results to return + maxResults: 6 + + # Automatically process documents after adding (recommended) + autoCognify: true + + # Batch size for processing multiple documents + cognifyBatchSize: 100 + + # Request timeout in seconds + timeoutSeconds: 30 + + # Enable experimental session memory indexing + experimental: + sessionMemory: true + +# To use default SQLite memory instead, set: +# agents: +# defaults: +# memorySearch: +# provider: auto # or "openai", "gemini", "local" diff --git a/examples/cognee-docker-compose.yaml b/examples/cognee-docker-compose.yaml new file mode 100644 index 000000000..045f8a7dc --- /dev/null +++ b/examples/cognee-docker-compose.yaml @@ -0,0 +1,77 @@ +# Docker Compose configuration for running Cognee locally with Clawdbot +# +# Usage: +# 1. Copy this file to your preferred location +# 2. Run: docker-compose -f cognee-docker-compose.yaml up -d +# 3. Verify: curl http://localhost:8000/status +# 4. Configure Clawdbot with baseUrl: http://localhost:8000 +# +# For production, see docs/memory-cognee.md for additional configuration + +version: '3.8' + +services: + # Cognee API server + cognee: + image: topoteretes/cognee:latest + container_name: cognee + ports: + - "8000:8000" + environment: + # Optional: Set API key for authentication + # Remove or comment out for local development without auth + - COGNEE_API_KEY=${COGNEE_API_KEY:-} + + # Database connection + - DATABASE_URL=postgresql://cognee:cognee@postgres:5432/cognee + + # Optional: Configure LLM provider for entity extraction + # Uncomment and set if you want to use specific providers + # - OPENAI_API_KEY=${OPENAI_API_KEY:-} + # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + volumes: + - cognee_data:/app/data + - cognee_logs:/app/logs + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + networks: + - cognee-network + + # PostgreSQL database for Cognee + postgres: + image: postgres:15-alpine + container_name: cognee-postgres + environment: + - POSTGRES_USER=cognee + - POSTGRES_PASSWORD=cognee + - POSTGRES_DB=cognee + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U cognee"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - cognee-network + +volumes: + cognee_data: + driver: local + cognee_logs: + driver: local + postgres_data: + driver: local + +networks: + cognee-network: + driver: bridge diff --git a/src/agents/memory-search.ts b/src/agents/memory-search.ts index c08161d4f..8fbdb2614 100644 --- a/src/agents/memory-search.ts +++ b/src/agents/memory-search.ts @@ -9,7 +9,7 @@ import { resolveAgentConfig } from "./agent-scope.js"; export type ResolvedMemorySearchConfig = { enabled: boolean; sources: Array<"memory" | "sessions">; - provider: "openai" | "local" | "gemini" | "auto"; + provider: "openai" | "local" | "gemini" | "auto" | "cognee"; remote?: { baseUrl?: string; apiKey?: string; @@ -25,12 +25,22 @@ export type ResolvedMemorySearchConfig = { experimental: { sessionMemory: boolean; }; - fallback: "openai" | "gemini" | "local" | "none"; + fallback: "openai" | "gemini" | "local" | "cognee" | "none"; model: string; local: { modelPath?: string; modelCacheDir?: string; }; + cognee?: { + baseUrl?: string; + apiKey?: string; + datasetName?: string; + searchType?: "insights" | "chunks" | "summaries"; + maxResults?: number; + timeoutSeconds?: number; + autoCognify?: boolean; + cognifyBatchSize?: number; + }; store: { driver: "sqlite"; path: string; @@ -222,6 +232,20 @@ function mergeConfig( enabled: overrides?.cache?.enabled ?? defaults?.cache?.enabled ?? DEFAULT_CACHE_ENABLED, maxEntries: overrides?.cache?.maxEntries ?? defaults?.cache?.maxEntries, }; + const cognee = + provider === "cognee" + ? { + baseUrl: overrides?.cognee?.baseUrl ?? defaults?.cognee?.baseUrl, + apiKey: overrides?.cognee?.apiKey ?? defaults?.cognee?.apiKey, + datasetName: overrides?.cognee?.datasetName ?? defaults?.cognee?.datasetName, + searchType: overrides?.cognee?.searchType ?? defaults?.cognee?.searchType, + maxResults: overrides?.cognee?.maxResults ?? defaults?.cognee?.maxResults, + timeoutSeconds: overrides?.cognee?.timeoutSeconds ?? defaults?.cognee?.timeoutSeconds, + autoCognify: overrides?.cognee?.autoCognify ?? defaults?.cognee?.autoCognify, + cognifyBatchSize: + overrides?.cognee?.cognifyBatchSize ?? defaults?.cognee?.cognifyBatchSize, + } + : undefined; const overlap = clampNumber(chunking.overlap, 0, Math.max(0, chunking.tokens - 1)); const minScore = clampNumber(query.minScore, 0, 1); @@ -244,6 +268,7 @@ function mergeConfig( fallback, model, local, + cognee, store, chunking: { tokens: Math.max(1, chunking.tokens), overlap }, sync: { diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index bb1d45bf0..7506204b9 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -232,7 +232,7 @@ export type MemorySearchConfig = { sessionMemory?: boolean; }; /** Embedding provider mode. */ - provider?: "openai" | "gemini" | "local"; + provider?: "openai" | "gemini" | "local" | "cognee"; remote?: { baseUrl?: string; apiKey?: string; @@ -251,7 +251,7 @@ export type MemorySearchConfig = { }; }; /** Fallback behavior when embeddings fail. */ - fallback?: "openai" | "gemini" | "local" | "none"; + fallback?: "openai" | "gemini" | "local" | "cognee" | "none"; /** Embedding model id (remote) or alias (local). */ model?: string; /** Local embedding settings (node-llama-cpp). */ @@ -261,6 +261,25 @@ export type MemorySearchConfig = { /** Optional cache directory for local models. */ modelCacheDir?: string; }; + /** Cognee knowledge graph memory settings. */ + cognee?: { + /** Cognee API endpoint (default: http://localhost:8000). */ + baseUrl?: string; + /** Cognee API key (required for cloud, optional for local). */ + apiKey?: string; + /** Dataset name for organizing memories (default: "clawdbot"). */ + datasetName?: string; + /** Search type: "insights", "chunks", or "summaries" (default: "insights"). */ + searchType?: "insights" | "chunks" | "summaries"; + /** Max results per search query (default: 6). */ + maxResults?: number; + /** Timeout for API requests in seconds (default: 30). */ + timeoutSeconds?: number; + /** Enable automatic cognify after adding documents (default: true). */ + autoCognify?: boolean; + /** Cognify batch size for processing (default: 100). */ + cognifyBatchSize?: number; + }; /** Index storage configuration. */ store?: { driver?: "sqlite"; diff --git a/src/memory/cognee-client.test.ts b/src/memory/cognee-client.test.ts new file mode 100644 index 000000000..6bbdb9a3a --- /dev/null +++ b/src/memory/cognee-client.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; +import { CogneeClient } from "./cognee-client.js"; +import { request } from "undici"; + +vi.mock("undici", () => ({ + request: vi.fn(), +})); + +describe("CogneeClient", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe("add", () => { + it("should add data successfully", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ + dataset_id: "test-dataset-id", + dataset_name: "test-dataset", + message: "Data added successfully", + }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient({ + baseUrl: "http://localhost:8000", + apiKey: "test-key", + }); + + const result = await client.add({ + data: "Test data", + datasetName: "test-dataset", + }); + + expect(result).toEqual({ + datasetId: "test-dataset-id", + datasetName: "test-dataset", + message: "Data added successfully", + }); + expect(request).toHaveBeenCalledWith( + "http://localhost:8000/add", + expect.objectContaining({ + method: "POST", + headers: expect.objectContaining({ + "Content-Type": "application/json", + "X-Api-Key": "test-key", + }), + }), + ); + }); + + it("should handle errors", async () => { + const mockResponse = { + statusCode: 500, + body: { + text: vi.fn().mockResolvedValue("Internal server error"), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient(); + + await expect( + client.add({ + data: "Test data", + datasetName: "test-dataset", + }), + ).rejects.toThrow("Cognee add failed with status 500"); + }); + }); + + describe("cognify", () => { + it("should run cognify successfully", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ + status: "success", + message: "Cognify completed", + }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient({ + baseUrl: "http://localhost:8000", + }); + + const result = await client.cognify({ + datasetIds: ["dataset-1"], + }); + + expect(result).toEqual({ + status: "success", + message: "Cognify completed", + }); + }); + }); + + describe("search", () => { + it("should search successfully", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ + results: [ + { + id: "result-1", + text: "Test result", + score: 0.9, + metadata: { path: "test.md" }, + }, + ], + query: "test query", + search_type: "insights", + }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient(); + + const result = await client.search({ + queryText: "test query", + searchType: "insights", + }); + + expect(result.results).toHaveLength(1); + expect(result.results[0]).toEqual({ + id: "result-1", + text: "Test result", + score: 0.9, + metadata: { path: "test.md" }, + }); + expect(result.query).toBe("test query"); + }); + + it("should use default search type", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ + results: [], + query: "test", + search_type: "insights", + }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient(); + await client.search({ queryText: "test" }); + + expect(request).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ + body: expect.stringContaining('"search_type":"insights"'), + }), + ); + }); + }); + + describe("status", () => { + it("should get status successfully", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ + status: "healthy", + version: "1.0.0", + datasets: [ + { + id: "dataset-1", + name: "test-dataset", + document_count: 10, + }, + ], + }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient(); + + const result = await client.status(); + + expect(result).toEqual({ + status: "healthy", + version: "1.0.0", + datasets: [ + { + id: "dataset-1", + name: "test-dataset", + documentCount: 10, + }, + ], + }); + }); + }); + + describe("healthCheck", () => { + it("should return true when status is successful", async () => { + const mockResponse = { + statusCode: 200, + body: { + json: vi.fn().mockResolvedValue({ status: "healthy" }), + text: vi.fn(), + }, + }; + vi.mocked(request).mockResolvedValue(mockResponse as any); + + const client = new CogneeClient(); + const result = await client.healthCheck(); + + expect(result).toBe(true); + }); + + it("should return false when status fails", async () => { + vi.mocked(request).mockRejectedValue(new Error("Connection failed")); + + const client = new CogneeClient(); + const result = await client.healthCheck(); + + expect(result).toBe(false); + }); + }); +}); diff --git a/src/memory/cognee-client.ts b/src/memory/cognee-client.ts new file mode 100644 index 000000000..c3ec3747a --- /dev/null +++ b/src/memory/cognee-client.ts @@ -0,0 +1,299 @@ +import { request } from "undici"; +import { createSubsystemLogger } from "../logging/subsystem.js"; + +const log = createSubsystemLogger("cognee"); + +const DEFAULT_BASE_URL = "http://localhost:8000"; +const DEFAULT_TIMEOUT_MS = 30_000; + +export type CogneeClientConfig = { + baseUrl?: string; + apiKey?: string; + timeoutMs?: number; +}; + +export type CogneeAddRequest = { + data: string; + datasetName?: string; + datasetId?: string; +}; + +export type CogneeAddResponse = { + datasetId: string; + datasetName: string; + message: string; +}; + +export type CogneeCognifyRequest = { + datasetIds?: string[]; +}; + +export type CogneeCognifyResponse = { + status: string; + message: string; +}; + +export type CogneeSearchRequest = { + queryText: string; + searchType?: "insights" | "chunks" | "summaries"; + datasetIds?: string[]; +}; + +export type CogneeSearchResult = { + id: string; + text: string; + score: number; + metadata?: Record<string, unknown>; +}; + +export type CogneeSearchResponse = { + results: CogneeSearchResult[]; + query: string; + searchType: string; +}; + +export type CogneeStatusResponse = { + status: string; + version?: string; + datasets?: Array<{ + id: string; + name: string; + documentCount?: number; + }>; +}; + +export class CogneeClient { + private readonly baseUrl: string; + private readonly apiKey?: string; + private readonly timeoutMs: number; + + constructor(config: CogneeClientConfig = {}) { + this.baseUrl = config.baseUrl?.replace(/\/$/, "") || DEFAULT_BASE_URL; + this.apiKey = config.apiKey; + this.timeoutMs = config.timeoutMs || DEFAULT_TIMEOUT_MS; + } + + async add(req: CogneeAddRequest): Promise<CogneeAddResponse> { + const url = `${this.baseUrl}/add`; + const headers: Record<string, string> = { + "Content-Type": "application/json", + }; + if (this.apiKey) { + headers["X-Api-Key"] = this.apiKey; + } + + log.debug("Adding data to Cognee", { + url, + datasetName: req.datasetName, + dataLength: req.data.length, + }); + + try { + const response = await request(url, { + method: "POST", + headers, + body: JSON.stringify({ + data: req.data, + dataset_name: req.datasetName, + dataset_id: req.datasetId, + }), + bodyTimeout: this.timeoutMs, + headersTimeout: this.timeoutMs, + }); + + if (response.statusCode !== 200) { + const errorText = await response.body.text(); + throw new Error( + `Cognee add failed with status ${response.statusCode}: ${errorText}`, + ); + } + + const data = (await response.body.json()) as { + dataset_id: string; + dataset_name: string; + message: string; + }; + + return { + datasetId: data.dataset_id, + datasetName: data.dataset_name, + message: data.message, + }; + } catch (error) { + log.error("Failed to add data to Cognee", { error }); + throw new Error( + `Cognee add request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + async cognify(req: CogneeCognifyRequest = {}): Promise<CogneeCognifyResponse> { + const url = `${this.baseUrl}/cognify`; + const headers: Record<string, string> = { + "Content-Type": "application/json", + }; + if (this.apiKey) { + headers["X-Api-Key"] = this.apiKey; + } + + log.debug("Running cognify", { url, datasetIds: req.datasetIds }); + + try { + const response = await request(url, { + method: "POST", + headers, + body: JSON.stringify({ + dataset_ids: req.datasetIds, + }), + bodyTimeout: this.timeoutMs, + headersTimeout: this.timeoutMs, + }); + + if (response.statusCode !== 200) { + const errorText = await response.body.text(); + throw new Error( + `Cognee cognify failed with status ${response.statusCode}: ${errorText}`, + ); + } + + const data = (await response.body.json()) as { + status: string; + message: string; + }; + + return { + status: data.status, + message: data.message, + }; + } catch (error) { + log.error("Failed to cognify", { error }); + throw new Error( + `Cognee cognify request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + async search(req: CogneeSearchRequest): Promise<CogneeSearchResponse> { + const url = `${this.baseUrl}/search`; + const headers: Record<string, string> = { + "Content-Type": "application/json", + }; + if (this.apiKey) { + headers["X-Api-Key"] = this.apiKey; + } + + log.debug("Searching Cognee", { + url, + query: req.queryText, + searchType: req.searchType, + }); + + try { + const response = await request(url, { + method: "POST", + headers, + body: JSON.stringify({ + query_text: req.queryText, + search_type: req.searchType || "insights", + dataset_ids: req.datasetIds, + }), + bodyTimeout: this.timeoutMs, + headersTimeout: this.timeoutMs, + }); + + if (response.statusCode !== 200) { + const errorText = await response.body.text(); + throw new Error( + `Cognee search failed with status ${response.statusCode}: ${errorText}`, + ); + } + + const data = (await response.body.json()) as { + results: Array<{ + id: string; + text: string; + score: number; + metadata?: Record<string, unknown>; + }>; + query: string; + search_type: string; + }; + + return { + results: data.results.map((r) => ({ + id: r.id, + text: r.text, + score: r.score, + metadata: r.metadata, + })), + query: data.query, + searchType: data.search_type, + }; + } catch (error) { + log.error("Failed to search Cognee", { error }); + throw new Error( + `Cognee search request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + async status(): Promise<CogneeStatusResponse> { + const url = `${this.baseUrl}/status`; + const headers: Record<string, string> = {}; + if (this.apiKey) { + headers["X-Api-Key"] = this.apiKey; + } + + log.debug("Checking Cognee status", { url }); + + try { + const response = await request(url, { + method: "GET", + headers, + bodyTimeout: this.timeoutMs, + headersTimeout: this.timeoutMs, + }); + + if (response.statusCode !== 200) { + const errorText = await response.body.text(); + throw new Error( + `Cognee status failed with status ${response.statusCode}: ${errorText}`, + ); + } + + const data = (await response.body.json()) as { + status: string; + version?: string; + datasets?: Array<{ + id: string; + name: string; + document_count?: number; + }>; + }; + + return { + status: data.status, + version: data.version, + datasets: data.datasets?.map((d) => ({ + id: d.id, + name: d.name, + documentCount: d.document_count, + })), + }; + } catch (error) { + log.error("Failed to get Cognee status", { error }); + throw new Error( + `Cognee status request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + + async healthCheck(): Promise<boolean> { + try { + await this.status(); + return true; + } catch { + return false; + } + } +} diff --git a/src/memory/cognee-provider.test.ts b/src/memory/cognee-provider.test.ts new file mode 100644 index 000000000..2b6bd64b6 --- /dev/null +++ b/src/memory/cognee-provider.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import { CogneeMemoryProvider } from "./cognee-provider.js"; +import type { ClawdbotConfig } from "../config/config.js"; + +vi.mock("./cognee-client.js", () => ({ + CogneeClient: vi.fn().mockImplementation(() => ({ + healthCheck: vi.fn().mockResolvedValue(true), + add: vi.fn().mockResolvedValue({ + datasetId: "test-dataset-id", + datasetName: "test-dataset", + message: "Success", + }), + cognify: vi.fn().mockResolvedValue({ + status: "success", + message: "Cognify completed", + }), + search: vi.fn().mockResolvedValue({ + results: [ + { + id: "result-1", + text: "Test result text", + score: 0.85, + metadata: { + path: "test.md", + source: "memory", + }, + }, + ], + query: "test query", + searchType: "insights", + }), + status: vi.fn().mockResolvedValue({ + status: "healthy", + version: "1.0.0", + datasets: [ + { + id: "test-dataset-id", + name: "clawdbot", + documentCount: 5, + }, + ], + }), + })), +})); + +vi.mock("./internal.js", () => ({ + listMemoryFiles: vi.fn().mockResolvedValue([]), + buildFileEntry: vi.fn(), + hashText: vi.fn().mockReturnValue("test-hash"), +})); + +vi.mock("node:fs/promises", () => ({ + default: { + readdir: vi.fn().mockResolvedValue([]), + readFile: vi.fn().mockResolvedValue("Test file content"), + stat: vi.fn().mockResolvedValue({ mtimeMs: 1234567890, size: 100 }), + }, +})); + +describe("CogneeMemoryProvider", () => { + const mockConfig: ClawdbotConfig = { + agents: { + defaults: { + workspace: "/tmp/test-workspace", + }, + }, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + describe("constructor", () => { + it("should initialize with default configuration", () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + expect(provider).toBeDefined(); + }); + + it("should initialize with custom configuration", () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"], { + baseUrl: "http://custom:8000", + apiKey: "custom-key", + datasetName: "custom-dataset", + searchType: "chunks", + maxResults: 10, + }); + + expect(provider).toBeDefined(); + }); + }); + + describe("healthCheck", () => { + it("should perform health check", async () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + const healthy = await provider.healthCheck(); + + expect(healthy).toBe(true); + }); + }); + + describe("search", () => { + it("should search and transform results", async () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + const results = await provider.search("test query"); + + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + path: "test.md", + source: "memory", + score: 0.85, + snippet: "Test result text", + }); + }); + + it("should respect maxResults setting", async () => { + const provider = new CogneeMemoryProvider( + mockConfig, + "test-agent", + ["memory"], + { maxResults: 5 }, + ); + + const results = await provider.search("test query"); + + expect(results.length).toBeLessThanOrEqual(5); + }); + }); + + describe("cognify", () => { + it("should run cognify", async () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + await expect(provider.cognify()).resolves.not.toThrow(); + }); + }); + + describe("getStatus", () => { + it("should return status information", async () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + const status = await provider.getStatus(); + + expect(status).toMatchObject({ + connected: true, + datasetName: "clawdbot", + syncedFileCount: 0, + }); + }); + }); +}); diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts new file mode 100644 index 000000000..0f6050b66 --- /dev/null +++ b/src/memory/cognee-provider.ts @@ -0,0 +1,331 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { ClawdbotConfig } from "../config/config.js"; +import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import type { MemorySearchResult } from "./index.js"; +import { + CogneeClient, + type CogneeClientConfig, + type CogneeSearchResult, +} from "./cognee-client.js"; +import { + buildFileEntry, + hashText, + listMemoryFiles, + type MemoryFileEntry, +} from "./internal.js"; + +const log = createSubsystemLogger("cognee-provider"); + +const DEFAULT_DATASET_NAME = "clawdbot"; +const DEFAULT_SEARCH_TYPE = "insights"; +const DEFAULT_MAX_RESULTS = 6; +const DEFAULT_TIMEOUT_SECONDS = 30; +const DEFAULT_AUTO_COGNIFY = true; +const DEFAULT_COGNIFY_BATCH_SIZE = 100; +const SNIPPET_MAX_CHARS = 700; + +export type CogneeProviderConfig = { + baseUrl?: string; + apiKey?: string; + datasetName?: string; + searchType?: "insights" | "chunks" | "summaries"; + maxResults?: number; + timeoutSeconds?: number; + autoCognify?: boolean; + cognifyBatchSize?: number; +}; + +export type CogneeMemorySource = "memory" | "sessions"; + +export class CogneeMemoryProvider { + private readonly client: CogneeClient; + private readonly cfg: ClawdbotConfig; + private readonly agentId: string; + private readonly workspaceDir: string; + private readonly datasetName: string; + private readonly searchType: "insights" | "chunks" | "summaries"; + private readonly maxResults: number; + private readonly autoCognify: boolean; + private readonly cognifyBatchSize: number; + private readonly sources: Set<CogneeMemorySource>; + private datasetId?: string; + private syncedFiles = new Map<string, string>(); // path -> hash + + constructor( + cfg: ClawdbotConfig, + agentId: string, + sources: Array<CogneeMemorySource>, + config: CogneeProviderConfig = {}, + ) { + const timeoutMs = (config.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS) * 1000; + const clientConfig: CogneeClientConfig = { + baseUrl: config.baseUrl, + apiKey: config.apiKey, + timeoutMs, + }; + + this.client = new CogneeClient(clientConfig); + this.cfg = cfg; + this.agentId = agentId; + this.workspaceDir = resolveAgentWorkspaceDir(cfg, agentId); + this.datasetName = config.datasetName || DEFAULT_DATASET_NAME; + this.searchType = config.searchType || DEFAULT_SEARCH_TYPE; + this.maxResults = config.maxResults || DEFAULT_MAX_RESULTS; + this.autoCognify = config.autoCognify ?? DEFAULT_AUTO_COGNIFY; + this.cognifyBatchSize = config.cognifyBatchSize || DEFAULT_COGNIFY_BATCH_SIZE; + this.sources = new Set(sources); + + log.info("Cognee memory provider initialized", { + agentId, + datasetName: this.datasetName, + searchType: this.searchType, + sources: Array.from(this.sources), + }); + } + + async healthCheck(): Promise<boolean> { + return await this.client.healthCheck(); + } + + async sync(): Promise<void> { + log.info("Starting Cognee memory sync", { agentId: this.agentId }); + + let addedCount = 0; + + // Sync memory files + if (this.sources.has("memory")) { + const memoryFiles = await this.collectMemoryFiles(); + addedCount += await this.syncFiles(memoryFiles, "memory"); + } + + // Sync session transcripts + if (this.sources.has("sessions")) { + const sessionFiles = await this.collectSessionFiles(); + addedCount += await this.syncFiles(sessionFiles, "sessions"); + } + + // Run cognify if auto-enabled and files were added + if (this.autoCognify && addedCount > 0) { + log.info("Running cognify after sync", { addedCount }); + await this.cognify(); + } + + log.info("Cognee memory sync completed", { + agentId: this.agentId, + addedCount, + }); + } + + async search(query: string): Promise<MemorySearchResult[]> { + log.debug("Searching Cognee memory", { query, searchType: this.searchType }); + + try { + const response = await this.client.search({ + queryText: query, + searchType: this.searchType, + datasetIds: this.datasetId ? [this.datasetId] : undefined, + }); + + const results: MemorySearchResult[] = response.results + .slice(0, this.maxResults) + .map((r) => this.transformResult(r)); + + log.debug("Cognee search completed", { query, resultCount: results.length }); + return results; + } catch (error) { + log.error("Cognee search failed", { error, query }); + throw error; + } + } + + async cognify(): Promise<void> { + try { + const response = await this.client.cognify({ + datasetIds: this.datasetId ? [this.datasetId] : undefined, + }); + log.info("Cognify completed", { status: response.status }); + } catch (error) { + log.error("Cognify failed", { error }); + throw error; + } + } + + async getStatus(): Promise<{ + connected: boolean; + datasetId?: string; + datasetName: string; + syncedFileCount: number; + version?: string; + }> { + try { + const status = await this.client.status(); + const dataset = status.datasets?.find((d) => d.name === this.datasetName); + + return { + connected: true, + datasetId: this.datasetId || dataset?.id, + datasetName: this.datasetName, + syncedFileCount: this.syncedFiles.size, + version: status.version, + }; + } catch (error) { + log.error("Failed to get Cognee status", { error }); + return { + connected: false, + datasetName: this.datasetName, + syncedFileCount: this.syncedFiles.size, + }; + } + } + + private async collectMemoryFiles(): Promise<MemoryFileEntry[]> { + const files: MemoryFileEntry[] = []; + const memoryPaths = await listMemoryFiles(this.workspaceDir); + + for (const absPath of memoryPaths) { + try { + const entry = await buildFileEntry(absPath, this.workspaceDir); + files.push(entry); + } catch (error) { + log.warn("Failed to process memory file", { absPath, error }); + } + } + + return files; + } + + private async collectSessionFiles(): Promise<MemoryFileEntry[]> { + const files: MemoryFileEntry[] = []; + const transcriptsDir = resolveSessionTranscriptsDirForAgent( + this.cfg, + this.agentId, + ); + + try { + const entries = await fs.readdir(transcriptsDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isFile() || !entry.name.endsWith(".jsonl")) continue; + + const absPath = path.join(transcriptsDir, entry.name); + try { + const stat = await fs.stat(absPath); + const content = await fs.readFile(absPath, "utf-8"); + const hash = hashText(content); + + files.push({ + path: `sessions/${entry.name}`, + absPath, + mtimeMs: stat.mtimeMs, + size: stat.size, + hash, + }); + } catch (error) { + log.warn("Failed to process session file", { absPath, error }); + } + } + } catch (error) { + log.debug("No session transcripts directory", { transcriptsDir }); + } + + return files; + } + + private async syncFiles( + files: MemoryFileEntry[], + source: CogneeMemorySource, + ): Promise<number> { + let addedCount = 0; + const batchSize = this.cognifyBatchSize; + + for (let i = 0; i < files.length; i += batchSize) { + const batch = files.slice(i, i + batchSize); + + for (const file of batch) { + const existingHash = this.syncedFiles.get(file.path); + if (existingHash === file.hash) { + log.debug("Skipping unchanged file", { path: file.path }); + continue; + } + + try { + const content = await fs.readFile(file.absPath, "utf-8"); + const metadata = { + path: file.path, + source, + agentId: this.agentId, + size: file.size, + mtimeMs: file.mtimeMs, + }; + + const dataWithMetadata = `# ${file.path}\n\n${content}\n\n---\nMetadata: ${JSON.stringify(metadata)}`; + + const response = await this.client.add({ + data: dataWithMetadata, + datasetName: this.datasetName, + }); + + if (!this.datasetId) { + this.datasetId = response.datasetId; + } + + this.syncedFiles.set(file.path, file.hash); + addedCount++; + + log.debug("Added file to Cognee", { + path: file.path, + datasetId: response.datasetId, + }); + } catch (error) { + log.error("Failed to add file to Cognee", { path: file.path, error }); + } + } + } + + return addedCount; + } + + private transformResult(result: CogneeSearchResult): MemorySearchResult { + // Extract path from metadata or text + const metadata = result.metadata || {}; + const path = (metadata.path as string) || "unknown"; + const source = (metadata.source as "memory" | "sessions") || "memory"; + + // Truncate snippet to max chars + let snippet = result.text; + if (snippet.length > SNIPPET_MAX_CHARS) { + snippet = snippet.slice(0, SNIPPET_MAX_CHARS) + "..."; + } + + return { + path, + startLine: 0, // Cognee doesn't provide line numbers + endLine: 0, + score: result.score, + snippet, + source, + }; + } +} + +export async function createCogneeProvider( + cfg: ClawdbotConfig, + agentId: string, + sources: Array<CogneeMemorySource>, + config: CogneeProviderConfig = {}, +): Promise<CogneeMemoryProvider> { + const provider = new CogneeMemoryProvider(cfg, agentId, sources, config); + + // Verify connection + const healthy = await provider.healthCheck(); + if (!healthy) { + throw new Error( + `Failed to connect to Cognee at ${config.baseUrl || "http://localhost:8000"}. ` + + `Ensure Cognee is running (see docs/memory-cognee.md for setup).`, + ); + } + + return provider; +} diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index 9bcd529f3..8503c01da 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -1,8 +1,10 @@ import type { MoltbotConfig } from "../config/config.js"; +import { resolveMemorySearchConfig } from "../agents/memory-search.js"; import type { MemoryIndexManager } from "./manager.js"; +import type { CogneeMemoryProvider } from "./cognee-provider.js"; export type MemorySearchManagerResult = { - manager: MemoryIndexManager | null; + manager: MemoryIndexManager | CogneeMemoryProvider | null; error?: string; }; @@ -11,6 +13,24 @@ export async function getMemorySearchManager(params: { agentId: string; }): Promise { try { + const config = resolveMemorySearchConfig(params.cfg, params.agentId); + if (!config) { + return { manager: null, error: "Memory search is disabled" }; + } + + // Route to Cognee provider if configured + if (config.provider === "cognee") { + const { createCogneeProvider } = await import("./cognee-provider.js"); + const manager = await createCogneeProvider( + params.cfg, + params.agentId, + config.sources as Array<"memory" | "sessions">, + config.cognee || {}, + ); + return { manager }; + } + + // Default to SQLite-based memory manager const { MemoryIndexManager } = await import("./manager.js"); const manager = await MemoryIndexManager.get(params); return { manager }; From a57cc024c63ae43368a2c90ed68aacbc3ad7569d Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Wed, 28 Jan 2026 00:59:37 +0100 Subject: [PATCH 2/8] add updates --- docs/memory-cognee.md | 157 ++++++++++------------ examples/.env.template | 7 + examples/cognee-clawdbot-config.yaml | 35 +++++ examples/cognee-config.yaml | 4 +- examples/cognee-docker-compose.yaml | 69 ++-------- examples/cognee-test-compose.yaml | 117 ++++++++++++++++ examples/test-cognee.sh | 66 +++++++++ src/agents/memory-search.ts | 2 +- src/config/schema.ts | 7 +- src/config/types.tools.ts | 4 +- src/config/zod-schema.agent-runtime.ts | 27 +++- src/memory/cognee-client.test.ts | 66 ++++----- src/memory/cognee-client.ts | 179 ++++++++++++++----------- src/memory/cognee-provider.test.ts | 47 +++---- src/memory/cognee-provider.ts | 173 +++++++++++++++++++----- src/memory/manager.ts | 10 +- 16 files changed, 636 insertions(+), 334 deletions(-) create mode 100644 examples/.env.template create mode 100644 examples/cognee-clawdbot-config.yaml create mode 100644 examples/cognee-test-compose.yaml create mode 100644 examples/test-cognee.sh diff --git a/docs/memory-cognee.md b/docs/memory-cognee.md index 6980513f5..1a82f6e1b 100644 --- a/docs/memory-cognee.md +++ b/docs/memory-cognee.md @@ -16,7 +16,7 @@ Cognee is an AI memory framework that: - Extracts entities (people, places, concepts) from documents - Builds a knowledge graph of relationships - Enables semantic search with LLM-powered reasoning -- Supports multiple search modes (insights, chunks, summaries) +- Supports multiple search modes (GRAPH_COMPLETION, chunks, summaries) Learn more at [docs.cognee.ai](https://docs.cognee.ai/). @@ -24,57 +24,18 @@ Learn more at [docs.cognee.ai](https://docs.cognee.ai/). ### Option 1: Local Docker (Recommended) -Run Cognee locally using Docker Compose: - -**Step 1: Create docker-compose.yml** - -```yaml -version: '3.8' - -services: - cognee: - image: topoteretes/cognee:latest - container_name: cognee - ports: - - "8000:8000" - environment: - # Optional: Set API key for authentication - - COGNEE_API_KEY=your-local-api-key - # Database configuration - - DATABASE_URL=postgresql://cognee:cognee@postgres:5432/cognee - volumes: - - cognee_data:/app/data - depends_on: - - postgres - restart: unless-stopped - - postgres: - image: postgres:15-alpine - container_name: cognee-postgres - environment: - - POSTGRES_USER=cognee - - POSTGRES_PASSWORD=cognee - - POSTGRES_DB=cognee - volumes: - - postgres_data:/var/lib/postgresql/data - restart: unless-stopped - -volumes: - cognee_data: - postgres_data: -``` - -**Step 2: Start Cognee** +Use the repo example compose file: ```bash -docker-compose up -d +docker compose -f examples/cognee-docker-compose.yaml up -d ``` -**Step 3: Verify** +This binds to `127.0.0.1:8000`, so Cognee is only reachable from your machine. + +**Verify:** ```bash -curl http://localhost:8000/status -# Should return: {"status":"healthy"} +curl http://localhost:8000/health ``` ### Option 2: Cognee Cloud @@ -87,26 +48,49 @@ Use the hosted Cognee service: ## Configuration -Add Cognee memory configuration to your `~/.clawdbot/config.yaml`: +Clawdbot reads `~/.clawdbot/clawdbot.json` (JSON5). For local + secure setup, use an env var for the Cognee token and reference it in config. -### Basic Configuration (Docker Local) +### Step 1: Put tokens in `examples/.env` -```yaml -agents: - defaults: - memorySearch: - enabled: true - provider: cognee - sources: [memory] # or [memory, sessions] - cognee: - baseUrl: http://localhost:8000 - datasetName: clawdbot - searchType: insights # or "chunks", "summaries" - maxResults: 6 - autoCognify: true +```bash +LLM_API_KEY="your-llm-api-key" +COGNEE_API_KEY="your-cognee-access-token" +CLAWDBOT_GATEWAY_TOKEN="your-random-gateway-token" ``` -### Cloud Configuration +### Step 2: Configure Cognee in `~/.clawdbot/clawdbot.json` + +```json5 +{ + agents: { + defaults: { + memorySearch: { + enabled: true, + provider: "cognee", + sources: ["memory", "sessions"], + experimental: { sessionMemory: true }, + cognee: { + baseUrl: "http://localhost:8000", + apiKey: "${COGNEE_API_KEY}", + datasetName: "clawdbot", + searchType: "GRAPH_COMPLETION", + maxResults: 6, + autoCognify: true, + timeoutSeconds: 180 + } + } + } + } +} +``` + +### Step 3: Start the gateway with env loaded + +```zsh +set -a; source "examples/.env"; set +a; pnpm clawdbot gateway --port 18789 --token "$CLAWDBOT_GATEWAY_TOKEN" --verbose +``` + +### Cloud Configuration (tbt) ```yaml agents: @@ -119,7 +103,7 @@ agents: baseUrl: https://cognee--cognee-saas-backend-serve.modal.run apiKey: your-api-key-here # Required for cloud datasetName: clawdbot - searchType: insights + searchType: GRAPH_COMPLETION maxResults: 8 autoCognify: true timeoutSeconds: 60 @@ -130,22 +114,21 @@ agents: | Option | Type | Default | Description | |--------|------|---------|-------------| | `baseUrl` | string | `http://localhost:8000` | Cognee API endpoint | -| `apiKey` | string | - | API key (required for cloud, optional for local) | +| `apiKey` | string | - | Access token (required if Cognee auth is enabled) | | `datasetName` | string | `"clawdbot"` | Dataset name for organizing memories | -| `searchType` | string | `"insights"` | Search mode: `insights`, `chunks`, or `summaries` | +| `searchType` | string | `"GRAPH_COMPLETION"` | Search mode: `GRAPH_COMPLETION`, `chunks`, or `summaries` | | `maxResults` | number | `6` | Maximum search results returned | | `autoCognify` | boolean | `true` | Auto-process documents after adding | | `cognifyBatchSize` | number | `100` | Batch size for processing | -| `timeoutSeconds` | number | `30` | Request timeout in seconds | +| `timeoutSeconds` | number | `180` | Request timeout in seconds | ## Search Types -Cognee offers three search modes: +Cognee offers these modes: -### Insights (Recommended) +### GRAPH_COMPLETION Best for: **High-level understanding and reasoning** -- Returns AI-generated insights from knowledge graph -- Combines multiple related facts +- Returns graph-completion outputs over the knowledge graph - Good for: "What projects am I working on?" or "Summarize my notes about X" ### Chunks @@ -181,20 +164,11 @@ agents: sessionMemory: true ``` -### Manual Sync - -Force a memory sync: +### Manual Sync and Status ```bash -# Not yet implemented - coming soon -clawdbot memory sync --provider cognee -``` - -### Check Status - -```bash -# Not yet implemented - coming soon -clawdbot memory status --provider cognee +# Force sync + cognify for the current agent +clawdbot memory status --index --json ``` ## How It Works @@ -231,7 +205,7 @@ clawdbot memory status --provider cognee **Solutions**: 1. Verify Docker is running: `docker ps | grep cognee` 2. Check Cognee logs: `docker logs cognee` -3. Test manually: `curl http://localhost:8000/status` +3. Test manually: `curl http://localhost:8000/health` 4. Ensure port 8000 is not blocked ### Slow Performance @@ -250,6 +224,15 @@ clawdbot memory status --provider cognee 3. Process fewer files at once 4. Clear old datasets via Cognee API +### 401 Unauthorized (Cognee auth) + +**Cause**: Cognee auth is enabled, but Clawdbot is missing/using an invalid `COGNEE_API_KEY`. + +**Fix**: +1. Log in to Cognee and get a fresh access token. +2. Update `COGNEE_API_KEY` in `examples/.env`. +3. Restart the gateway with env loaded (see Step 3 above). + ## Advanced Configuration ### Per-Agent Override @@ -265,7 +248,7 @@ agents: memorySearch: provider: cognee # Override for this agent cognee: - searchType: insights + searchType: GRAPH_COMPLETION maxResults: 10 ``` @@ -285,7 +268,7 @@ Add health checks to docker-compose.yml: services: cognee: healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 30s timeout: 10s retries: 3 @@ -313,15 +296,13 @@ Mount volumes for persistence: ```yaml volumes: - - ./cognee_data:/app/data + - ./cognee_data:/app/cognee/.cognee_system - ./cognee_logs:/app/logs ``` ## Roadmap Planned features: -- [ ] `clawdbot memory status --provider cognee` command -- [ ] `clawdbot memory sync --provider cognee` command - [ ] Hybrid mode (Cognee + SQLite) - [ ] Graph visualization export - [ ] Manual entity management diff --git a/examples/.env.template b/examples/.env.template new file mode 100644 index 000000000..3bf48a935 --- /dev/null +++ b/examples/.env.template @@ -0,0 +1,7 @@ +LLM_API_KEY= +ENABLE_BACKEND_ACCESS_CONTROL="false" + + +COGNEE_API_KEY= + +CLAWDBOT_GATEWAY_TOKEN= \ No newline at end of file diff --git a/examples/cognee-clawdbot-config.yaml b/examples/cognee-clawdbot-config.yaml new file mode 100644 index 000000000..69aa763d3 --- /dev/null +++ b/examples/cognee-clawdbot-config.yaml @@ -0,0 +1,35 @@ +# Sample Clawdbot config for testing Cognee integration +# Copy this to ~/.clawdbot/config.yaml inside the dev container +# +# Inside container: +# mkdir -p ~/.clawdbot +# cp /app/examples/cognee-clawdbot-config.yaml ~/.clawdbot/config.yaml + +agents: + defaults: + # Enable Cognee as the memory provider + memorySearch: + enabled: true + provider: cognee + sources: [memory] # Start with just memory files + + cognee: + # Inside Docker network, Cognee is at this address + baseUrl: http://cognee:8000 + # No API key needed for local Docker setup + # apiKey: "" + + # Dataset name for organizing memories + datasetName: clawdbot-test + + # Search mode: "insights" (recommended), "chunks", or "summaries" + searchType: insights + + # Max results per search + maxResults: 6 + + # Auto-process documents after adding + autoCognify: true + + # Request timeout + timeoutSeconds: 30 diff --git a/examples/cognee-config.yaml b/examples/cognee-config.yaml index b8abc922b..4279c0add 100644 --- a/examples/cognee-config.yaml +++ b/examples/cognee-config.yaml @@ -21,8 +21,8 @@ agents: # Dataset name for organizing memories datasetName: clawdbot - # Search mode: "insights" (recommended), "chunks", or "summaries" - searchType: insights + # Search mode: "GRAPH_COMPLETION" (recommended), "chunks", or "summaries" + searchType: GRAPH_COMPLETION # Maximum search results to return maxResults: 6 diff --git a/examples/cognee-docker-compose.yaml b/examples/cognee-docker-compose.yaml index 045f8a7dc..8269e14ec 100644 --- a/examples/cognee-docker-compose.yaml +++ b/examples/cognee-docker-compose.yaml @@ -1,77 +1,36 @@ -# Docker Compose configuration for running Cognee locally with Clawdbot +# Minimal Docker Compose for Cognee (local-only) # # Usage: -# 1. Copy this file to your preferred location -# 2. Run: docker-compose -f cognee-docker-compose.yaml up -d -# 3. Verify: curl http://localhost:8000/status +# 1. Export your LLM key: export LLM_API_KEY="your-openai-api-key" +# 2. Run: docker compose -f examples/cognee-docker-compose.yaml up -d +# 3. Verify: curl http://localhost:8000/health # 4. Configure Clawdbot with baseUrl: http://localhost:8000 # -# For production, see docs/memory-cognee.md for additional configuration +# Defaults (no extra DB setup): +# - Relational DB: SQLite (file-based) +# - Vector DB: LanceDB (file-based) +# - Graph DB: Kuzu (file-based) -version: '3.8' +version: "3.8" services: - # Cognee API server cognee: - image: topoteretes/cognee:latest + image: cognee/cognee:latest container_name: cognee ports: - - "8000:8000" + - "127.0.0.1:8000:8000" environment: - # Optional: Set API key for authentication - # Remove or comment out for local development without auth - - COGNEE_API_KEY=${COGNEE_API_KEY:-} - - # Database connection - - DATABASE_URL=postgresql://cognee:cognee@postgres:5432/cognee - - # Optional: Configure LLM provider for entity extraction - # Uncomment and set if you want to use specific providers - # - OPENAI_API_KEY=${OPENAI_API_KEY:-} - # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + - LLM_API_KEY=${LLM_API_KEY} volumes: - - cognee_data:/app/data - - cognee_logs:/app/logs - depends_on: - postgres: - condition: service_healthy + - cognee_data:/app/cognee/.cognee_system restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] interval: 30s timeout: 10s retries: 3 start_period: 40s - networks: - - cognee-network - - # PostgreSQL database for Cognee - postgres: - image: postgres:15-alpine - container_name: cognee-postgres - environment: - - POSTGRES_USER=cognee - - POSTGRES_PASSWORD=cognee - - POSTGRES_DB=cognee - volumes: - - postgres_data:/var/lib/postgresql/data - restart: unless-stopped - healthcheck: - test: ["CMD-SHELL", "pg_isready -U cognee"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - cognee-network volumes: cognee_data: driver: local - cognee_logs: - driver: local - postgres_data: - driver: local - -networks: - cognee-network: - driver: bridge diff --git a/examples/cognee-test-compose.yaml b/examples/cognee-test-compose.yaml new file mode 100644 index 000000000..865b334c8 --- /dev/null +++ b/examples/cognee-test-compose.yaml @@ -0,0 +1,117 @@ +# Docker Compose for testing Clawdbot + Cognee integration +# +# Usage: +# 1. cd examples +# 2. docker-compose -f cognee-test-compose.yaml up -d +# 3. docker-compose -f cognee-test-compose.yaml exec dev bash +# 4. Inside container: +# - pnpm install # Install dependencies +# - pnpm build # Build Clawdbot +# - pnpm test src/memory/cognee-* # Run unit tests +# - pnpm clawdbot agent --message "test" # Run Clawdbot CLI +# +# To tear down: docker-compose -f cognee-test-compose.yaml down -v + +version: '3.8' + +services: + # Cognee API server + cognee: + image: topoteretes/cognee:latest + container_name: cognee-test + ports: + - "8000:8000" + environment: + # No API key for local testing + - COGNEE_API_KEY= + # Database connection + - DATABASE_URL=postgresql://cognee:cognee@postgres:5432/cognee + volumes: + - cognee_data:/app/data + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/status"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + networks: + - cognee-test-network + + # PostgreSQL database for Cognee + postgres: + image: postgres:15-alpine + container_name: cognee-test-postgres + environment: + - POSTGRES_USER=cognee + - POSTGRES_PASSWORD=cognee + - POSTGRES_DB=cognee + volumes: + - postgres_data:/var/lib/postgresql/data + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "pg_isready -U cognee"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - cognee-test-network + + # Development container for running Clawdbot + tests + dev: + image: node:22-bookworm + container_name: clawdbot-cognee-dev + working_dir: /app + volumes: + # Mount the entire clawdbot repo + - ..:/app + # Use a separate node_modules volume to avoid conflicts + - node_modules:/app/node_modules + # Clawdbot config directory (persisted) + - clawdbot_config:/root/.clawdbot + environment: + - HOME=/root + - TERM=xterm-256color + # Point to Cognee running in Docker network + - COGNEE_BASE_URL=http://cognee:8000 + # Optional: Add your API keys here for full testing + # - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-} + # - OPENAI_API_KEY=${OPENAI_API_KEY:-} + depends_on: + cognee: + condition: service_healthy + stdin_open: true + tty: true + # Install pnpm and keep container running + command: > + bash -c " + corepack enable && + echo '=== Clawdbot + Cognee Test Environment ===' && + echo 'Cognee is available at: http://cognee:8000' && + echo '' && + echo 'Quick start:' && + echo ' pnpm install && pnpm build' && + echo ' pnpm test src/memory/cognee-client.test.ts' && + echo ' pnpm clawdbot --help' && + echo '' && + sleep infinity + " + networks: + - cognee-test-network + +volumes: + cognee_data: + driver: local + postgres_data: + driver: local + node_modules: + driver: local + clawdbot_config: + driver: local + +networks: + cognee-test-network: + driver: bridge diff --git a/examples/test-cognee.sh b/examples/test-cognee.sh new file mode 100644 index 000000000..09a9f1f3c --- /dev/null +++ b/examples/test-cognee.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Test script for Cognee integration +# Run this inside the dev container after starting docker-compose + +set -e + +echo "=== Clawdbot + Cognee Integration Test ===" +echo "" + +# 1. Check if Cognee is reachable +echo "1. Testing Cognee connection..." +if curl -sf http://cognee:8000/status > /dev/null 2>&1; then + echo " ✓ Cognee is healthy at http://cognee:8000" + curl -s http://cognee:8000/status | head -c 200 + echo "" +else + echo " ✗ Cannot reach Cognee at http://cognee:8000" + echo " Make sure Cognee container is running and healthy" + exit 1 +fi +echo "" + +# 2. Install dependencies if needed +if [ ! -d "node_modules/.pnpm" ]; then + echo "2. Installing dependencies..." + pnpm install +else + echo "2. Dependencies already installed" +fi +echo "" + +# 3. Run unit tests +echo "3. Running Cognee unit tests..." +pnpm test src/memory/cognee-client.test.ts src/memory/cognee-provider.test.ts +echo "" + +# 4. Quick integration check (manual API test) +echo "4. Quick API smoke test..." +echo " Adding test data to Cognee..." + +# Add test data +RESULT=$(curl -sf -X POST http://cognee:8000/add \ + -H "Content-Type: application/json" \ + -d '{ + "data": "Test memory entry: The capital of France is Paris. This is a test document for Clawdbot integration.", + "dataset_name": "clawdbot-test" + }' 2>&1) || true + +if echo "$RESULT" | grep -q "dataset"; then + echo " ✓ Data added successfully" + echo " Response: $RESULT" +else + echo " ⚠ Add may have failed (this is OK if Cognee API differs)" + echo " Response: $RESULT" +fi +echo "" + +echo "=== Test Complete ===" +echo "" +echo "Next steps:" +echo " - The unit tests verify the client/provider logic" +echo " - For full integration, configure Clawdbot with:" +echo " memorySearch:" +echo " provider: cognee" +echo " cognee:" +echo " baseUrl: http://cognee:8000 # or http://localhost:8000 from host" diff --git a/src/agents/memory-search.ts b/src/agents/memory-search.ts index 8fbdb2614..9deaae970 100644 --- a/src/agents/memory-search.ts +++ b/src/agents/memory-search.ts @@ -35,7 +35,7 @@ export type ResolvedMemorySearchConfig = { baseUrl?: string; apiKey?: string; datasetName?: string; - searchType?: "insights" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; maxResults?: number; timeoutSeconds?: number; autoCognify?: boolean; diff --git a/src/config/schema.ts b/src/config/schema.ts index b4ec8723b..aacd8f16d 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -501,7 +501,8 @@ const FIELD_HELP: Record = { 'Sources to index for memory search (default: ["memory"]; add "sessions" to include session transcripts).', "agents.defaults.memorySearch.experimental.sessionMemory": "Enable experimental session transcript indexing for memory search (default: false).", - "agents.defaults.memorySearch.provider": 'Embedding provider ("openai", "gemini", or "local").', + "agents.defaults.memorySearch.provider": + 'Embedding provider ("openai", "gemini", "local", or "cognee").', "agents.defaults.memorySearch.remote.baseUrl": "Custom base URL for remote embeddings (OpenAI-compatible proxies or Gemini overrides).", "agents.defaults.memorySearch.remote.apiKey": "Custom API key for the remote embedding provider.", @@ -517,10 +518,12 @@ const FIELD_HELP: Record = { "Polling interval in ms for batch status (default: 2000).", "agents.defaults.memorySearch.remote.batch.timeoutMinutes": "Timeout in minutes for batch indexing (default: 60).", + "agents.defaults.memorySearch.cognee": + "Cognee provider configuration (baseUrl, apiKey, datasetName, searchType, maxResults, autoCognify).", "agents.defaults.memorySearch.local.modelPath": "Local GGUF model path or hf: URI (node-llama-cpp).", "agents.defaults.memorySearch.fallback": - 'Fallback provider when embeddings fail ("openai", "gemini", "local", or "none").', + 'Fallback provider when embeddings fail ("openai", "gemini", "local", "cognee", or "none").', "agents.defaults.memorySearch.store.path": "SQLite index path (default: ~/.clawdbot/memory/{agentId}.sqlite).", "agents.defaults.memorySearch.store.vector.enabled": diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index 7506204b9..85254b80e 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -269,8 +269,8 @@ export type MemorySearchConfig = { apiKey?: string; /** Dataset name for organizing memories (default: "clawdbot"). */ datasetName?: string; - /** Search type: "insights", "chunks", or "summaries" (default: "insights"). */ - searchType?: "insights" | "chunks" | "summaries"; + /** Search type: "GRAPH_COMPLETION", "chunks", or "summaries" (default: "GRAPH_COMPLETION"). */ + searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; /** Max results per search query (default: 6). */ maxResults?: number; /** Timeout for API requests in seconds (default: 30). */ diff --git a/src/config/zod-schema.agent-runtime.ts b/src/config/zod-schema.agent-runtime.ts index 7a63e307d..1fcca8524 100644 --- a/src/config/zod-schema.agent-runtime.ts +++ b/src/config/zod-schema.agent-runtime.ts @@ -310,7 +310,9 @@ export const MemorySearchSchema = z }) .strict() .optional(), - provider: z.union([z.literal("openai"), z.literal("local"), z.literal("gemini")]).optional(), + provider: z + .union([z.literal("openai"), z.literal("local"), z.literal("gemini"), z.literal("cognee")]) + .optional(), remote: z .object({ baseUrl: z.string().optional(), @@ -330,9 +332,30 @@ export const MemorySearchSchema = z .strict() .optional(), fallback: z - .union([z.literal("openai"), z.literal("gemini"), z.literal("local"), z.literal("none")]) + .union([ + z.literal("openai"), + z.literal("gemini"), + z.literal("local"), + z.literal("cognee"), + z.literal("none"), + ]) .optional(), model: z.string().optional(), + cognee: z + .object({ + baseUrl: z.string().optional(), + apiKey: z.string().optional(), + datasetName: z.string().optional(), + searchType: z + .union([z.literal("GRAPH_COMPLETION"), z.literal("CHUNKS"), z.literal("SUMMARIES")]) + .optional(), + maxResults: z.number().int().positive().optional(), + autoCognify: z.boolean().optional(), + cognifyBatchSize: z.number().int().positive().optional(), + timeoutSeconds: z.number().int().positive().optional(), + }) + .strict() + .optional(), local: z .object({ modelPath: z.string().optional(), diff --git a/src/memory/cognee-client.test.ts b/src/memory/cognee-client.test.ts index 6bbdb9a3a..822de5dd4 100644 --- a/src/memory/cognee-client.test.ts +++ b/src/memory/cognee-client.test.ts @@ -2,21 +2,21 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { CogneeClient } from "./cognee-client.js"; import { request } from "undici"; -vi.mock("undici", () => ({ +vi.mock("undici", () => ({ request: vi.fn(), })); -describe("CogneeClient", () => { - beforeEach(() => { +describe("CogneeClient", () => { + beforeEach(() => { vi.clearAllMocks(); }); - afterEach(() => { + afterEach(() => { vi.restoreAllMocks(); }); - describe("add", () => { - it("should add data successfully", async () => { + describe("add", () => { + it("should add data successfully", async () => { const mockResponse = { statusCode: 200, body: { @@ -46,18 +46,18 @@ describe("CogneeClient", () => { message: "Data added successfully", }); expect(request).toHaveBeenCalledWith( - "http://localhost:8000/add", + "http://localhost:8000/api/v1/add", expect.objectContaining({ method: "POST", headers: expect.objectContaining({ - "Content-Type": "application/json", + Authorization: "Bearer test-key", "X-Api-Key": "test-key", }), }), ); }); - it("should handle errors", async () => { + it("should handle errors", async () => { const mockResponse = { statusCode: 500, body: { @@ -77,8 +77,8 @@ describe("CogneeClient", () => { }); }); - describe("cognify", () => { - it("should run cognify successfully", async () => { + describe("cognify", () => { + it("should run cognify successfully", async () => { const mockResponse = { statusCode: 200, body: { @@ -106,8 +106,8 @@ describe("CogneeClient", () => { }); }); - describe("search", () => { - it("should search successfully", async () => { + describe("search", () => { + it("should search successfully", async () => { const mockResponse = { statusCode: 200, body: { @@ -121,7 +121,7 @@ describe("CogneeClient", () => { }, ], query: "test query", - search_type: "insights", + search_type: "GRAPH_COMPLETION", }), text: vi.fn(), }, @@ -132,7 +132,7 @@ describe("CogneeClient", () => { const result = await client.search({ queryText: "test query", - searchType: "insights", + searchType: "GRAPH_COMPLETION", }); expect(result.results).toHaveLength(1); @@ -145,14 +145,14 @@ describe("CogneeClient", () => { expect(result.query).toBe("test query"); }); - it("should use default search type", async () => { + it("should use default search type", async () => { const mockResponse = { statusCode: 200, body: { json: vi.fn().mockResolvedValue({ results: [], query: "test", - search_type: "insights", + search_type: "GRAPH_COMPLETION", }), text: vi.fn(), }, @@ -165,28 +165,18 @@ describe("CogneeClient", () => { expect(request).toHaveBeenCalledWith( expect.any(String), expect.objectContaining({ - body: expect.stringContaining('"search_type":"insights"'), + body: expect.stringContaining('"searchType":"GRAPH_COMPLETION"'), }), ); }); }); - describe("status", () => { - it("should get status successfully", async () => { + describe("status", () => { + it("should get status successfully", async () => { const mockResponse = { statusCode: 200, body: { - json: vi.fn().mockResolvedValue({ - status: "healthy", - version: "1.0.0", - datasets: [ - { - id: "dataset-1", - name: "test-dataset", - document_count: 10, - }, - ], - }), + json: vi.fn().mockResolvedValue({ status: "healthy" }), text: vi.fn(), }, }; @@ -198,20 +188,12 @@ describe("CogneeClient", () => { expect(result).toEqual({ status: "healthy", - version: "1.0.0", - datasets: [ - { - id: "dataset-1", - name: "test-dataset", - documentCount: 10, - }, - ], }); }); }); - describe("healthCheck", () => { - it("should return true when status is successful", async () => { + describe("healthCheck", () => { + it("should return true when status is successful", async () => { const mockResponse = { statusCode: 200, body: { @@ -227,7 +209,7 @@ describe("CogneeClient", () => { expect(result).toBe(true); }); - it("should return false when status fails", async () => { + it("should return false when status fails", async () => { vi.mocked(request).mockRejectedValue(new Error("Connection failed")); const client = new CogneeClient(); diff --git a/src/memory/cognee-client.ts b/src/memory/cognee-client.ts index c3ec3747a..c442460d6 100644 --- a/src/memory/cognee-client.ts +++ b/src/memory/cognee-client.ts @@ -1,10 +1,12 @@ -import { request } from "undici"; +import { Blob } from "buffer"; +import { FormData, request } from "undici"; import { createSubsystemLogger } from "../logging/subsystem.js"; const log = createSubsystemLogger("cognee"); const DEFAULT_BASE_URL = "http://localhost:8000"; const DEFAULT_TIMEOUT_MS = 30_000; +const API_PREFIX = "/api/v1"; export type CogneeClientConfig = { baseUrl?: string; @@ -35,7 +37,7 @@ export type CogneeCognifyResponse = { export type CogneeSearchRequest = { queryText: string; - searchType?: "insights" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; datasetIds?: string[]; }; @@ -43,7 +45,7 @@ export type CogneeSearchResult = { id: string; text: string; score: number; - metadata?: Record<string, unknown>; + metadata?: Record; }; export type CogneeSearchResponse = { @@ -55,13 +57,15 @@ export type CogneeSearchResponse = { export type CogneeStatusResponse = { status: string; version?: string; - datasets?: Array<{ + datasets?: Array<{ id: string; name: string; documentCount?: number; - }>; + }>; }; +type CogneeSearchApiType = "SUMMARIES" | "CHUNKS" | "GRAPH_COMPLETION"; + export class CogneeClient { private readonly baseUrl: string; private readonly apiKey?: string; @@ -73,12 +77,11 @@ export class CogneeClient { this.timeoutMs = config.timeoutMs || DEFAULT_TIMEOUT_MS; } - async add(req: CogneeAddRequest): Promise<CogneeAddResponse> { - const url = `${this.baseUrl}/add`; - const headers: Record<string, string> = { - "Content-Type": "application/json", - }; + async add(req: CogneeAddRequest): Promise { + const url = `${this.baseUrl}${API_PREFIX}/add`; + const headers: Record = {}; if (this.apiKey) { + headers.Authorization = `Bearer ${this.apiKey}`; headers["X-Api-Key"] = this.apiKey; } @@ -89,23 +92,27 @@ export class CogneeClient { }); try { + const formData = new FormData(); + const blob = new Blob([req.data], { type: "text/plain" }); + formData.append("data", blob, "clawdbot-memory.txt"); + if (req.datasetName) { + formData.append("datasetName", req.datasetName); + } + if (req.datasetId) { + formData.append("datasetId", req.datasetId); + } + const response = await request(url, { method: "POST", headers, - body: JSON.stringify({ - data: req.data, - dataset_name: req.datasetName, - dataset_id: req.datasetId, - }), + body: formData, bodyTimeout: this.timeoutMs, headersTimeout: this.timeoutMs, }); if (response.statusCode !== 200) { const errorText = await response.body.text(); - throw new Error( - `Cognee add failed with status ${response.statusCode}: ${errorText}`, - ); + throw new Error(`Cognee add failed with status ${response.statusCode}: ${errorText}`); } const data = (await response.body.json()) as { @@ -127,12 +134,13 @@ export class CogneeClient { } } - async cognify(req: CogneeCognifyRequest = {}): Promise<CogneeCognifyResponse> { - const url = `${this.baseUrl}/cognify`; - const headers: Record<string, string> = { + async cognify(req: CogneeCognifyRequest = {}): Promise { + const url = `${this.baseUrl}${API_PREFIX}/cognify`; + const headers: Record = { "Content-Type": "application/json", }; if (this.apiKey) { + headers.Authorization = `Bearer ${this.apiKey}`; headers["X-Api-Key"] = this.apiKey; } @@ -143,7 +151,7 @@ export class CogneeClient { method: "POST", headers, body: JSON.stringify({ - dataset_ids: req.datasetIds, + datasetIds: req.datasetIds, }), bodyTimeout: this.timeoutMs, headersTimeout: this.timeoutMs, @@ -151,9 +159,7 @@ export class CogneeClient { if (response.statusCode !== 200) { const errorText = await response.body.text(); - throw new Error( - `Cognee cognify failed with status ${response.statusCode}: ${errorText}`, - ); + throw new Error(`Cognee cognify failed with status ${response.statusCode}: ${errorText}`); } const data = (await response.body.json()) as { @@ -173,12 +179,13 @@ export class CogneeClient { } } - async search(req: CogneeSearchRequest): Promise<CogneeSearchResponse> { - const url = `${this.baseUrl}/search`; - const headers: Record<string, string> = { + async search(req: CogneeSearchRequest): Promise { + const url = `${this.baseUrl}${API_PREFIX}/search`; + const headers: Record = { "Content-Type": "application/json", }; if (this.apiKey) { + headers.Authorization = `Bearer ${this.apiKey}`; headers["X-Api-Key"] = this.apiKey; } @@ -193,9 +200,9 @@ export class CogneeClient { method: "POST", headers, body: JSON.stringify({ - query_text: req.queryText, - search_type: req.searchType || "insights", - dataset_ids: req.datasetIds, + query: req.queryText, + searchType: this.mapSearchType(req.searchType), + datasetIds: req.datasetIds, }), bodyTimeout: this.timeoutMs, headersTimeout: this.timeoutMs, @@ -203,31 +210,16 @@ export class CogneeClient { if (response.statusCode !== 200) { const errorText = await response.body.text(); - throw new Error( - `Cognee search failed with status ${response.statusCode}: ${errorText}`, - ); + throw new Error(`Cognee search failed with status ${response.statusCode}: ${errorText}`); } - const data = (await response.body.json()) as { - results: Array<{ - id: string; - text: string; - score: number; - metadata?: Record<string, unknown>; - }>; - query: string; - search_type: string; - }; + const data = (await response.body.json()) as unknown; + const results = this.normalizeSearchResults(data); return { - results: data.results.map((r) => ({ - id: r.id, - text: r.text, - score: r.score, - metadata: r.metadata, - })), - query: data.query, - searchType: data.search_type, + results, + query: req.queryText, + searchType: req.searchType || "GRAPH_COMPLETION", }; } catch (error) { log.error("Failed to search Cognee", { error }); @@ -237,10 +229,11 @@ export class CogneeClient { } } - async status(): Promise<CogneeStatusResponse> { - const url = `${this.baseUrl}/status`; - const headers: Record<string, string> = {}; + async status(): Promise { + const url = `${this.baseUrl}/health`; + const headers: Record = {}; if (this.apiKey) { + headers.Authorization = `Bearer ${this.apiKey}`; headers["X-Api-Key"] = this.apiKey; } @@ -256,29 +249,13 @@ export class CogneeClient { if (response.statusCode !== 200) { const errorText = await response.body.text(); - throw new Error( - `Cognee status failed with status ${response.statusCode}: ${errorText}`, - ); + throw new Error(`Cognee status failed with status ${response.statusCode}: ${errorText}`); } - const data = (await response.body.json()) as { - status: string; - version?: string; - datasets?: Array<{ - id: string; - name: string; - document_count?: number; - }>; - }; + const data = (await response.body.json()) as { status?: string }; return { - status: data.status, - version: data.version, - datasets: data.datasets?.map((d) => ({ - id: d.id, - name: d.name, - documentCount: d.document_count, - })), + status: data.status || "healthy", }; } catch (error) { log.error("Failed to get Cognee status", { error }); @@ -288,7 +265,7 @@ export class CogneeClient { } } - async healthCheck(): Promise<boolean> { + async healthCheck(): Promise { try { await this.status(); return true; @@ -296,4 +273,56 @@ export class CogneeClient { return false; } } + + private mapSearchType(type?: CogneeSearchRequest["searchType"]): CogneeSearchApiType { + switch (type) { + case "chunks": + return "CHUNKS"; + case "summaries": + return "SUMMARIES"; + case "GRAPH_COMPLETION": + default: + return "GRAPH_COMPLETION"; + } + } + + private normalizeSearchResults(data: unknown): CogneeSearchResult[] { + if (Array.isArray(data)) { + return data.map((item, index) => { + if (typeof item === "string") { + return { id: `result-${index}`, text: item, score: 0 }; + } + + if (item && typeof item === "object") { + const record = item as Record; + const raw = + record.search_result ?? record.result ?? record.context ?? record.text ?? record; + const text = typeof raw === "string" ? raw : JSON.stringify(raw, null, 2); + const metadata = + record.dataset_name || record.dataset_id + ? { + datasetName: record.dataset_name, + datasetId: record.dataset_id, + } + : undefined; + return { id: `result-${index}`, text, score: 0, metadata }; + } + + return { + id: `result-${index}`, + text: String(item), + score: 0, + }; + }); + } + + if (data && typeof data === "object" && "results" in data) { + const results = (data as { results?: unknown }).results; + if (Array.isArray(results)) { + return this.normalizeSearchResults(results); + } + } + + return []; + } } diff --git a/src/memory/cognee-provider.test.ts b/src/memory/cognee-provider.test.ts index 2b6bd64b6..d0bd31bad 100644 --- a/src/memory/cognee-provider.test.ts +++ b/src/memory/cognee-provider.test.ts @@ -2,8 +2,8 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { CogneeMemoryProvider } from "./cognee-provider.js"; import type { ClawdbotConfig } from "../config/config.js"; -vi.mock("./cognee-client.js", () => ({ - CogneeClient: vi.fn().mockImplementation(() => ({ +vi.mock("./cognee-client.js", () => ({ + CogneeClient: vi.fn().mockImplementation(() => ({ healthCheck: vi.fn().mockResolvedValue(true), add: vi.fn().mockResolvedValue({ datasetId: "test-dataset-id", @@ -27,7 +27,7 @@ vi.mock("./cognee-client.js", () => ({ }, ], query: "test query", - searchType: "insights", + searchType: "GRAPH_COMPLETION", }), status: vi.fn().mockResolvedValue({ status: "healthy", @@ -43,13 +43,13 @@ vi.mock("./cognee-client.js", () => ({ })), })); -vi.mock("./internal.js", () => ({ +vi.mock("./internal.js", () => ({ listMemoryFiles: vi.fn().mockResolvedValue([]), buildFileEntry: vi.fn(), hashText: vi.fn().mockReturnValue("test-hash"), })); -vi.mock("node:fs/promises", () => ({ +vi.mock("node:fs/promises", () => ({ default: { readdir: vi.fn().mockResolvedValue([]), readFile: vi.fn().mockResolvedValue("Test file content"), @@ -57,7 +57,7 @@ vi.mock("node:fs/promises", () => ({ }, })); -describe("CogneeMemoryProvider", () => { +describe("CogneeMemoryProvider", () => { const mockConfig: ClawdbotConfig = { agents: { defaults: { @@ -66,18 +66,18 @@ describe("CogneeMemoryProvider", () => { }, }; - beforeEach(() => { + beforeEach(() => { vi.clearAllMocks(); }); - describe("constructor", () => { - it("should initialize with default configuration", () => { + describe("constructor", () => { + it("should initialize with default configuration", () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); expect(provider).toBeDefined(); }); - it("should initialize with custom configuration", () => { + it("should initialize with custom configuration", () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"], { baseUrl: "http://custom:8000", apiKey: "custom-key", @@ -90,8 +90,8 @@ describe("CogneeMemoryProvider", () => { }); }); - describe("healthCheck", () => { - it("should perform health check", async () => { + describe("healthCheck", () => { + it("should perform health check", async () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); const healthy = await provider.healthCheck(); @@ -100,8 +100,8 @@ describe("CogneeMemoryProvider", () => { }); }); - describe("search", () => { - it("should search and transform results", async () => { + describe("search", () => { + it("should search and transform results", async () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); const results = await provider.search("test query"); @@ -115,13 +115,10 @@ describe("CogneeMemoryProvider", () => { }); }); - it("should respect maxResults setting", async () => { - const provider = new CogneeMemoryProvider( - mockConfig, - "test-agent", - ["memory"], - { maxResults: 5 }, - ); + it("should respect maxResults setting", async () => { + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"], { + maxResults: 5, + }); const results = await provider.search("test query"); @@ -129,16 +126,16 @@ describe("CogneeMemoryProvider", () => { }); }); - describe("cognify", () => { - it("should run cognify", async () => { + describe("cognify", () => { + it("should run cognify", async () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); await expect(provider.cognify()).resolves.not.toThrow(); }); }); - describe("getStatus", () => { - it("should return status information", async () => { + describe("getStatus", () => { + it("should return status information", async () => { const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); const status = await provider.getStatus(); diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts index 0f6050b66..11f224411 100644 --- a/src/memory/cognee-provider.ts +++ b/src/memory/cognee-provider.ts @@ -5,22 +5,20 @@ import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import type { MemorySearchResult } from "./index.js"; -import { - CogneeClient, - type CogneeClientConfig, - type CogneeSearchResult, -} from "./cognee-client.js"; +import { CogneeClient, type CogneeClientConfig, type CogneeSearchResult } from "./cognee-client.js"; import { buildFileEntry, hashText, listMemoryFiles, + isMemoryPath, + normalizeRelPath, type MemoryFileEntry, } from "./internal.js"; const log = createSubsystemLogger("cognee-provider"); const DEFAULT_DATASET_NAME = "clawdbot"; -const DEFAULT_SEARCH_TYPE = "insights"; +const DEFAULT_SEARCH_TYPE = "GRAPH_COMPLETION"; const DEFAULT_MAX_RESULTS = 6; const DEFAULT_TIMEOUT_SECONDS = 30; const DEFAULT_AUTO_COGNIFY = true; @@ -31,7 +29,7 @@ export type CogneeProviderConfig = { baseUrl?: string; apiKey?: string; datasetName?: string; - searchType?: "insights" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; maxResults?: number; timeoutSeconds?: number; autoCognify?: boolean; @@ -46,18 +44,18 @@ export class CogneeMemoryProvider { private readonly agentId: string; private readonly workspaceDir: string; private readonly datasetName: string; - private readonly searchType: "insights" | "chunks" | "summaries"; + private readonly searchType: "GRAPH_COMPLETION" | "chunks" | "summaries"; private readonly maxResults: number; private readonly autoCognify: boolean; private readonly cognifyBatchSize: number; - private readonly sources: Set<CogneeMemorySource>; + private readonly sources: Set; private datasetId?: string; - private syncedFiles = new Map<string, string>(); // path -> hash + private syncedFiles = new Map(); // path -> hash constructor( cfg: ClawdbotConfig, agentId: string, - sources: Array<CogneeMemorySource>, + sources: Array, config: CogneeProviderConfig = {}, ) { const timeoutMs = (config.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS) * 1000; @@ -86,11 +84,15 @@ export class CogneeMemoryProvider { }); } - async healthCheck(): Promise<boolean> { + async healthCheck(): Promise { return await this.client.healthCheck(); } - async sync(): Promise<void> { + async sync(params?: { + reason?: string; + force?: boolean; + progress?: (update: { completed: number; total: number; label?: string }) => void; + }): Promise { log.info("Starting Cognee memory sync", { agentId: this.agentId }); let addedCount = 0; @@ -108,7 +110,7 @@ export class CogneeMemoryProvider { } // Run cognify if auto-enabled and files were added - if (this.autoCognify && addedCount > 0) { + if (this.autoCognify && addedCount > 0) { log.info("Running cognify after sync", { addedCount }); await this.cognify(); } @@ -117,9 +119,20 @@ export class CogneeMemoryProvider { agentId: this.agentId, addedCount, }); + + if (params?.progress) { + params.progress({ + completed: addedCount, + total: addedCount, + label: params.reason ? `Synced (${params.reason})` : "Synced", + }); + } } - async search(query: string): Promise<MemorySearchResult[]> { + async search( + query: string, + opts?: { maxResults?: number; minScore?: number; sessionKey?: string }, + ): Promise { log.debug("Searching Cognee memory", { query, searchType: this.searchType }); try { @@ -129,9 +142,12 @@ export class CogneeMemoryProvider { datasetIds: this.datasetId ? [this.datasetId] : undefined, }); + const maxResults = opts?.maxResults ?? this.maxResults; + const minScore = opts?.minScore ?? 0; const results: MemorySearchResult[] = response.results - .slice(0, this.maxResults) - .map((r) => this.transformResult(r)); + .map((r) => this.transformResult(r)) + .filter((r) => r.score >= minScore) + .slice(0, maxResults); log.debug("Cognee search completed", { query, resultCount: results.length }); return results; @@ -141,7 +157,7 @@ export class CogneeMemoryProvider { } } - async cognify(): Promise<void> { + async cognify(): Promise { try { const response = await this.client.cognify({ datasetIds: this.datasetId ? [this.datasetId] : undefined, @@ -153,16 +169,16 @@ export class CogneeMemoryProvider { } } - async getStatus(): Promise<{ + async getStatus(): Promise<{ connected: boolean; datasetId?: string; datasetName: string; syncedFileCount: number; version?: string; - }> { + }> { try { const status = await this.client.status(); - const dataset = status.datasets?.find((d) => d.name === this.datasetName); + const dataset = status.datasets?.find((d) => d.name === this.datasetName); return { connected: true, @@ -181,7 +197,98 @@ export class CogneeMemoryProvider { } } - private async collectMemoryFiles(): Promise<MemoryFileEntry[]> { + status(): { + files: number; + chunks: number; + dirty: boolean; + workspaceDir: string; + dbPath: string; + provider: string; + model: string; + requestedProvider: string; + sources: Array; + sourceCounts: Array<{ source: CogneeMemorySource; files: number; chunks: number }>; + cache?: { enabled: boolean; entries?: number; maxEntries?: number }; + fts?: { enabled: boolean; available: boolean; error?: string }; + fallback?: { from: string; reason?: string }; + vector?: { + enabled: boolean; + available?: boolean; + extensionPath?: string; + loadError?: string; + dims?: number; + }; + batch?: { + enabled: boolean; + failures: number; + limit: number; + wait: boolean; + concurrency: number; + pollIntervalMs: number; + timeoutMs: number; + lastError?: string; + lastProvider?: string; + }; + } { + const sources = Array.from(this.sources); + const files = this.syncedFiles.size; + return { + files, + chunks: 0, + dirty: false, + workspaceDir: this.workspaceDir, + dbPath: "cognee", + provider: "cognee", + model: this.searchType, + requestedProvider: "cognee", + sources, + sourceCounts: sources.map((source) => ({ source, files, chunks: 0 })), + vector: { + enabled: false, + available: false, + }, + fts: { + enabled: false, + available: false, + }, + }; + } + + async readFile(params: { + relPath: string; + from?: number; + lines?: number; + }): Promise<{ text: string; path: string }> { + const relPath = normalizeRelPath(params.relPath); + if (!relPath || !isMemoryPath(relPath)) { + throw new Error("path required"); + } + const absPath = path.resolve(this.workspaceDir, relPath); + if (!absPath.startsWith(this.workspaceDir)) { + throw new Error("path escapes workspace"); + } + const content = await fs.readFile(absPath, "utf-8"); + if (!params.from && !params.lines) { + return { text: content, path: relPath }; + } + const lines = content.split("\n"); + const start = Math.max(1, params.from ?? 1); + const count = Math.max(1, params.lines ?? lines.length); + const slice = lines.slice(start - 1, start - 1 + count); + return { text: slice.join("\n"), path: relPath }; + } + + async probeEmbeddingAvailability(): Promise<{ ok: boolean; error?: string }> { + return { ok: false, error: "Cognee provider does not use embeddings." }; + } + + async probeVectorAvailability(): Promise { + return false; + } + + async close(): Promise {} + + private async collectMemoryFiles(): Promise { const files: MemoryFileEntry[] = []; const memoryPaths = await listMemoryFiles(this.workspaceDir); @@ -197,12 +304,9 @@ export class CogneeMemoryProvider { return files; } - private async collectSessionFiles(): Promise<MemoryFileEntry[]> { + private async collectSessionFiles(): Promise { const files: MemoryFileEntry[] = []; - const transcriptsDir = resolveSessionTranscriptsDirForAgent( - this.cfg, - this.agentId, - ); + const transcriptsDir = resolveSessionTranscriptsDirForAgent(this.agentId); try { const entries = await fs.readdir(transcriptsDir, { withFileTypes: true }); @@ -227,20 +331,17 @@ export class CogneeMemoryProvider { } } } catch (error) { - log.debug("No session transcripts directory", { transcriptsDir }); + log.debug("No session transcripts directory", { transcriptsDir, error }); } return files; } - private async syncFiles( - files: MemoryFileEntry[], - source: CogneeMemorySource, - ): Promise<number> { + private async syncFiles(files: MemoryFileEntry[], source: CogneeMemorySource): Promise { let addedCount = 0; const batchSize = this.cognifyBatchSize; - for (let i = 0; i < files.length; i += batchSize) { + for (let i = 0; i < files.length; i += batchSize) { const batch = files.slice(i, i + batchSize); for (const file of batch) { @@ -295,7 +396,7 @@ export class CogneeMemoryProvider { // Truncate snippet to max chars let snippet = result.text; - if (snippet.length > SNIPPET_MAX_CHARS) { + if (snippet.length > SNIPPET_MAX_CHARS) { snippet = snippet.slice(0, SNIPPET_MAX_CHARS) + "..."; } @@ -313,9 +414,9 @@ export class CogneeMemoryProvider { export async function createCogneeProvider( cfg: ClawdbotConfig, agentId: string, - sources: Array<CogneeMemorySource>, + sources: Array, config: CogneeProviderConfig = {}, -): Promise<CogneeMemoryProvider> { +): Promise { const provider = new CogneeMemoryProvider(cfg, agentId, sources, config); // Verify connection diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 9a9991d10..5fbc3ceea 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -183,13 +183,15 @@ export class MemoryIndexManager { const key = `${agentId}:${workspaceDir}:${JSON.stringify(settings)}`; const existing = INDEX_CACHE.get(key); if (existing) return existing; + const provider = settings.provider === "cognee" ? "auto" : settings.provider; + const fallback = settings.fallback === "cognee" ? "none" : settings.fallback; const providerResult = await createEmbeddingProvider({ config: cfg, agentDir: resolveAgentDir(cfg, agentId), - provider: settings.provider, + provider, remote: settings.remote, model: settings.model, - fallback: settings.fallback, + fallback, local: settings.local, }); const manager = new MemoryIndexManager({ @@ -197,7 +199,7 @@ export class MemoryIndexManager { cfg, agentId, workspaceDir, - settings, + settings: { ...settings, provider, fallback }, providerResult, }); INDEX_CACHE.set(key, manager); @@ -1261,7 +1263,7 @@ export class MemoryIndexManager { } private async activateFallbackProvider(reason: string): Promise { - const fallback = this.settings.fallback; + const fallback = this.settings.fallback === "cognee" ? "none" : this.settings.fallback; if (!fallback || fallback === "none" || fallback === this.provider.id) return false; if (this.fallbackFrom) return false; const fallbackFrom = this.provider.id as "openai" | "gemini" | "local"; From 140a640ebf8d09e5fdb324b301ad30463c207e2d Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Wed, 28 Jan 2026 15:08:04 +0100 Subject: [PATCH 3/8] fix conflicts with the new version --- examples/.env.template | 2 ++ examples/cognee-config.yaml | 3 ++- src/config/config.ts | 2 ++ src/memory/cognee-provider.ts | 8 ++++---- 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/examples/.env.template b/examples/.env.template index 3bf48a935..65c1dc759 100644 --- a/examples/.env.template +++ b/examples/.env.template @@ -1,7 +1,9 @@ +# example .env file for cognee docker integration LLM_API_KEY= ENABLE_BACKEND_ACCESS_CONTROL="false" +# ~/.clawdbot/ .env file for clawdbot gateway COGNEE_API_KEY= CLAWDBOT_GATEWAY_TOKEN= \ No newline at end of file diff --git a/examples/cognee-config.yaml b/examples/cognee-config.yaml index 4279c0add..5412b992b 100644 --- a/examples/cognee-config.yaml +++ b/examples/cognee-config.yaml @@ -16,7 +16,8 @@ agents: # For Cognee Cloud, use: # baseUrl: https://cognee--cognee-saas-backend-serve.modal.run - # apiKey: your-api-key-here + # Cognee API key is stored in the .env file + apiKey: ${COGNEE_API_KEY} # Dataset name for organizing memories datasetName: clawdbot diff --git a/src/config/config.ts b/src/config/config.ts index e0bfc48a5..ab78f38f7 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -12,3 +12,5 @@ export * from "./runtime-overrides.js"; export * from "./types.js"; export { validateConfigObject, validateConfigObjectWithPlugins } from "./validation.js"; export { MoltbotSchema } from "./zod-schema.js"; + +export type { MoltbotConfig as ClawdbotConfig } from "./types.js"; diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts index 11f224411..44ecc309b 100644 --- a/src/memory/cognee-provider.ts +++ b/src/memory/cognee-provider.ts @@ -1,6 +1,6 @@ import fs from "node:fs/promises"; import path from "node:path"; -import type { ClawdbotConfig } from "../config/config.js"; +import type { MoltbotConfig } from "../config/config.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -40,7 +40,7 @@ export type CogneeMemorySource = "memory" | "sessions"; export class CogneeMemoryProvider { private readonly client: CogneeClient; - private readonly cfg: ClawdbotConfig; + private readonly cfg: MoltbotConfig; private readonly agentId: string; private readonly workspaceDir: string; private readonly datasetName: string; @@ -53,7 +53,7 @@ export class CogneeMemoryProvider { private syncedFiles = new Map(); // path -> hash constructor( - cfg: ClawdbotConfig, + cfg: MoltbotConfig, agentId: string, sources: Array, config: CogneeProviderConfig = {}, @@ -412,7 +412,7 @@ export class CogneeMemoryProvider { } export async function createCogneeProvider( - cfg: ClawdbotConfig, + cfg: MoltbotConfig, agentId: string, sources: Array, config: CogneeProviderConfig = {}, From 79a157db62490dfb49a57b2ba0a13a23b8f3371b Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Wed, 28 Jan 2026 17:07:25 +0100 Subject: [PATCH 4/8] add fixes --- docs/memory-cognee.md | 81 ++++++++---------------------- src/memory/cognee-client.test.ts | 14 ++++-- src/memory/cognee-client.ts | 23 ++++++++- src/memory/cognee-provider.test.ts | 28 ++++++----- 4 files changed, 67 insertions(+), 79 deletions(-) diff --git a/docs/memory-cognee.md b/docs/memory-cognee.md index 1a82f6e1b..623e6c1c7 100644 --- a/docs/memory-cognee.md +++ b/docs/memory-cognee.md @@ -8,15 +8,16 @@ read_when: # Cognee Memory Provider -Clawdbot supports **Cognee** as an optional memory provider. Unlike the default SQLite-based vector memory, Cognee builds a knowledge graph with entity extraction and semantic relationships, providing richer contextual memory for your AI agent. +MoltBot supports [**Cognee**](https://www.cognee.ai/) as an optional memory provider. Unlike the default SQLite-based vector memory, Cognee builds a knowledge graph with entity extraction and semantic relationships, providing richer contextual memory for your AI agent. ## What is Cognee? -Cognee is an AI memory framework that: +Cognee is an [open source AI memory](https://github.com/topoteretes/cognee) framework that: +- Ingests unstructured/structured data from 30+ sources - Extracts entities (people, places, concepts) from documents -- Builds a knowledge graph of relationships +- Builds a knowledge graph of relationships backed by embeddings - Enables semantic search with LLM-powered reasoning -- Supports multiple search modes (GRAPH_COMPLETION, chunks, summaries) +- Supports multiple search modes (e.g., GRAPH_COMPLETION, chunks, summaries) Learn more at [docs.cognee.ai](https://docs.cognee.ai/). @@ -38,6 +39,8 @@ This binds to `127.0.0.1:8000`, so Cognee is only reachable from your machine. curl http://localhost:8000/health ``` +You can use `http://localhost:8000/docs` to register user and login to get your token. + ### Option 2: Cognee Cloud Use the hosted Cognee service: @@ -48,17 +51,16 @@ Use the hosted Cognee service: ## Configuration -Clawdbot reads `~/.clawdbot/clawdbot.json` (JSON5). For local + secure setup, use an env var for the Cognee token and reference it in config. +Moltbot reads `~/.clawdbot/moltbot.json` (JSON5). For local + secure setup, use an env var for the Cognee token and reference it in config. -### Step 1: Put tokens in `examples/.env` +### Step 1: Put tokens in `~/.clawdbot/.env` ```bash -LLM_API_KEY="your-llm-api-key" COGNEE_API_KEY="your-cognee-access-token" CLAWDBOT_GATEWAY_TOKEN="your-random-gateway-token" ``` -### Step 2: Configure Cognee in `~/.clawdbot/clawdbot.json` +### Step 2: Configure Cognee in `~/.clawdbot/moltbot.json` ```json5 { @@ -87,27 +89,11 @@ CLAWDBOT_GATEWAY_TOKEN="your-random-gateway-token" ### Step 3: Start the gateway with env loaded ```zsh -set -a; source "examples/.env"; set +a; pnpm clawdbot gateway --port 18789 --token "$CLAWDBOT_GATEWAY_TOKEN" --verbose +set -a; source "$HOME/.clawdbot/.env"; set +a +pnpm moltbot gateway --port 18789 --token "$CLAWDBOT_GATEWAY_TOKEN" --verbose ``` -### Cloud Configuration (tbt) -```yaml -agents: - defaults: - memorySearch: - enabled: true - provider: cognee - sources: [memory, sessions] - cognee: - baseUrl: https://cognee--cognee-saas-backend-serve.modal.run - apiKey: your-api-key-here # Required for cloud - datasetName: clawdbot - searchType: GRAPH_COMPLETION - maxResults: 8 - autoCognify: true - timeoutSeconds: 60 -``` ## Configuration Options @@ -124,23 +110,9 @@ agents: ## Search Types -Cognee offers these modes: +Cognee offers these modes. GRAPH_COMPLETION is the default mode. -### GRAPH_COMPLETION -Best for: **High-level understanding and reasoning** -- Returns graph-completion outputs over the knowledge graph -- Good for: "What projects am I working on?" or "Summarize my notes about X" - -### Chunks -Best for: **Specific text matching** -- Returns raw document chunks -- Similar to traditional vector search -- Good for: Finding exact quotes or specific information - -### Summaries -Best for: **Document overviews** -- Returns condensed summaries -- Good for: Quick scanning of content +Learn more from [cognee documentation](https://docs.cognee.ai/core-concepts/main-operations/search) ## Usage @@ -167,8 +139,8 @@ agents: ### Manual Sync and Status ```bash -# Force sync + cognify for the current agent -clawdbot memory status --index --json +# Manually sync cognee memory for the current agent +pnpm moltbot memory status --index --json ``` ## How It Works @@ -188,12 +160,12 @@ clawdbot memory status --index --json | Feature | Cognee | SQLite (Default) | |---------|--------|------------------| | **Setup** | Requires Docker/cloud | Built-in, no setup | -| **Offline** | No (needs service) | Yes (fully local) | +| **Offline** | Yes (Optional cloud) | Yes (fully local) | | **Search** | Knowledge graph + LLM | Vector + BM25 hybrid | | **Entities** | Extracted automatically | Not available | | **Relationships** | Yes (graph-based) | No | | **Speed** | Slower (API calls) | Faster (local DB) | -| **Memory** | Stored externally | SQLite file | +| **Memory** | Stored locally (Optional cognee configs) | SQLite file | | **Best for** | Rich context, reasoning | Fast lookup, privacy | ## Troubleshooting @@ -226,11 +198,11 @@ clawdbot memory status --index --json ### 401 Unauthorized (Cognee auth) -**Cause**: Cognee auth is enabled, but Clawdbot is missing/using an invalid `COGNEE_API_KEY`. +**Cause**: Cognee auth is enabled, but Moltbot is missing/using an invalid `COGNEE_API_KEY`. **Fix**: 1. Log in to Cognee and get a fresh access token. -2. Update `COGNEE_API_KEY` in `examples/.env`. +2. Update `COGNEE_API_KEY` in `.clawd/.env`. 3. Restart the gateway with env loaded (see Step 3 above). ## Advanced Configuration @@ -252,12 +224,6 @@ agents: maxResults: 10 ``` -### Hybrid Setup (Not Yet Supported) - -Future versions may support using both Cognee and SQLite: -- Cognee for semantic understanding -- SQLite for fast local lookup - ## Docker Production Tips ### Health Checks @@ -300,13 +266,6 @@ volumes: - ./cognee_logs:/app/logs ``` -## Roadmap - -Planned features: -- [ ] Hybrid mode (Cognee + SQLite) -- [ ] Graph visualization export -- [ ] Manual entity management - ## Resources - [Cognee Documentation](https://docs.cognee.ai/) diff --git a/src/memory/cognee-client.test.ts b/src/memory/cognee-client.test.ts index 822de5dd4..91b5c1a78 100644 --- a/src/memory/cognee-client.test.ts +++ b/src/memory/cognee-client.test.ts @@ -2,9 +2,13 @@ import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; import { CogneeClient } from "./cognee-client.js"; import { request } from "undici"; -vi.mock("undici", () => ({ - request: vi.fn(), -})); +vi.mock("undici", async () => { + const actual = await vi.importActual("undici"); + return { + ...actual, + request: vi.fn(), + }; +}); describe("CogneeClient", () => { beforeEach(() => { @@ -73,7 +77,9 @@ describe("CogneeClient", () => { data: "Test data", datasetName: "test-dataset", }), - ).rejects.toThrow("Cognee add failed with status 500"); + ).rejects.toThrow( + "Cognee add request failed: Cognee add failed with status 500: Internal server error", + ); }); }); diff --git a/src/memory/cognee-client.ts b/src/memory/cognee-client.ts index c442460d6..b092239ae 100644 --- a/src/memory/cognee-client.ts +++ b/src/memory/cognee-client.ts @@ -295,16 +295,37 @@ export class CogneeClient { if (item && typeof item === "object") { const record = item as Record; + const hasStructuredFields = + "id" in record || "text" in record || "score" in record || "metadata" in record; const raw = record.search_result ?? record.result ?? record.context ?? record.text ?? record; const text = typeof raw === "string" ? raw : JSON.stringify(raw, null, 2); - const metadata = + const datasetMetadata = record.dataset_name || record.dataset_id ? { datasetName: record.dataset_name, datasetId: record.dataset_id, } : undefined; + const recordMetadata = + record.metadata && typeof record.metadata === "object" + ? (record.metadata as Record) + : undefined; + const metadata = recordMetadata + ? datasetMetadata + ? { ...datasetMetadata, ...recordMetadata } + : recordMetadata + : datasetMetadata; + + if (hasStructuredFields) { + return { + id: typeof record.id === "string" ? record.id : `result-${index}`, + text, + score: typeof record.score === "number" ? record.score : 0, + metadata, + }; + } + return { id: `result-${index}`, text, score: 0, metadata }; } diff --git a/src/memory/cognee-provider.test.ts b/src/memory/cognee-provider.test.ts index d0bd31bad..3bc90c1b0 100644 --- a/src/memory/cognee-provider.test.ts +++ b/src/memory/cognee-provider.test.ts @@ -2,19 +2,19 @@ import { describe, expect, it, vi, beforeEach } from "vitest"; import { CogneeMemoryProvider } from "./cognee-provider.js"; import type { ClawdbotConfig } from "../config/config.js"; -vi.mock("./cognee-client.js", () => ({ - CogneeClient: vi.fn().mockImplementation(() => ({ - healthCheck: vi.fn().mockResolvedValue(true), - add: vi.fn().mockResolvedValue({ +vi.mock("./cognee-client.js", () => { + class CogneeClient { + healthCheck = vi.fn().mockResolvedValue(true); + add = vi.fn().mockResolvedValue({ datasetId: "test-dataset-id", datasetName: "test-dataset", message: "Success", - }), - cognify: vi.fn().mockResolvedValue({ + }); + cognify = vi.fn().mockResolvedValue({ status: "success", message: "Cognify completed", - }), - search: vi.fn().mockResolvedValue({ + }); + search = vi.fn().mockResolvedValue({ results: [ { id: "result-1", @@ -28,8 +28,8 @@ vi.mock("./cognee-client.js", () => ({ ], query: "test query", searchType: "GRAPH_COMPLETION", - }), - status: vi.fn().mockResolvedValue({ + }); + status = vi.fn().mockResolvedValue({ status: "healthy", version: "1.0.0", datasets: [ @@ -39,9 +39,11 @@ vi.mock("./cognee-client.js", () => ({ documentCount: 5, }, ], - }), - })), -})); + }); + } + + return { CogneeClient }; +}); vi.mock("./internal.js", () => ({ listMemoryFiles: vi.fn().mockResolvedValue([]), From c71eca52371b154a1254b69c222b1bd5a348dfde Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Thu, 29 Jan 2026 20:48:23 +0100 Subject: [PATCH 5/8] add update --- examples/cognee-docker-compose.yaml | 3 +- src/cli/memory-cli.ts | 14 ++- src/memory/cognee-client.ts | 91 ++++++++++++++++ src/memory/cognee-provider.ts | 157 +++++++++++++++++++++++++--- src/memory/manager.ts | 1 + 5 files changed, 246 insertions(+), 20 deletions(-) diff --git a/examples/cognee-docker-compose.yaml b/examples/cognee-docker-compose.yaml index 8269e14ec..d272378af 100644 --- a/examples/cognee-docker-compose.yaml +++ b/examples/cognee-docker-compose.yaml @@ -11,8 +11,6 @@ # - Vector DB: LanceDB (file-based) # - Graph DB: Kuzu (file-based) -version: "3.8" - services: cognee: image: cognee/cognee:latest @@ -21,6 +19,7 @@ services: - "127.0.0.1:8000:8000" environment: - LLM_API_KEY=${LLM_API_KEY} + - ENABLE_BACKEND_ACCESS_CONTROL=false volumes: - cognee_data:/app/cognee/.cognee_system restart: unless-stopped diff --git a/src/cli/memory-cli.ts b/src/cli/memory-cli.ts index 68894adf5..222304fef 100644 --- a/src/cli/memory-cli.ts +++ b/src/cli/memory-cli.ts @@ -24,6 +24,7 @@ type MemoryCommandOptions = { json?: boolean; deep?: boolean; index?: boolean; + updateCognee?: boolean; verbose?: boolean; }; @@ -197,6 +198,7 @@ async function scanMemorySources(params: { export async function runMemoryStatus(opts: MemoryCommandOptions) { setVerbose(Boolean(opts.verbose)); + const updateCognee = opts.updateCognee ?? process.argv.includes("--update-cognee"); const cfg = loadConfig(); const agentIds = resolveAgentIds(cfg, opts.agent); const allResults: Array<{ @@ -205,6 +207,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { embeddingProbe?: Awaited>; indexError?: string; scan?: MemorySourceScan; + cogneeUpdate?: boolean; }> = []; for (const agentId of agentIds) { @@ -240,6 +243,7 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { try { await manager.sync({ reason: "cli", + update: updateCognee, progress: (syncUpdate) => { update({ completed: syncUpdate.completed, @@ -269,7 +273,14 @@ export async function runMemoryStatus(opts: MemoryCommandOptions) { agentId, sources, }); - allResults.push({ agentId, status, embeddingProbe, indexError, scan }); + allResults.push({ + agentId, + status, + embeddingProbe, + indexError, + scan, + cogneeUpdate: Boolean(updateCognee), + }); }, }); } @@ -434,6 +445,7 @@ export function registerMemoryCli(program: Command) { .option("--json", "Print JSON") .option("--deep", "Probe embedding provider availability") .option("--index", "Reindex if dirty (implies --deep)") + .option("--update-cognee", "Use Cognee update when file data ids are known", false) .option("--verbose", "Verbose logging", false) .action(async (opts: MemoryCommandOptions) => { await runMemoryStatus(opts); diff --git a/src/memory/cognee-client.ts b/src/memory/cognee-client.ts index b092239ae..6ed294e26 100644 --- a/src/memory/cognee-client.ts +++ b/src/memory/cognee-client.ts @@ -24,6 +24,21 @@ export type CogneeAddResponse = { datasetId: string; datasetName: string; message: string; + dataId?: string; +}; + +export type CogneeUpdateRequest = { + dataId: string; + datasetId: string; + data: string; +}; + +export type CogneeUpdateResponse = { + datasetId?: string; + datasetName?: string; + message?: string; + status?: string; + dataId?: string; }; export type CogneeCognifyRequest = { @@ -119,12 +134,15 @@ export class CogneeClient { dataset_id: string; dataset_name: string; message: string; + data_id?: unknown; + data_ingestion_info?: unknown; }; return { datasetId: data.dataset_id, datasetName: data.dataset_name, message: data.message, + dataId: this.extractDataId(data.data_id ?? data.data_ingestion_info), }; } catch (error) { log.error("Failed to add data to Cognee", { error }); @@ -134,6 +152,63 @@ export class CogneeClient { } } + async update(req: CogneeUpdateRequest): Promise { + const url = new URL(`${this.baseUrl}${API_PREFIX}/update`); + url.searchParams.set("data_id", req.dataId); + url.searchParams.set("dataset_id", req.datasetId); + const headers: Record = {}; + if (this.apiKey) { + headers.Authorization = `Bearer ${this.apiKey}`; + headers["X-Api-Key"] = this.apiKey; + } + + log.debug("Updating data in Cognee", { + url: url.toString(), + dataLength: req.data.length, + }); + + try { + const formData = new FormData(); + const blob = new Blob([req.data], { type: "text/plain" }); + formData.append("data", blob, "clawdbot-memory.txt"); + + const response = await request(url.toString(), { + method: "PATCH", + headers, + body: formData, + bodyTimeout: this.timeoutMs, + headersTimeout: this.timeoutMs, + }); + + if (response.statusCode !== 200) { + const errorText = await response.body.text(); + throw new Error(`Cognee update failed with status ${response.statusCode}: ${errorText}`); + } + + const data = (await response.body.json()) as { + status?: string; + message?: string; + dataset_id?: string; + dataset_name?: string; + data_id?: unknown; + data_ingestion_info?: unknown; + }; + + return { + status: data.status, + message: data.message, + datasetId: data.dataset_id ?? req.datasetId, + datasetName: data.dataset_name, + dataId: this.extractDataId(data.data_id ?? data.data_ingestion_info) ?? req.dataId, + }; + } catch (error) { + log.error("Failed to update data in Cognee", { error }); + throw new Error( + `Cognee update request failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + async cognify(req: CogneeCognifyRequest = {}): Promise { const url = `${this.baseUrl}${API_PREFIX}/cognify`; const headers: Record = { @@ -265,6 +340,22 @@ export class CogneeClient { } } + private extractDataId(value: unknown): string | undefined { + if (!value) return undefined; + if (typeof value === "string") return value; + if (Array.isArray(value)) { + for (const entry of value) { + const id = this.extractDataId(entry); + if (id) return id; + } + return undefined; + } + if (typeof value !== "object") return undefined; + const record = value as { data_id?: unknown; data_ingestion_info?: unknown }; + if (typeof record.data_id === "string") return record.data_id; + return this.extractDataId(record.data_ingestion_info); + } + async healthCheck(): Promise { try { await this.status(); diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts index 44ecc309b..2ec6015ab 100644 --- a/src/memory/cognee-provider.ts +++ b/src/memory/cognee-provider.ts @@ -1,7 +1,9 @@ import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import type { MoltbotConfig } from "../config/config.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; +import { resolveStateDir } from "../config/paths.js"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import type { MemorySearchResult } from "./index.js"; @@ -25,6 +27,12 @@ const DEFAULT_AUTO_COGNIFY = true; const DEFAULT_COGNIFY_BATCH_SIZE = 100; const SNIPPET_MAX_CHARS = 700; +type CogneeSyncIndex = { + datasetId?: string; + datasetName?: string; + files: Record; +}; + export type CogneeProviderConfig = { baseUrl?: string; apiKey?: string; @@ -51,6 +59,10 @@ export class CogneeMemoryProvider { private readonly sources: Set; private datasetId?: string; private syncedFiles = new Map(); // path -> hash + private readonly syncIndexPath: string; + private syncIndexLoaded = false; + private syncIndex: CogneeSyncIndex = { files: {} }; + private syncIndexDirty = false; constructor( cfg: MoltbotConfig, @@ -75,6 +87,12 @@ export class CogneeMemoryProvider { this.autoCognify = config.autoCognify ?? DEFAULT_AUTO_COGNIFY; this.cognifyBatchSize = config.cognifyBatchSize || DEFAULT_COGNIFY_BATCH_SIZE; this.sources = new Set(sources); + this.syncIndexPath = path.join( + resolveStateDir(process.env, os.homedir), + "memory", + "cognee", + `${agentId}.json`, + ); log.info("Cognee memory provider initialized", { agentId, @@ -91,30 +109,38 @@ export class CogneeMemoryProvider { async sync(params?: { reason?: string; force?: boolean; + update?: boolean; progress?: (update: { completed: number; total: number; label?: string }) => void; }): Promise { log.info("Starting Cognee memory sync", { agentId: this.agentId }); let addedCount = 0; + await this.loadSyncIndex(); + const force = Boolean(params?.force); + const update = Boolean(params?.update); // Sync memory files if (this.sources.has("memory")) { const memoryFiles = await this.collectMemoryFiles(); - addedCount += await this.syncFiles(memoryFiles, "memory"); + addedCount += await this.syncFiles(memoryFiles, "memory", { update }); } // Sync session transcripts if (this.sources.has("sessions")) { const sessionFiles = await this.collectSessionFiles(); - addedCount += await this.syncFiles(sessionFiles, "sessions"); + addedCount += await this.syncFiles(sessionFiles, "sessions", { update }); } // Run cognify if auto-enabled and files were added - if (this.autoCognify && addedCount > 0) { + if ((this.autoCognify && addedCount > 0) || (this.autoCognify && force)) { log.info("Running cognify after sync", { addedCount }); await this.cognify(); } + if (this.syncIndexDirty) { + await this.saveSyncIndex(); + } + log.info("Cognee memory sync completed", { agentId: this.agentId, addedCount, @@ -288,6 +314,60 @@ export class CogneeMemoryProvider { async close(): Promise {} + private async loadSyncIndex(): Promise { + if (this.syncIndexLoaded) return; + this.syncIndexLoaded = true; + try { + const raw = await fs.readFile(this.syncIndexPath, "utf-8"); + const parsed = JSON.parse(raw) as CogneeSyncIndex; + if (!parsed || typeof parsed !== "object") return; + this.syncIndex = { + datasetId: parsed.datasetId, + datasetName: parsed.datasetName, + files: parsed.files && typeof parsed.files === "object" ? parsed.files : {}, + }; + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code !== "ENOENT") { + log.warn("Failed to load Cognee sync index", { error }); + } + } + + if (this.syncIndex.datasetName && this.syncIndex.datasetName !== this.datasetName) { + log.info("Resetting Cognee sync index (dataset name changed)", { + from: this.syncIndex.datasetName, + to: this.datasetName, + }); + this.syncIndex = { files: {} }; + this.syncIndexDirty = true; + } + + if (this.syncIndex.datasetId && this.datasetId && this.syncIndex.datasetId !== this.datasetId) { + log.info("Resetting Cognee sync index (dataset id changed)", { + from: this.syncIndex.datasetId, + to: this.datasetId, + }); + this.syncIndex = { files: {} }; + this.syncIndexDirty = true; + } + + if (!this.datasetId && this.syncIndex.datasetId) { + this.datasetId = this.syncIndex.datasetId; + } + } + + private async saveSyncIndex(): Promise { + const dir = path.dirname(this.syncIndexPath); + await fs.mkdir(dir, { recursive: true }); + const payload: CogneeSyncIndex = { + datasetId: this.datasetId ?? this.syncIndex.datasetId, + datasetName: this.datasetName, + files: this.syncIndex.files, + }; + await fs.writeFile(this.syncIndexPath, JSON.stringify(payload, null, 2), "utf-8"); + this.syncIndexDirty = false; + } + private async collectMemoryFiles(): Promise { const files: MemoryFileEntry[] = []; const memoryPaths = await listMemoryFiles(this.workspaceDir); @@ -337,9 +417,14 @@ export class CogneeMemoryProvider { return files; } - private async syncFiles(files: MemoryFileEntry[], source: CogneeMemorySource): Promise { + private async syncFiles( + files: MemoryFileEntry[], + source: CogneeMemorySource, + opts?: { update?: boolean }, + ): Promise { let addedCount = 0; const batchSize = this.cognifyBatchSize; + const update = Boolean(opts?.update); for (let i = 0; i < files.length; i += batchSize) { const batch = files.slice(i, i + batchSize); @@ -363,24 +448,62 @@ export class CogneeMemoryProvider { const dataWithMetadata = `# ${file.path}\n\n${content}\n\n---\nMetadata: ${JSON.stringify(metadata)}`; - const response = await this.client.add({ - data: dataWithMetadata, - datasetName: this.datasetName, - }); + const record = this.syncIndex.files[file.path]; + const datasetId = this.datasetId ?? this.syncIndex.datasetId; + const canUpdate = update && record?.dataId && datasetId; - if (!this.datasetId) { - this.datasetId = response.datasetId; + if (canUpdate && datasetId && record?.dataId) { + await this.client.update({ + dataId: record.dataId, + datasetId, + data: dataWithMetadata, + }); + addedCount++; + log.debug("Updated file in Cognee", { + path: file.path, + datasetId, + dataId: record.dataId, + }); + } else { + const response = await this.client.add({ + data: dataWithMetadata, + datasetName: this.datasetName, + datasetId, + }); + + if (!this.datasetId) { + this.datasetId = response.datasetId; + } + if (response.dataId) { + this.syncIndex.files[file.path] = { + hash: file.hash, + dataId: response.dataId, + }; + } else { + this.syncIndex.files[file.path] = { hash: file.hash }; + } + this.syncIndex.datasetId = this.datasetId ?? this.syncIndex.datasetId; + this.syncIndex.datasetName = this.datasetName; + this.syncIndexDirty = true; + + this.syncedFiles.set(file.path, file.hash); + addedCount++; + + log.debug("Added file to Cognee", { + path: file.path, + datasetId: response.datasetId, + }); + continue; } + const dataId = record?.dataId; + this.syncIndex.files[file.path] = { hash: file.hash, dataId }; + this.syncIndex.datasetId = datasetId ?? this.syncIndex.datasetId; + this.syncIndex.datasetName = this.datasetName; + this.syncIndexDirty = true; this.syncedFiles.set(file.path, file.hash); - addedCount++; - - log.debug("Added file to Cognee", { - path: file.path, - datasetId: response.datasetId, - }); } catch (error) { - log.error("Failed to add file to Cognee", { path: file.path, error }); + log.error("Failed to sync file to Cognee", { path: file.path, error }); } } } diff --git a/src/memory/manager.ts b/src/memory/manager.ts index 5fbc3ceea..627c7720c 100644 --- a/src/memory/manager.ts +++ b/src/memory/manager.ts @@ -384,6 +384,7 @@ export class MemoryIndexManager { async sync(params?: { reason?: string; force?: boolean; + update?: boolean; progress?: (update: MemorySyncProgressUpdate) => void; }): Promise { if (this.syncing) return this.syncing; From e8b4540c2568462b1ff1ee61ae239e9fed55b371 Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Thu, 29 Jan 2026 21:55:47 +0100 Subject: [PATCH 6/8] add changes to docs and test --- docs/memory-cognee.md | 226 ++-------------------------- examples/cognee-config.yaml | 2 +- src/memory/cognee-client.test.ts | 227 ----------------------------- src/memory/cognee-provider.test.ts | 182 ++++++++--------------- 4 files changed, 75 insertions(+), 562 deletions(-) delete mode 100644 src/memory/cognee-client.test.ts diff --git a/docs/memory-cognee.md b/docs/memory-cognee.md index 623e6c1c7..cc5a1e943 100644 --- a/docs/memory-cognee.md +++ b/docs/memory-cognee.md @@ -1,66 +1,37 @@ --- -summary: "Cognee knowledge graph memory: setup, Docker config, and usage" +summary: "Cognee memory quick setup and usage" read_when: - Setting up Cognee memory provider - Configuring knowledge graph memory - - Running Cognee with Docker --- # Cognee Memory Provider -MoltBot supports [**Cognee**](https://www.cognee.ai/) as an optional memory provider. Unlike the default SQLite-based vector memory, Cognee builds a knowledge graph with entity extraction and semantic relationships, providing richer contextual memory for your AI agent. +Moltbot supports [Cognee](https://www.cognee.ai/) - [open source AI memory](https://github.com/topoteretes/cognee) - as an optional memory provider. Cognee builds knowledge graph memory backed by embeddings from any data and can be run locally with Docker. Learn more from [Cognee Documentation](https://docs.cognee.ai/). -## What is Cognee? +## Quickstart with Docker -Cognee is an [open source AI memory](https://github.com/topoteretes/cognee) framework that: -- Ingests unstructured/structured data from 30+ sources -- Extracts entities (people, places, concepts) from documents -- Builds a knowledge graph of relationships backed by embeddings -- Enables semantic search with LLM-powered reasoning -- Supports multiple search modes (e.g., GRAPH_COMPLETION, chunks, summaries) - -Learn more at [docs.cognee.ai](https://docs.cognee.ai/). - -## Setup Options - -### Option 1: Local Docker (Recommended) - -Use the repo example compose file: +Run the example compose file: ```bash docker compose -f examples/cognee-docker-compose.yaml up -d ``` - -This binds to `127.0.0.1:8000`, so Cognee is only reachable from your machine. - -**Verify:** +Verify: ```bash curl http://localhost:8000/health ``` -You can use `http://localhost:8000/docs` to register user and login to get your token. - -### Option 2: Cognee Cloud - -Use the hosted Cognee service: - -1. Sign up at [platform.cognee.ai](https://platform.cognee.ai/) -2. Get your API key from the dashboard -3. Use base URL: `https://cognee--cognee-saas-backend-serve.modal.run` - ## Configuration -Moltbot reads `~/.clawdbot/moltbot.json` (JSON5). For local + secure setup, use an env var for the Cognee token and reference it in config. - -### Step 1: Put tokens in `~/.clawdbot/.env` +Put the token in `~/.clawdbot/.env`: ```bash COGNEE_API_KEY="your-cognee-access-token" CLAWDBOT_GATEWAY_TOKEN="your-random-gateway-token" ``` -### Step 2: Configure Cognee in `~/.clawdbot/moltbot.json` +Configure `~/.clawdbot/moltbot.json` (JSON5): ```json5 { @@ -86,195 +57,30 @@ CLAWDBOT_GATEWAY_TOKEN="your-random-gateway-token" } ``` -### Step 3: Start the gateway with env loaded +Start the gateway with env loaded: ```zsh set -a; source "$HOME/.clawdbot/.env"; set +a pnpm moltbot gateway --port 18789 --token "$CLAWDBOT_GATEWAY_TOKEN" --verbose ``` - - -## Configuration Options - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `baseUrl` | string | `http://localhost:8000` | Cognee API endpoint | -| `apiKey` | string | - | Access token (required if Cognee auth is enabled) | -| `datasetName` | string | `"clawdbot"` | Dataset name for organizing memories | -| `searchType` | string | `"GRAPH_COMPLETION"` | Search mode: `GRAPH_COMPLETION`, `chunks`, or `summaries` | -| `maxResults` | number | `6` | Maximum search results returned | -| `autoCognify` | boolean | `true` | Auto-process documents after adding | -| `cognifyBatchSize` | number | `100` | Batch size for processing | -| `timeoutSeconds` | number | `180` | Request timeout in seconds | - -## Search Types - -Cognee offers these modes. GRAPH_COMPLETION is the default mode. - -Learn more from [cognee documentation](https://docs.cognee.ai/core-concepts/main-operations/search) - ## Usage -### Memory Files +Cognee indexes `MEMORY.md` in workspace root, `memory/*.md`, and session transcripts when `sources: ["sessions"]` is enabled. -Cognee automatically syncs your memory files: -- `MEMORY.md` or `memory.md` in workspace root -- All `*.md` files in `memory/` directory - -### Session Transcripts (Optional) - -Enable session memory to index conversation history: - -```yaml -agents: - defaults: - memorySearch: - provider: cognee - sources: [memory, sessions] # Include sessions - experimental: - sessionMemory: true -``` - -### Manual Sync and Status +1. Initial index and status: ```bash -# Manually sync cognee memory for the current agent pnpm moltbot memory status --index --json ``` -## How It Works +2. Memory updates: -1. **Add**: Memory files are sent to Cognee with metadata -2. **Cognify**: Cognee processes documents: - - Extracts entities (people, places, concepts) - - Identifies relationships - - Builds knowledge graph -3. **Search**: Agent queries use semantic search: - - Searches knowledge graph - - Returns relevant insights/chunks/summaries - - Includes metadata and scores - -## Comparison: Cognee vs SQLite Memory - -| Feature | Cognee | SQLite (Default) | -|---------|--------|------------------| -| **Setup** | Requires Docker/cloud | Built-in, no setup | -| **Offline** | Yes (Optional cloud) | Yes (fully local) | -| **Search** | Knowledge graph + LLM | Vector + BM25 hybrid | -| **Entities** | Extracted automatically | Not available | -| **Relationships** | Yes (graph-based) | No | -| **Speed** | Slower (API calls) | Faster (local DB) | -| **Memory** | Stored locally (Optional cognee configs) | SQLite file | -| **Best for** | Rich context, reasoning | Fast lookup, privacy | +```bash +pnpm moltbot memory status --index --update-cognee --json +``` ## Troubleshooting -### Connection Failed - -**Error**: `Failed to connect to Cognee at http://localhost:8000` - -**Solutions**: -1. Verify Docker is running: `docker ps | grep cognee` -2. Check Cognee logs: `docker logs cognee` -3. Test manually: `curl http://localhost:8000/health` -4. Ensure port 8000 is not blocked - -### Slow Performance - -**Solutions**: -1. Reduce `maxResults` (try 3-5 instead of 10+) -2. Use `searchType: "chunks"` for faster results -3. Set `autoCognify: false` and cognify manually -4. Check Docker resource limits - -### Out of Memory - -**Solutions**: -1. Increase Docker memory limit (Docker Desktop settings) -2. Reduce `cognifyBatchSize` (try 50 instead of 100) -3. Process fewer files at once -4. Clear old datasets via Cognee API - -### 401 Unauthorized (Cognee auth) - -**Cause**: Cognee auth is enabled, but Moltbot is missing/using an invalid `COGNEE_API_KEY`. - -**Fix**: -1. Log in to Cognee and get a fresh access token. -2. Update `COGNEE_API_KEY` in `.clawd/.env`. -3. Restart the gateway with env loaded (see Step 3 above). - -## Advanced Configuration - -### Per-Agent Override - -```yaml -agents: - defaults: - memorySearch: - provider: openai # Default for all agents - - agents: - research-bot: - memorySearch: - provider: cognee # Override for this agent - cognee: - searchType: GRAPH_COMPLETION - maxResults: 10 -``` - -## Docker Production Tips - -### Health Checks - -Add health checks to docker-compose.yml: - -```yaml -services: - cognee: - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8000/health"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 40s -``` - -### Resource Limits - -```yaml -services: - cognee: - deploy: - resources: - limits: - cpus: '2' - memory: 4G - reservations: - cpus: '1' - memory: 2G -``` - -### Persistent Storage - -Mount volumes for persistence: - -```yaml -volumes: - - ./cognee_data:/app/cognee/.cognee_system - - ./cognee_logs:/app/logs -``` - -## Resources - -- [Cognee Documentation](https://docs.cognee.ai/) -- [Cognee GitHub](https://github.com/topoteretes/cognee) -- [Clawdbot Memory Guide](/memory) -- [Docker Setup Guide](/install/docker) - -## Feedback - -Cognee integration is new. Report issues at: -- Clawdbot: [github.com/clawdbot/clawdbot/issues](https://github.com/clawdbot/clawdbot/issues) -- Cognee: [github.com/topoteretes/cognee/issues](https://github.com/topoteretes/cognee/issues) +- Connection test: `curl http://localhost:8000/health` +- Reset cached values that Moltbot reuses: `mv "$HOME/.clawdbot/memory/cognee/main.json" "$HOME/.clawdbot/memory/cognee/main.json.bak"` \ No newline at end of file diff --git a/examples/cognee-config.yaml b/examples/cognee-config.yaml index 5412b992b..411e83ca9 100644 --- a/examples/cognee-config.yaml +++ b/examples/cognee-config.yaml @@ -35,7 +35,7 @@ agents: cognifyBatchSize: 100 # Request timeout in seconds - timeoutSeconds: 30 + timeoutSeconds: 180 # Enable experimental session memory indexing experimental: diff --git a/src/memory/cognee-client.test.ts b/src/memory/cognee-client.test.ts deleted file mode 100644 index 91b5c1a78..000000000 --- a/src/memory/cognee-client.test.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { describe, expect, it, vi, beforeEach, afterEach } from "vitest"; -import { CogneeClient } from "./cognee-client.js"; -import { request } from "undici"; - -vi.mock("undici", async () => { - const actual = await vi.importActual("undici"); - return { - ...actual, - request: vi.fn(), - }; -}); - -describe("CogneeClient", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - afterEach(() => { - vi.restoreAllMocks(); - }); - - describe("add", () => { - it("should add data successfully", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ - dataset_id: "test-dataset-id", - dataset_name: "test-dataset", - message: "Data added successfully", - }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient({ - baseUrl: "http://localhost:8000", - apiKey: "test-key", - }); - - const result = await client.add({ - data: "Test data", - datasetName: "test-dataset", - }); - - expect(result).toEqual({ - datasetId: "test-dataset-id", - datasetName: "test-dataset", - message: "Data added successfully", - }); - expect(request).toHaveBeenCalledWith( - "http://localhost:8000/api/v1/add", - expect.objectContaining({ - method: "POST", - headers: expect.objectContaining({ - Authorization: "Bearer test-key", - "X-Api-Key": "test-key", - }), - }), - ); - }); - - it("should handle errors", async () => { - const mockResponse = { - statusCode: 500, - body: { - text: vi.fn().mockResolvedValue("Internal server error"), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient(); - - await expect( - client.add({ - data: "Test data", - datasetName: "test-dataset", - }), - ).rejects.toThrow( - "Cognee add request failed: Cognee add failed with status 500: Internal server error", - ); - }); - }); - - describe("cognify", () => { - it("should run cognify successfully", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ - status: "success", - message: "Cognify completed", - }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient({ - baseUrl: "http://localhost:8000", - }); - - const result = await client.cognify({ - datasetIds: ["dataset-1"], - }); - - expect(result).toEqual({ - status: "success", - message: "Cognify completed", - }); - }); - }); - - describe("search", () => { - it("should search successfully", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ - results: [ - { - id: "result-1", - text: "Test result", - score: 0.9, - metadata: { path: "test.md" }, - }, - ], - query: "test query", - search_type: "GRAPH_COMPLETION", - }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient(); - - const result = await client.search({ - queryText: "test query", - searchType: "GRAPH_COMPLETION", - }); - - expect(result.results).toHaveLength(1); - expect(result.results[0]).toEqual({ - id: "result-1", - text: "Test result", - score: 0.9, - metadata: { path: "test.md" }, - }); - expect(result.query).toBe("test query"); - }); - - it("should use default search type", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ - results: [], - query: "test", - search_type: "GRAPH_COMPLETION", - }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient(); - await client.search({ queryText: "test" }); - - expect(request).toHaveBeenCalledWith( - expect.any(String), - expect.objectContaining({ - body: expect.stringContaining('"searchType":"GRAPH_COMPLETION"'), - }), - ); - }); - }); - - describe("status", () => { - it("should get status successfully", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ status: "healthy" }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient(); - - const result = await client.status(); - - expect(result).toEqual({ - status: "healthy", - }); - }); - }); - - describe("healthCheck", () => { - it("should return true when status is successful", async () => { - const mockResponse = { - statusCode: 200, - body: { - json: vi.fn().mockResolvedValue({ status: "healthy" }), - text: vi.fn(), - }, - }; - vi.mocked(request).mockResolvedValue(mockResponse as any); - - const client = new CogneeClient(); - const result = await client.healthCheck(); - - expect(result).toBe(true); - }); - - it("should return false when status fails", async () => { - vi.mocked(request).mockRejectedValue(new Error("Connection failed")); - - const client = new CogneeClient(); - const result = await client.healthCheck(); - - expect(result).toBe(false); - }); - }); -}); diff --git a/src/memory/cognee-provider.test.ts b/src/memory/cognee-provider.test.ts index 3bc90c1b0..500bdc356 100644 --- a/src/memory/cognee-provider.test.ts +++ b/src/memory/cognee-provider.test.ts @@ -1,27 +1,34 @@ -import { describe, expect, it, vi, beforeEach } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { CogneeMemoryProvider } from "./cognee-provider.js"; import type { ClawdbotConfig } from "../config/config.js"; +const searchMock = vi.fn(); + vi.mock("./cognee-client.js", () => { class CogneeClient { - healthCheck = vi.fn().mockResolvedValue(true); - add = vi.fn().mockResolvedValue({ - datasetId: "test-dataset-id", - datasetName: "test-dataset", - message: "Success", - }); - cognify = vi.fn().mockResolvedValue({ - status: "success", - message: "Cognify completed", - }); - search = vi.fn().mockResolvedValue({ + search = searchMock; + } + + return { CogneeClient }; +}); + +describe("CogneeMemoryProvider", () => { + it("maps search results into memory snippets", async () => { + const mockConfig: ClawdbotConfig = { + agents: { + defaults: { + workspace: "/tmp/test-workspace", + }, + }, + }; + searchMock.mockResolvedValue({ results: [ { id: "result-1", - text: "Test result text", + text: "A".repeat(800), score: 0.85, metadata: { - path: "test.md", + path: "memory/test.md", source: "memory", }, }, @@ -29,124 +36,51 @@ vi.mock("./cognee-client.js", () => { query: "test query", searchType: "GRAPH_COMPLETION", }); - status = vi.fn().mockResolvedValue({ - status: "healthy", - version: "1.0.0", - datasets: [ + + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); + + const results = await provider.search("test query"); + + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + path: "memory/test.md", + source: "memory", + score: 0.85, + }); + expect(results[0].snippet.length).toBeGreaterThan(700); + expect(results[0].snippet.endsWith("...")).toBe(true); + }); + + it("defaults missing metadata to safe values", async () => { + searchMock.mockResolvedValue({ + results: [ { - id: "test-dataset-id", - name: "clawdbot", - documentCount: 5, + id: "result-2", + text: "Short result", + score: 0.2, }, ], + query: "missing metadata", + searchType: "GRAPH_COMPLETION", }); - } - return { CogneeClient }; -}); - -vi.mock("./internal.js", () => ({ - listMemoryFiles: vi.fn().mockResolvedValue([]), - buildFileEntry: vi.fn(), - hashText: vi.fn().mockReturnValue("test-hash"), -})); - -vi.mock("node:fs/promises", () => ({ - default: { - readdir: vi.fn().mockResolvedValue([]), - readFile: vi.fn().mockResolvedValue("Test file content"), - stat: vi.fn().mockResolvedValue({ mtimeMs: 1234567890, size: 100 }), - }, -})); - -describe("CogneeMemoryProvider", () => { - const mockConfig: ClawdbotConfig = { - agents: { - defaults: { - workspace: "/tmp/test-workspace", + const mockConfig: ClawdbotConfig = { + agents: { + defaults: { + workspace: "/tmp/test-workspace", + }, }, - }, - }; + }; + const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - beforeEach(() => { - vi.clearAllMocks(); - }); + const results = await provider.search("missing metadata"); - describe("constructor", () => { - it("should initialize with default configuration", () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - - expect(provider).toBeDefined(); - }); - - it("should initialize with custom configuration", () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"], { - baseUrl: "http://custom:8000", - apiKey: "custom-key", - datasetName: "custom-dataset", - searchType: "chunks", - maxResults: 10, - }); - - expect(provider).toBeDefined(); - }); - }); - - describe("healthCheck", () => { - it("should perform health check", async () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - - const healthy = await provider.healthCheck(); - - expect(healthy).toBe(true); - }); - }); - - describe("search", () => { - it("should search and transform results", async () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - - const results = await provider.search("test query"); - - expect(results).toHaveLength(1); - expect(results[0]).toMatchObject({ - path: "test.md", - source: "memory", - score: 0.85, - snippet: "Test result text", - }); - }); - - it("should respect maxResults setting", async () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"], { - maxResults: 5, - }); - - const results = await provider.search("test query"); - - expect(results.length).toBeLessThanOrEqual(5); - }); - }); - - describe("cognify", () => { - it("should run cognify", async () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - - await expect(provider.cognify()).resolves.not.toThrow(); - }); - }); - - describe("getStatus", () => { - it("should return status information", async () => { - const provider = new CogneeMemoryProvider(mockConfig, "test-agent", ["memory"]); - - const status = await provider.getStatus(); - - expect(status).toMatchObject({ - connected: true, - datasetName: "clawdbot", - syncedFileCount: 0, - }); + expect(results).toHaveLength(1); + expect(results[0]).toMatchObject({ + path: "unknown", + source: "memory", + score: 0.2, + snippet: "Short result", }); }); }); From 454467ec5084825970464a8fa17d6edb34b920b1 Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Fri, 30 Jan 2026 14:58:31 +0100 Subject: [PATCH 7/8] add fixes --- src/agents/memory-search.ts | 3 ++- src/config/types.tools.ts | 4 ++-- src/memory/cognee-client.ts | 6 +++--- src/memory/cognee-provider.ts | 12 ++++++------ src/memory/search-manager.ts | 1 + 5 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/agents/memory-search.ts b/src/agents/memory-search.ts index 5659c8571..8cd1fdb73 100644 --- a/src/agents/memory-search.ts +++ b/src/agents/memory-search.ts @@ -9,6 +9,7 @@ import { resolveAgentConfig } from "./agent-scope.js"; export type ResolvedMemorySearchConfig = { enabled: boolean; sources: Array<"memory" | "sessions">; + extraPaths: string[]; provider: "openai" | "local" | "gemini" | "auto" | "cognee"; remote?: { baseUrl?: string; @@ -35,7 +36,7 @@ export type ResolvedMemorySearchConfig = { baseUrl?: string; apiKey?: string; datasetName?: string; - searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "CHUNKS" | "SUMMARIES"; maxResults?: number; timeoutSeconds?: number; autoCognify?: boolean; diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index 8ee3552b5..fcf93d976 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -271,8 +271,8 @@ export type MemorySearchConfig = { apiKey?: string; /** Dataset name for organizing memories (default: "clawdbot"). */ datasetName?: string; - /** Search type: "GRAPH_COMPLETION", "chunks", or "summaries" (default: "GRAPH_COMPLETION"). */ - searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; + /** Search type: "GRAPH_COMPLETION", "CHUNKS", or "SUMMARIES" (default: "GRAPH_COMPLETION"). */ + searchType?: "GRAPH_COMPLETION" | "CHUNKS" | "SUMMARIES"; /** Max results per search query (default: 6). */ maxResults?: number; /** Timeout for API requests in seconds (default: 30). */ diff --git a/src/memory/cognee-client.ts b/src/memory/cognee-client.ts index 6ed294e26..0b351ca2e 100644 --- a/src/memory/cognee-client.ts +++ b/src/memory/cognee-client.ts @@ -52,7 +52,7 @@ export type CogneeCognifyResponse = { export type CogneeSearchRequest = { queryText: string; - searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "CHUNKS" | "SUMMARIES"; datasetIds?: string[]; }; @@ -367,9 +367,9 @@ export class CogneeClient { private mapSearchType(type?: CogneeSearchRequest["searchType"]): CogneeSearchApiType { switch (type) { - case "chunks": + case "CHUNKS": return "CHUNKS"; - case "summaries": + case "SUMMARIES": return "SUMMARIES"; case "GRAPH_COMPLETION": default: diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts index 2ec6015ab..56b70adb6 100644 --- a/src/memory/cognee-provider.ts +++ b/src/memory/cognee-provider.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import type { MoltbotConfig } from "../config/config.js"; +import type { OpenClawConfig } from "../config/config.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; import { resolveStateDir } from "../config/paths.js"; import { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; @@ -37,7 +37,7 @@ export type CogneeProviderConfig = { baseUrl?: string; apiKey?: string; datasetName?: string; - searchType?: "GRAPH_COMPLETION" | "chunks" | "summaries"; + searchType?: "GRAPH_COMPLETION" | "CHUNKS" | "SUMMARIES"; maxResults?: number; timeoutSeconds?: number; autoCognify?: boolean; @@ -48,11 +48,11 @@ export type CogneeMemorySource = "memory" | "sessions"; export class CogneeMemoryProvider { private readonly client: CogneeClient; - private readonly cfg: MoltbotConfig; + private readonly cfg: OpenClawConfig; private readonly agentId: string; private readonly workspaceDir: string; private readonly datasetName: string; - private readonly searchType: "GRAPH_COMPLETION" | "chunks" | "summaries"; + private readonly searchType: "GRAPH_COMPLETION" | "CHUNKS" | "SUMMARIES"; private readonly maxResults: number; private readonly autoCognify: boolean; private readonly cognifyBatchSize: number; @@ -65,7 +65,7 @@ export class CogneeMemoryProvider { private syncIndexDirty = false; constructor( - cfg: MoltbotConfig, + cfg: OpenClawConfig, agentId: string, sources: Array, config: CogneeProviderConfig = {}, @@ -535,7 +535,7 @@ export class CogneeMemoryProvider { } export async function createCogneeProvider( - cfg: MoltbotConfig, + cfg: OpenClawConfig, agentId: string, sources: Array, config: CogneeProviderConfig = {}, diff --git a/src/memory/search-manager.ts b/src/memory/search-manager.ts index c442d11e9..77801292e 100644 --- a/src/memory/search-manager.ts +++ b/src/memory/search-manager.ts @@ -1,6 +1,7 @@ import type { OpenClawConfig } from "../config/config.js"; import type { MemoryIndexManager } from "./manager.js"; import type { CogneeMemoryProvider } from "./cognee-provider.js"; +import { resolveMemorySearchConfig } from "../agents/memory-search.js"; export type MemorySearchManagerResult = { manager: MemoryIndexManager | CogneeMemoryProvider | null; From 394831662cb2e0fe6507a02aed5d92250ac7c39c Mon Sep 17 00:00:00 2001 From: Hande <159312713+hande-k@users.noreply.github.com> Date: Fri, 30 Jan 2026 15:06:39 +0100 Subject: [PATCH 8/8] add fix --- src/memory/cognee-provider.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/memory/cognee-provider.ts b/src/memory/cognee-provider.ts index 56b70adb6..d9f3028c7 100644 --- a/src/memory/cognee-provider.ts +++ b/src/memory/cognee-provider.ts @@ -233,6 +233,7 @@ export class CogneeMemoryProvider { model: string; requestedProvider: string; sources: Array; + extraPaths: string[]; sourceCounts: Array<{ source: CogneeMemorySource; files: number; chunks: number }>; cache?: { enabled: boolean; entries?: number; maxEntries?: number }; fts?: { enabled: boolean; available: boolean; error?: string }; @@ -268,6 +269,7 @@ export class CogneeMemoryProvider { model: this.searchType, requestedProvider: "cognee", sources, + extraPaths: [], sourceCounts: sources.map((source) => ({ source, files, chunks: 0 })), vector: { enabled: false,