diff --git a/extensions/weave/README.md b/extensions/weave/README.md new file mode 100644 index 000000000..f1fbb957d --- /dev/null +++ b/extensions/weave/README.md @@ -0,0 +1,127 @@ +# W&B Weave Plugin for Clawdbot + +Integrates [Weights & Biases Weave](https://weave-docs.wandb.ai/) for comprehensive LLM observability and tracing. + +## Features + +- **Full System Prompt Capture** - Captures the complete system prompt, tracks changes across requests +- **Multi-Turn Conversation Tracing** - Tracks all LLM requests in a conversation, not just the first +- **Tool Call Spans** - Logs each tool call with inputs, outputs, and duration as child spans +- **Session Lifecycle Tracking** - Monitors session start/end with aggregated metrics +- **All LLM Providers** - Works with Claude, GPT, Gemini, Bedrock, and all pi-ai supported models + +## Installation + +The plugin is included in Clawdbot. Enable it in your configuration: + +```json +{ + "plugins": { + "entries": { + "weave": { + "enabled": true, + "config": { + "apiKey": "wandb_...", + "entity": "your-username" + } + } + } + } +} +``` + +## Configuration + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `apiKey` | string | **required** | W&B API key ([get one here](https://wandb.ai/settings)) | +| `entity` | string | **required** | W&B entity (username or team name) | +| `project` | string | `"clawdbot"` | Project name for traces | +| `autoTrace` | boolean | `true` | Automatically trace all agent runs | +| `traceToolCalls` | boolean | `true` | Log tool calls as child spans | +| `traceSessions` | boolean | `true` | Track session lifecycle | +| `baseUrl` | string | - | Custom W&B server URL (for self-hosted) | +| `sampleRate` | number | `1.0` | Trace sampling rate (0.0 to 1.0) | +| `debug` | boolean | `false` | Enable verbose debug logging | + +## Trace Data + +Each agent run trace includes: + +### Inputs +- User messages +- Session context (compaction summaries) + +### Outputs +- Assistant messages +- Final response +- Tool call results + +### Metadata +- **systemPrompt** - Full system prompt (latest version if changed) +- **systemPromptChanges** - Number of times the system prompt changed +- **llmRequests** - Array of all LLM API requests (each with full system prompt) +- **llmRequestCount** - Number of LLM requests made +- **llmTools** - Available tools/functions +- **toolCalls** - Detailed tool execution data + +### Tool Call Spans +Each tool call is logged as a child span with: +- Tool name +- Input parameters +- Output/result +- Duration +- Success/error status + +## Supported LLM Providers + +The plugin works with all providers supported by pi-ai: + +| Provider | API | Models | +|----------|-----|--------| +| Anthropic | `anthropic-messages` | Claude Opus 4.5, Claude Sonnet 4.5, etc. | +| Google | `google-generative-ai` | Gemini 3 Pro, Gemini 3 Flash, etc. | +| OpenAI | `openai-completions` | GPT-5.2, GPT-4o, o3, etc. | +| OpenAI | `openai-responses` | Newer response API | +| AWS | `bedrock-converse-stream` | Bedrock models | +| Google Cloud | `google-vertex` | Vertex AI models | +| Others | `openai-completions` | OpenRouter, DeepSeek, Grok, Mistral, etc. | + +## Viewing Traces + +1. Go to [wandb.ai](https://wandb.ai) +2. Navigate to your project (e.g., `your-username/clawdbot`) +3. Click on "Weave" in the left sidebar +4. Browse traces, view spans, and analyze metrics + +## Troubleshooting + +### No traces appearing +- Verify your API key is correct +- Check that `autoTrace` is `true` +- Look for errors in Clawdbot logs with `[weave]` prefix + +### Missing system prompt +- The `llm_request` hook must be enabled in Clawdbot core +- Check `systemPromptChanges` field to see if prompt changed during conversation + +### Partial traces +- Check `sampleRate` - if less than 1.0, some traces are skipped +- Ensure agent completes normally (crashes may lose trace data) + +## Development + +```bash +# Build +cd extensions/weave +pnpm build + +# Watch mode +pnpm dev +``` + +## Links + +- [W&B Weave Documentation](https://weave-docs.wandb.ai/) +- [Weave TypeScript SDK](https://www.npmjs.com/package/weave) +- [Clawdbot Plugin SDK](../../docs/plugins.md) diff --git a/extensions/weave/clawdbot.plugin.json b/extensions/weave/clawdbot.plugin.json new file mode 100644 index 000000000..b71bc2018 --- /dev/null +++ b/extensions/weave/clawdbot.plugin.json @@ -0,0 +1,92 @@ +{ + "id": "weave", + "name": "W&B Weave", + "description": "Weights & Biases Weave integration for LLM observability and tracing", + "version": "0.1.0", + "configSchema": { + "type": "object", + "properties": { + "apiKey": { + "type": "string", + "description": "W&B API key for authentication", + "uiHints": { + "label": "API Key", + "sensitive": true, + "help": "Get your API key from wandb.ai/settings" + } + }, + "entity": { + "type": "string", + "description": "W&B entity (username or team name)", + "uiHints": { + "label": "Entity", + "placeholder": "your-username" + } + }, + "project": { + "type": "string", + "description": "Default W&B project name for traces", + "default": "clawdbot", + "uiHints": { + "label": "Project", + "placeholder": "clawdbot" + } + }, + "autoTrace": { + "type": "boolean", + "description": "Automatically trace all agent runs", + "default": true, + "uiHints": { + "label": "Auto-trace Agent Runs" + } + }, + "traceToolCalls": { + "type": "boolean", + "description": "Log tool calls as child spans", + "default": true, + "uiHints": { + "label": "Trace Tool Calls" + } + }, + "traceSessions": { + "type": "boolean", + "description": "Track session lifecycle", + "default": true, + "uiHints": { + "label": "Trace Sessions" + } + }, + "baseUrl": { + "type": "string", + "description": "Custom W&B server URL (leave empty for default)", + "uiHints": { + "label": "Base URL", + "advanced": true, + "placeholder": "https://api.wandb.ai" + } + }, + "sampleRate": { + "type": "number", + "description": "Trace sampling rate (0.0 to 1.0)", + "default": 1.0, + "minimum": 0, + "maximum": 1, + "uiHints": { + "label": "Sample Rate", + "advanced": true, + "help": "Set to less than 1.0 to sample traces" + } + }, + "debug": { + "type": "boolean", + "description": "Enable verbose debug logging", + "default": false, + "uiHints": { + "label": "Debug Mode", + "advanced": true + } + } + }, + "required": ["apiKey", "entity"] + } +} diff --git a/extensions/weave/index.ts b/extensions/weave/index.ts new file mode 100644 index 000000000..166caffd9 --- /dev/null +++ b/extensions/weave/index.ts @@ -0,0 +1,80 @@ +/** + * W&B Weave Plugin for Clawdbot + * + * Provides LLM observability and tracing capabilities + * through integration with Weights & Biases Weave. + * + * Features: + * - Automatic tracing of agent runs and tool calls + * - Full system prompt capture via llm_request hook + * - Multi-turn conversation tracing + * - Session-level observability + * + * @see https://weave-docs.wandb.ai/ + */ + +import type { ClawdbotPluginApi } from 'clawdbot/plugin-sdk'; +import type { WeavePluginConfig } from './src/types.js'; +import { normalizeConfig } from './src/config.js'; +import { initializeWeaveClient } from './src/client.js'; +import { registerHooks } from './src/hooks.js'; + +/** + * Clawdbot plugin definition + */ +const pluginDefinition = { + id: 'weave', + name: 'W&B Weave', + description: 'Weights & Biases Weave integration for LLM observability and tracing', + version: '0.1.0', + + // Synchronous register - Clawdbot ignores async registration! + register(api: ClawdbotPluginApi) { + const logger = api.logger; + + // Get and validate plugin config + const rawConfig = api.pluginConfig as Partial | undefined; + + if (!rawConfig?.apiKey) { + logger.warn( + '[weave] Plugin disabled: No API key configured. Set plugins.entries.weave.config.apiKey in your config.' + ); + return; + } + + let config: WeavePluginConfig; + try { + config = normalizeConfig(rawConfig); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + logger.error(`[weave] Configuration error: ${msg}`); + return; + } + + // Initialize Weave client synchronously (starts the async init in background) + // The client will be ready by the time hooks fire + initializeWeaveClient(config) + .then(() => { + logger.info(`[weave] Initialized for ${config.entity}/${config.project}`); + }) + .catch((error) => { + const msg = error instanceof Error ? error.message : String(error); + logger.error(`[weave] Failed to initialize: ${msg}`); + }); + + // Register observability hooks (they'll wait for client to be ready) + registerHooks(api, config); + logger.info('[weave] Observability hooks registered'); + + // Log feature status + const features = []; + if (config.autoTrace) features.push('auto-tracing'); + if (config.traceToolCalls) features.push('tool-tracing'); + if (config.traceSessions) features.push('session-tracking'); + if (features.length > 0) { + logger.info(`[weave] Features enabled: ${features.join(', ')}`); + } + }, +}; + +export default pluginDefinition; diff --git a/extensions/weave/package.json b/extensions/weave/package.json new file mode 100644 index 000000000..fdc280c46 --- /dev/null +++ b/extensions/weave/package.json @@ -0,0 +1,12 @@ +{ + "name": "@clawdbot/weave", + "version": "2026.1.25", + "type": "module", + "description": "W&B Weave integration for Clawdbot - LLM observability and tracing", + "clawdbot": { + "extensions": ["./index.ts"] + }, + "dependencies": { + "weave": "^0.11.0" + } +} diff --git a/extensions/weave/src/clawdbot.d.ts b/extensions/weave/src/clawdbot.d.ts new file mode 100644 index 000000000..27dcd9f89 --- /dev/null +++ b/extensions/weave/src/clawdbot.d.ts @@ -0,0 +1,93 @@ +/** + * Type declarations for clawdbot/plugin-sdk + * + * These are minimal declarations to satisfy TypeScript. + * The actual types come from the clawdbot package at runtime. + */ + +declare module 'clawdbot/plugin-sdk' { + import type { Static, TSchema } from '@sinclair/typebox'; + + export interface PluginLogger { + debug?: (message: string) => void; + info: (message: string) => void; + warn: (message: string) => void; + error: (message: string) => void; + } + + export interface ClawdbotConfig { + [key: string]: unknown; + } + + export interface PluginRuntime { + tools: { + createMemorySearchTool: (opts: unknown) => unknown; + }; + [key: string]: unknown; + } + + export interface ClawdbotPluginToolContext { + config?: ClawdbotConfig; + workspaceDir?: string; + agentDir?: string; + agentId?: string; + sessionKey?: string; + messageChannel?: string; + agentAccountId?: string; + sandboxed?: boolean; + } + + export interface ToolResult { + content: Array<{ type: string; text: string }>; + details?: Record; + } + + export interface AgentTool { + name: string; + label?: string; + description: string; + parameters: TParams; + execute: ( + toolCallId: string, + params: Static, + ctx?: ClawdbotPluginToolContext + ) => Promise | ToolResult; + } + + export interface ClawdbotPluginApi { + id: string; + name: string; + version?: string; + description?: string; + source: string; + config: ClawdbotConfig; + pluginConfig?: Record; + runtime: PluginRuntime; + logger: PluginLogger; + + registerTool: (tool: AgentTool | unknown, opts?: { name?: string; names?: string[] }) => void; + + registerHook: ( + events: string | string[], + handler: (event: unknown, ctx: unknown) => unknown, + opts?: unknown + ) => void; + + registerHttpHandler: (handler: unknown) => void; + registerChannel: (registration: unknown) => void; + registerGatewayMethod: (method: string, handler: unknown) => void; + registerCli: (registrar: unknown, opts?: unknown) => void; + registerService: (service: unknown) => void; + registerProvider: (provider: unknown) => void; + registerCommand: (command: unknown) => void; + resolvePath: (input: string) => string; + + on: ( + hookName: K, + handler: (event: unknown, ctx: unknown) => unknown, + opts?: { priority?: number } + ) => void; + } + + export function emptyPluginConfigSchema(): unknown; +} diff --git a/extensions/weave/src/client.ts b/extensions/weave/src/client.ts new file mode 100644 index 000000000..66c84637d --- /dev/null +++ b/extensions/weave/src/client.ts @@ -0,0 +1,396 @@ +/** + * W&B Weave Client Wrapper + * + * Provides a centralized interface for Weave SDK operations + */ + +import { init, op, Dataset } from 'weave'; +import type { + WeavePluginConfig, + TraceContext, + WeaveSpan, + WeaveLogEntry, + WeaveFeedback, + WeaveQuery, + WeaveDatasetConfig, +} from './types.js'; +import { getProjectPath } from './config.js'; + +/** + * Debug logging helper - only logs when debug is enabled + */ +function debugLog(debug: boolean | undefined, ...args: unknown[]): void { + if (debug) { + console.log('[weave]', ...args); + } +} + +/** + * Ensure project exists with RESTRICTED visibility. + * - If project doesn't exist: create it with RESTRICTED visibility + * - If project exists: don't touch visibility (respect user's settings) + */ +async function ensureProjectExists(config: WeavePluginConfig): Promise { + const { entity, project, apiKey, baseUrl, debug } = config; + debugLog(debug, `ensureProjectExists called for ${entity}/${project}`); + if (!apiKey || !entity || !project) { + debugLog(debug, 'ensureProjectExists: missing config, skipping'); + return; + } + + const encoded = Buffer.from(`api:${apiKey}`).toString('base64'); + const headers = { + 'Authorization': `Basic ${encoded}`, + 'Content-Type': 'application/json', + }; + + // Use custom base URL for self-hosted instances, default to W&B cloud + const graphqlUrl = baseUrl + ? `${baseUrl.replace(/\/$/, '')}/graphql` + : 'https://api.wandb.ai/graphql'; + + try { + // Check if project exists + debugLog(debug, `Checking if project ${entity}/${project} exists...`); + const checkResponse = await fetch(graphqlUrl, { + method: 'POST', + headers, + body: JSON.stringify({ + query: `{ project(name: "${project}", entityName: "${entity}") { name } }`, + }), + }); + + if (!checkResponse.ok) { + debugLog(debug, `Project check failed: ${checkResponse.status} ${checkResponse.statusText}`); + return; + } + + const checkResult = await checkResponse.json() as { data?: { project?: { name?: string } | null } }; + debugLog(debug, `Project check result:`, JSON.stringify(checkResult)); + + // Project already exists - don't touch it + if (checkResult?.data?.project?.name) { + debugLog(debug, `Project ${entity}/${project} already exists, not touching visibility`); + return; + } + + debugLog(debug, `Project ${entity}/${project} does not exist, creating with RESTRICTED visibility...`); + + // Project doesn't exist - create it with RESTRICTED visibility + const createResponse = await fetch(graphqlUrl, { + method: 'POST', + headers, + body: JSON.stringify({ + query: `mutation { upsertModel(input: {entityName: "${entity}", name: "${project}", access: "RESTRICTED", framework: "weave"}) { model { name access } } }`, + }), + }); + + if (createResponse.ok) { + const result = await createResponse.json() as { data?: { upsertModel?: { model?: { name?: string; access?: string } } } }; + debugLog(debug, `Create response:`, JSON.stringify(result)); + const created = result?.data?.upsertModel?.model; + if (created?.name) { + // Always log successful project creation (important info) + console.log(`[weave] Created project ${entity}/${project} with RESTRICTED visibility`); + } else { + debugLog(debug, `Create response OK but no model in result`); + } + } else { + const errorText = await createResponse.text(); + // Always log failures (important for troubleshooting) + console.warn(`[weave] Failed to create project: ${createResponse.status} ${createResponse.statusText}: ${errorText}`); + } + } catch (error) { + // Don't fail initialization - Weave SDK will create the project if needed + console.warn(`[weave] Could not pre-create project: ${error}`); + } +} + +/** + * Active trace contexts by session key + */ +const activeTraces = new Map(); + +/** + * Weave client singleton + */ +let weaveClient: WeaveClient | null = null; + +/** + * SDK client from Weave init() - used for flushing + */ +let sdkClient: { waitForBatchProcessing(): Promise } | null = null; + +/** + * WeaveClient wraps the Weave SDK for Clawdbot integration + */ +export class WeaveClient { + private config: WeavePluginConfig; + private initialized = false; + + constructor(config: WeavePluginConfig) { + this.config = config; + } + + /** + * Initialize the Weave SDK + */ + async initialize(): Promise { + const { debug } = this.config; + debugLog(debug, 'WeaveClient.initialize() called'); + if (this.initialized) { + debugLog(debug, 'Already initialized, skipping'); + return; + } + + // Set API key in environment for Weave SDK + if (this.config.apiKey) { + process.env.WANDB_API_KEY = this.config.apiKey; + debugLog(debug, 'API key set in environment'); + } + + // Set base URL for self-hosted instances + if (this.config.baseUrl) { + process.env.WANDB_BASE_URL = this.config.baseUrl; + debugLog(debug, `Base URL set to: ${this.config.baseUrl}`); + } + + // Pre-create project with RESTRICTED visibility (if it doesn't exist) + // This must happen BEFORE init() to prevent Weave from auto-creating with PRIVATE + debugLog(debug, 'Calling ensureProjectExists...'); + await ensureProjectExists(this.config); + debugLog(debug, 'ensureProjectExists completed'); + + // Initialize Weave with project - init() returns the SDK client + const client = await init(getProjectPath(this.config)); + sdkClient = client; + + this.initialized = true; + } + + /** + * Check if client is initialized + */ + isInitialized(): boolean { + return this.initialized; + } + + /** + * Start a new trace for an agent run + */ + startTrace(sessionKey: string, metadata: Record = {}): TraceContext { + const traceId = `trace_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`; + + const context: TraceContext = { + traceId, + sessionKey, + rootSpan: null, + activeSpan: null, + startTime: Date.now(), + metadata, + }; + + activeTraces.set(sessionKey, context); + return context; + } + + /** + * Get the active trace context for a session + */ + getTrace(sessionKey: string): TraceContext | undefined { + return activeTraces.get(sessionKey); + } + + /** + * Start a span within a trace + */ + startSpan( + sessionKey: string, + name: string, + inputs: Record = {}, + attributes: Record = {} + ): WeaveSpan { + const trace = activeTraces.get(sessionKey); + const parentSpan = trace?.activeSpan; + + const span: WeaveSpan = { + id: `span_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`, + name, + parentId: parentSpan?.id ?? null, + startTime: Date.now(), + endTime: null, + inputs, + outputs: null, + attributes, + status: 'running', + }; + + if (trace) { + if (!trace.rootSpan) { + trace.rootSpan = span; + } + trace.activeSpan = span; + } + + return span; + } + + /** + * End a span with outputs + */ + endSpan( + span: WeaveSpan, + outputs: Record = {}, + success = true, + error?: string + ): void { + span.endTime = Date.now(); + span.outputs = outputs; + span.status = success ? 'success' : 'error'; + if (error) { + span.error = error; + } + } + + /** + * End the trace for a session + */ + async endTrace(sessionKey: string): Promise { + console.log(`[weave] endTrace called for sessionKey: ${sessionKey}`); + const trace = activeTraces.get(sessionKey); + if (!trace) { + console.log(`[weave] No trace found for sessionKey: ${sessionKey}`); + return; + } + + console.log(`[weave] Found trace, logging to Weave...`); + // Log the complete trace to Weave + await this.logTrace(trace); + + activeTraces.delete(sessionKey); + console.log(`[weave] Trace deleted from activeTraces`); + } + + /** + * Log a complete trace to Weave + */ + private async logTrace(trace: TraceContext): Promise { + console.log(`[weave] logTrace called, rootSpan: ${trace.rootSpan?.name}`); + if (!trace.rootSpan) { + console.log(`[weave] No rootSpan, skipping`); + return; + } + + // Use Weave's op decorator pattern for logging + const traceFn = op( + async (input: Record) => { + return trace.rootSpan?.outputs ?? {}; + }, + { + name: trace.rootSpan.name, + } + ); + + try { + console.log(`[weave] Calling traceFn with inputs:`, JSON.stringify(trace.rootSpan.inputs).slice(0, 200)); + console.log(`[weave] Outputs to log:`, JSON.stringify(trace.rootSpan.outputs).slice(0, 300)); + await traceFn(trace.rootSpan.inputs); + console.log(`[weave] traceFn completed, flushing...`); + // Flush the trace to ensure it's sent to W&B + await this.flush(); + console.log(`[weave] Flush completed`); + } catch (error) { + console.error('[weave] Failed to log trace:', error); + } + } + + /** + * Flush any pending traces to W&B + */ + async flush(): Promise { + if (sdkClient && typeof sdkClient.waitForBatchProcessing === 'function') { + await sdkClient.waitForBatchProcessing(); + } + } + + /** + * Log custom data to Weave + */ + async log(entry: WeaveLogEntry): Promise { + const logFn = op(async (data: unknown) => data, { name: entry.name }); + + try { + await logFn(entry.value); + } catch (error) { + console.error('[weave] Failed to log entry:', error); + } + } + + /** + * Add feedback to a trace/call + */ + async addFeedback(feedback: WeaveFeedback): Promise { + // Feedback is typically added through the Weave UI or API + // This is a placeholder for future API integration + console.log('[weave] Feedback:', feedback); + } + + /** + * Query traces from Weave + */ + async query(params: WeaveQuery): Promise { + // Query functionality requires Weave API calls + // This is a placeholder for future API integration + console.log('[weave] Query:', params); + return []; + } + + /** + * Create or update a dataset + */ + async createDataset(config: WeaveDatasetConfig): Promise { + const dataset = new Dataset({ + name: config.name, + description: config.description, + rows: config.rows, + }); + + // Save the dataset to Weave + try { + await dataset.save(); + console.log(`[weave] Created dataset: ${config.name} with ${config.rows.length} rows`); + } catch (error) { + console.error(`[weave] Failed to create dataset: ${error}`); + throw error; + } + } + + /** + * Get configuration + */ + getConfig(): WeavePluginConfig { + return this.config; + } +} + +/** + * Get or create the Weave client singleton + */ +export function getWeaveClient(config?: WeavePluginConfig): WeaveClient { + if (!weaveClient && config) { + weaveClient = new WeaveClient(config); + } + if (!weaveClient) { + throw new Error('Weave client not initialized. Provide config on first call.'); + } + return weaveClient; +} + +/** + * Initialize the Weave client + */ +export async function initializeWeaveClient(config: WeavePluginConfig): Promise { + const client = getWeaveClient(config); + await client.initialize(); + return client; +} diff --git a/extensions/weave/src/config.ts b/extensions/weave/src/config.ts new file mode 100644 index 000000000..df800caa8 --- /dev/null +++ b/extensions/weave/src/config.ts @@ -0,0 +1,67 @@ +/** + * W&B Weave Plugin Configuration + */ + +import type { WeavePluginConfig } from './types.js'; + +/** + * Default configuration values + */ +export const DEFAULT_CONFIG: Partial = { + project: 'clawdbot', + autoTrace: true, + traceToolCalls: true, + traceSessions: true, + sampleRate: 1.0, + debug: false, +}; + +/** + * Validates and normalizes plugin configuration + */ +export function normalizeConfig(config: Partial): WeavePluginConfig { + const normalized: WeavePluginConfig = { + apiKey: config.apiKey ?? '', + entity: config.entity ?? '', + project: config.project ?? DEFAULT_CONFIG.project!, + autoTrace: config.autoTrace ?? DEFAULT_CONFIG.autoTrace, + traceToolCalls: config.traceToolCalls ?? DEFAULT_CONFIG.traceToolCalls, + traceSessions: config.traceSessions ?? DEFAULT_CONFIG.traceSessions, + baseUrl: config.baseUrl, + sampleRate: config.sampleRate ?? DEFAULT_CONFIG.sampleRate, + debug: config.debug ?? DEFAULT_CONFIG.debug, + }; + + // Validate required fields + if (!normalized.apiKey) { + throw new Error('Weave plugin requires apiKey configuration'); + } + if (!normalized.entity) { + throw new Error('Weave plugin requires entity configuration'); + } + + // Validate sample rate + if (normalized.sampleRate !== undefined) { + if (normalized.sampleRate < 0 || normalized.sampleRate > 1) { + throw new Error('Weave plugin sampleRate must be between 0 and 1'); + } + } + + return normalized; +} + +/** + * Determines if a trace should be sampled based on sample rate + */ +export function shouldSample(sampleRate: number): boolean { + if (sampleRate >= 1.0) return true; + if (sampleRate <= 0) return false; + return Math.random() < sampleRate; +} + +/** + * Gets the full project path (entity/project) + */ +export function getProjectPath(config: WeavePluginConfig): string { + return `${config.entity}/${config.project}`; +} diff --git a/extensions/weave/src/hooks.ts b/extensions/weave/src/hooks.ts new file mode 100644 index 000000000..434da6389 --- /dev/null +++ b/extensions/weave/src/hooks.ts @@ -0,0 +1,625 @@ +/** + * W&B Weave Plugin Hooks + * + * Lifecycle hooks for automatic observability and tracing + */ + +import type { ClawdbotPluginApi } from 'clawdbot/plugin-sdk'; +import type { WeavePluginConfig } from './types.js'; +import { getWeaveClient } from './client.js'; +import { shouldSample } from './config.js'; + +/** + * Hook event and context types (simplified for our needs) + */ +interface SessionStartEvent { + sessionId: string; + resumedFrom?: string; +} + +interface SessionEndEvent { + sessionId: string; + messageCount: number; + durationMs?: number; +} + +interface SessionContext { + agentId?: string; + sessionId: string; +} + +interface AgentStartEvent { + prompt: string; + messages?: unknown[]; +} + +interface AgentContext { + agentId?: string; + sessionKey?: string; + workspaceDir?: string; + messageProvider?: string; +} + +interface AgentEndEvent { + messages: unknown[]; + success: boolean; + error?: string; + durationMs?: number; +} + +interface ToolCallEvent { + toolName: string; + params: Record; +} + +interface ToolCallEndEvent { + toolName: string; + params: Record; + result?: unknown; + error?: string; + durationMs?: number; +} + +interface ToolContext { + agentId?: string; + sessionKey?: string; + toolName: string; +} + +interface MessageReceivedEvent { + from: string; + content: string; + timestamp?: number; + metadata?: Record; +} + +interface MessageSentEvent { + to: string; + content: string; + success: boolean; + error?: string; +} + +interface LlmRequestEvent { + payload: { + system?: unknown; + messages?: unknown[]; + model?: string; + max_tokens?: number; + temperature?: number; + tools?: unknown[]; + [key: string]: unknown; + }; + payloadDigest?: string; + timestamp: string; +} + +interface LlmRequestContext { + agentId?: string; + sessionKey?: string; + sessionId?: string; + runId?: string; + provider?: string; + modelId?: string; + modelApi?: string | null; + workspaceDir?: string; +} + +/** + * Track active spans by session key for tool call nesting + */ +const toolSpans = new Map(); + +/** + * Register all observability hooks + */ +export function registerHooks(api: ClawdbotPluginApi, config: WeavePluginConfig): void { + const logger = api.logger; + + // Session tracking hooks + if (config.traceSessions) { + api.on('session_start', async (rawEvent, rawCtx) => { + const event = rawEvent as SessionStartEvent; + const ctx = rawCtx as SessionContext; + + if (!shouldSample(config.sampleRate ?? 1.0)) return; + + try { + const client = getWeaveClient(); + client.startTrace(ctx.sessionId, { + type: 'session', + resumedFrom: event.resumedFrom, + }); + logger.debug?.(`[weave] Session started: ${ctx.sessionId}`); + } catch (error) { + logger.error(`[weave] Failed to start session trace: ${error}`); + } + }); + + api.on('session_end', async (rawEvent, rawCtx) => { + const event = rawEvent as SessionEndEvent; + const ctx = rawCtx as SessionContext; + + try { + const client = getWeaveClient(); + const trace = client.getTrace(ctx.sessionId); + if (trace) { + trace.metadata.messageCount = event.messageCount; + trace.metadata.durationMs = event.durationMs; + await client.endTrace(ctx.sessionId); + logger.debug?.(`[weave] Session ended: ${ctx.sessionId}`); + } + } catch (error) { + logger.error(`[weave] Failed to end session trace: ${error}`); + } + }); + } + + // Agent run tracing hooks + if (config.autoTrace) { + console.log('[weave] Registering before_agent_start hook'); + api.on('before_agent_start', async (rawEvent, rawCtx) => { + console.log('[weave] HOOK FIRED: before_agent_start'); + const event = rawEvent as AgentStartEvent; + const ctx = rawCtx as AgentContext; + // Log ALL available data to discover system prompt + console.log('[weave] before_agent_start event keys:', Object.keys(rawEvent as object)); + console.log('[weave] before_agent_start context keys:', Object.keys(rawCtx as object)); + console.log('[weave] Full event (truncated):', JSON.stringify(rawEvent).slice(0, 1500)); + console.log('[weave] Full context (truncated):', JSON.stringify(rawCtx).slice(0, 1500)); + + if (!shouldSample(config.sampleRate ?? 1.0)) return; + + try { + const client = getWeaveClient(); + const sessionKey = ctx.sessionKey ?? 'unknown'; + + // Start or get existing trace + let trace = client.getTrace(sessionKey); + if (!trace) { + trace = client.startTrace(sessionKey, { + agentId: ctx.agentId, + provider: ctx.messageProvider, + }); + } + + // Start agent run span + const span = client.startSpan( + sessionKey, + 'agent_run', + { + prompt: event.prompt, + messageCount: event.messages?.length ?? 0, + }, + { + agentId: ctx.agentId, + workspaceDir: ctx.workspaceDir, + } + ); + + console.log(`[weave] Agent run started: ${span.id}`); + logger.debug?.(`[weave] Agent run started: ${span.id}`); + } catch (error) { + console.error(`[weave] Failed to start agent trace: ${error}`); + logger.error(`[weave] Failed to start agent trace: ${error}`); + } + + // Return nothing - we don't modify the prompt + return; + }); + + console.log('[weave] Registering agent_end hook'); + api.on('agent_end', async (rawEvent, rawCtx) => { + console.log('[weave] HOOK FIRED: agent_end'); + const event = rawEvent as AgentEndEvent; + const ctx = rawCtx as AgentContext; + + try { + const client = getWeaveClient(); + const sessionKey = ctx.sessionKey ?? 'unknown'; + const trace = client.getTrace(sessionKey); + + if (trace?.activeSpan) { + // Debug: log raw event to understand ALL available data + console.log(`[weave] agent_end event keys:`, Object.keys(event)); + console.log(`[weave] agent_end full event (truncated):`, JSON.stringify(event).slice(0, 1000)); + console.log(`[weave] messages type:`, typeof event.messages, Array.isArray(event.messages) ? `array[${event.messages.length}]` : ''); + + // Log ALL unique roles in messages + if (event.messages && Array.isArray(event.messages)) { + const roles = new Set(); + for (const msg of event.messages as Array<{ role?: string }>) { + if (msg.role) roles.add(msg.role); + } + console.log(`[weave] Message roles found:`, Array.from(roles).join(', ')); + + // Log first message of each role type for debugging + const seenRoles = new Set(); + for (const msg of event.messages as Array<{ role?: string }>) { + if (msg.role && !seenRoles.has(msg.role)) { + seenRoles.add(msg.role); + console.log(`[weave] Sample ${msg.role} message:`, JSON.stringify(msg).slice(0, 400)); + } + } + } + + // Type definitions for message parsing + type ToolUseBlock = { type: 'tool_use'; id?: string; name: string; input: Record }; + type ToolResultBlock = { type: 'tool_result'; tool_use_id?: string; content?: string }; + type ContentBlock = { type?: string; text?: string; thinking?: string } | ToolUseBlock | ToolResultBlock; + type Message = { role?: string; content?: string | ContentBlock[] }; + const messages = event.messages as Message[] | undefined; + + // Helper to extract text from content (string or array of blocks) + const extractText = (content: string | ContentBlock[] | undefined): string => { + if (!content) return ''; + if (typeof content === 'string') return content; + return content + .filter((block): block is ContentBlock & { text: string } => block.type === 'text' && typeof block.text === 'string') + .map(block => block.text) + .join(''); + }; + + // Extended message type for Clawdbot-specific roles + type ExtendedMessage = Message & { + summary?: string; + toolCallId?: string; + toolName?: string; + }; + const extMessages = messages as ExtendedMessage[] | undefined; + + // Extract the COMPLETE conversation by role + const compactionSummaries: string[] = []; + const userMessages: string[] = []; + const assistantMessages: string[] = []; + const toolCalls: Array<{ name: string; input: Record; result?: string }> = []; + const toolResults: Array<{ toolName: string; toolCallId: string; result: string }> = []; + + if (extMessages) { + for (const msg of extMessages) { + // Capture compaction summaries (session context) + if (msg.role === 'compactionSummary' && msg.summary) { + compactionSummaries.push(msg.summary); + } + + // Capture all user messages + if (msg.role === 'user') { + const text = extractText(msg.content); + if (text) userMessages.push(text); + } + + // Capture all assistant messages + if (msg.role === 'assistant') { + const text = extractText(msg.content); + if (text) assistantMessages.push(text); + } + + // Capture tool results (Clawdbot uses role: "toolResult") + if (msg.role === 'toolResult' && msg.toolName && msg.toolCallId) { + const resultText = extractText(msg.content); + toolResults.push({ + toolName: msg.toolName, + toolCallId: msg.toolCallId, + result: resultText.slice(0, 2000), + }); + } + + // Extract tool calls from content blocks (tool_use) + if (Array.isArray(msg.content)) { + for (const block of msg.content) { + if (block.type === 'tool_use' && 'name' in block) { + toolCalls.push({ + name: block.name, + input: block.input, + }); + } + } + } + } + + // Match tool results to tool calls by toolCallId + for (const result of toolResults) { + const matchingCall = toolCalls.find(tc => !tc.result); + if (matchingCall) { + matchingCall.result = result.result; + } + } + } + + console.log(`[weave] Extracted - Summaries: ${compactionSummaries.length}, User: ${userMessages.length}, Assistant: ${assistantMessages.length}, ToolCalls: ${toolCalls.length}, ToolResults: ${toolResults.length}`); + if (compactionSummaries.length > 0) { + console.log(`[weave] Compaction summary length: ${compactionSummaries.join('').length} chars`); + } + if (toolCalls.length > 0) { + console.log(`[weave] Tool calls:`, toolCalls.map(tc => tc.name).join(', ')); + } + if (toolResults.length > 0) { + console.log(`[weave] Tool results:`, toolResults.map(tr => tr.toolName).join(', ')); + } + + // Get the final response (last assistant message) + const response = assistantMessages[assistantMessages.length - 1] ?? ''; + console.log(`[weave] Final response length: ${response.length} chars`); + + // Include system prompt and all LLM requests from llm_request hook + const systemPrompt = trace.metadata.systemPrompt as string | undefined; + const systemPromptChanges = trace.metadata.systemPromptChanges as number | undefined; + const llmRequest = trace.metadata.llmRequest as Record | undefined; + const llmRequests = trace.metadata.llmRequests as unknown[] | undefined; + const llmTools = trace.metadata.llmTools as unknown[] | undefined; + + console.log(`[weave] LLM Requests captured: ${llmRequests?.length ?? 0}, System prompt changes: ${systemPromptChanges ?? 0}`); + + client.endSpan( + trace.activeSpan, + { + // FULL SYSTEM PROMPT - captured from llm_request hook (latest version) + systemPrompt: systemPrompt ?? undefined, + systemPromptChanges: systemPromptChanges ?? 0, + // ALL LLM Requests (for multi-turn conversations) + llmRequests: llmRequests ?? undefined, + llmRequestCount: llmRequests?.length ?? 0, + // First LLM Request metadata (backward compatible) + llmRequest: llmRequest ?? undefined, + // Available tools/functions + llmTools: llmTools ?? undefined, + // Session context (compaction summaries contain read files, etc.) + sessionContext: compactionSummaries.length > 0 ? compactionSummaries.join('\n\n---\n\n') : undefined, + // Full conversation + userMessages: userMessages.length > 0 ? userMessages : undefined, + assistantMessages: assistantMessages.length > 0 ? assistantMessages : undefined, + // Final response for quick access + response, + // Tool calls with results + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + toolResults: toolResults.length > 0 ? toolResults : undefined, + toolCallCount: toolCalls.length + toolResults.length, + // Metadata + success: event.success, + messageCount: event.messages?.length ?? 0, + durationMs: event.durationMs, + }, + event.success, + event.error + ); + + trace.metadata.durationMs = event.durationMs; + trace.metadata.success = event.success; + if (event.error) { + trace.metadata.error = event.error; + } + + console.log(`[weave] Agent run ended: ${trace.activeSpan.id}`); + logger.debug?.(`[weave] Agent run ended: ${trace.activeSpan.id}`); + } + + // Always end trace after agent run to ensure it's logged to Weave + // Even with session tracking, we want per-run traces + console.log(`[weave] Ending trace for sessionKey: ${sessionKey}`); + await client.endTrace(sessionKey); + console.log(`[weave] Trace ended and logged to Weave`); + } catch (error) { + console.error(`[weave] Failed to end agent trace: ${error}`); + logger.error(`[weave] Failed to end agent trace: ${error}`); + } + }); + + // LLM Request hook - captures the FULL payload sent to the LLM API + // NOTE: This hook requires Clawdbot core support! + try { + console.log('[weave] Registering llm_request hook'); + api.on('llm_request', async (rawEvent, rawCtx) => { + console.log('[weave] HOOK FIRED: llm_request'); + const event = rawEvent as LlmRequestEvent; + const ctx = rawCtx as LlmRequestContext; + + try { + const client = getWeaveClient(); + const sessionKey = ctx.sessionKey ?? 'unknown'; + const trace = client.getTrace(sessionKey); + + // Extract system prompt from payload + const systemPrompt = event.payload.system; + const systemPromptStr = typeof systemPrompt === 'string' + ? systemPrompt + : JSON.stringify(systemPrompt); + + // Build request record with FULL system prompt for complete observability + const requestRecord = { + timestamp: event.timestamp, + payloadDigest: event.payloadDigest, + model: event.payload.model, + maxTokens: event.payload.max_tokens, + temperature: event.payload.temperature, + // Include FULL system prompt for this specific request + systemPrompt: systemPromptStr ?? undefined, + systemPromptLength: systemPromptStr?.length ?? 0, + messagesCount: event.payload.messages?.length ?? 0, + toolsCount: event.payload.tools?.length ?? 0, + // Include messages for this specific request + messages: event.payload.messages, + }; + + if (trace) { + // Initialize arrays if needed + if (!trace.metadata.llmRequests) { + trace.metadata.llmRequests = []; + } + + // Add this request to the array (complete multi-request tracing) + (trace.metadata.llmRequests as unknown[]).push(requestRecord); + + const requestIndex = (trace.metadata.llmRequests as unknown[]).length; + console.log(`[weave] LLM Request #${requestIndex} captured - System prompt: ${systemPromptStr?.length ?? 0} chars, Messages: ${event.payload.messages?.length ?? 0}`); + + // Track system prompt changes - store latest version (most relevant for final response) + if (systemPromptStr) { + const currentLength = (trace.metadata.systemPrompt as string | undefined)?.length ?? 0; + const newLength = systemPromptStr.length; + + if (!trace.metadata.systemPrompt) { + // First system prompt + trace.metadata.systemPrompt = systemPromptStr; + trace.metadata.systemPromptChanges = 0; + console.log(`[weave] System prompt stored (first request, ${newLength} chars)`); + } else if (newLength !== currentLength) { + // System prompt changed - update to latest version + trace.metadata.systemPrompt = systemPromptStr; + trace.metadata.systemPromptChanges = ((trace.metadata.systemPromptChanges as number) ?? 0) + 1; + console.log(`[weave] System prompt CHANGED (${currentLength} -> ${newLength} chars, change #${trace.metadata.systemPromptChanges})`); + } + } + + // Store tools only on first request (they don't change either) + if (!trace.metadata.llmTools && event.payload.tools) { + trace.metadata.llmTools = event.payload.tools; + } + + // Keep backward-compatible single llmRequest field (points to first request) + if (!trace.metadata.llmRequest) { + trace.metadata.llmRequest = requestRecord; + } + + logger.debug?.(`[weave] LLM request #${requestIndex} captured for session: ${sessionKey}`); + } else { + // No active trace - log as orphaned request (happens after agent_end) + console.log(`[weave] Orphaned LLM request (no active trace) - sessionKey: ${sessionKey}, messages: ${event.payload.messages?.length ?? 0}`); + + // Log orphaned requests to Weave as standalone traces + try { + const { op } = await import('weave'); + const orphanedRequestFn = op( + async (input: { sessionKey: string; request: typeof requestRecord; systemPromptLength: number }) => { + return { + type: 'orphaned_llm_request', + sessionKey: input.sessionKey, + messagesCount: input.request.messagesCount, + model: input.request.model, + }; + }, + { name: 'orphaned_llm_request' } + ); + await orphanedRequestFn({ + sessionKey, + request: requestRecord, + systemPromptLength: systemPromptStr?.length ?? 0, + }); + console.log(`[weave] Orphaned request logged as standalone trace`); + } catch (orphanError) { + console.error(`[weave] Failed to log orphaned request: ${orphanError}`); + } + } + } catch (error) { + console.error(`[weave] Failed to capture llm_request: ${error}`); + logger.error(`[weave] Failed to capture llm_request: ${error}`); + } + }); + } catch (error) { + // llm_request hook not available in this Clawdbot version - graceful degradation + console.log('[weave] llm_request hook not available (requires Clawdbot core update) - system prompt capture disabled'); + logger.info('[weave] llm_request hook not available - running without system prompt capture'); + } + } + + // Tool call tracing hooks + if (config.traceToolCalls) { + console.log('[weave] Registering before_tool_call hook'); + api.on('before_tool_call', async (rawEvent, rawCtx) => { + console.log('[weave] HOOK FIRED: before_tool_call'); + const event = rawEvent as ToolCallEvent; + const ctx = rawCtx as ToolContext; + console.log(`[weave] Tool call: ${event.toolName}`); + + try { + const client = getWeaveClient(); + const sessionKey = ctx.sessionKey ?? 'unknown'; + const trace = client.getTrace(sessionKey); + + if (trace) { + const span = client.startSpan( + sessionKey, + `tool:${event.toolName}`, + { params: event.params }, + { toolName: event.toolName } + ); + + // Track span for this session + const spans = toolSpans.get(sessionKey) ?? []; + spans.push(span.id); + toolSpans.set(sessionKey, spans); + + console.log(`[weave] Tool span started: ${span.id}`); + logger.debug?.(`[weave] Tool call started: ${event.toolName}`); + } else { + console.log(`[weave] No trace found for tool call, sessionKey: ${sessionKey}`); + } + } catch (error) { + console.error(`[weave] Failed to trace tool call: ${error}`); + logger.error(`[weave] Failed to trace tool call: ${error}`); + } + + // Return nothing - we don't block or modify + return; + }); + + console.log('[weave] Registering after_tool_call hook'); + api.on('after_tool_call', async (rawEvent, rawCtx) => { + console.log('[weave] HOOK FIRED: after_tool_call'); + const event = rawEvent as ToolCallEndEvent; + const ctx = rawCtx as ToolContext; + console.log(`[weave] Tool call ended: ${event.toolName} (${event.durationMs}ms)`); + + try { + const client = getWeaveClient(); + const sessionKey = ctx.sessionKey ?? 'unknown'; + const trace = client.getTrace(sessionKey); + + if (trace?.activeSpan && trace.activeSpan.name === `tool:${event.toolName}`) { + client.endSpan( + trace.activeSpan, + { + result: event.result, + durationMs: event.durationMs, + }, + !event.error, + event.error + ); + + // Pop span from stack + const spans = toolSpans.get(sessionKey) ?? []; + spans.pop(); + toolSpans.set(sessionKey, spans); + + console.log(`[weave] Tool span ended: ${event.toolName}`); + logger.debug?.(`[weave] Tool call ended: ${event.toolName} (${event.durationMs}ms)`); + } else { + console.log(`[weave] No matching span for tool: ${event.toolName}`); + } + } catch (error) { + console.error(`[weave] Failed to end tool trace: ${error}`); + logger.error(`[weave] Failed to end tool trace: ${error}`); + } + }); + } + + // Message hooks for additional context + api.on('message_received', async (rawEvent, _rawCtx) => { + const event = rawEvent as MessageReceivedEvent; + + try { + logger.debug?.(`[weave] Message received from: ${event.from}`); + } catch (_error) { + // Silently fail - message logging is optional + } + }); + + api.on('message_sent', async (rawEvent, _rawCtx) => { + const event = rawEvent as MessageSentEvent; + + try { + logger.debug?.(`[weave] Message sent to: ${event.to}`); + } catch (_error) { + // Silently fail - message logging is optional + } + }); +} diff --git a/extensions/weave/src/types.ts b/extensions/weave/src/types.ts new file mode 100644 index 000000000..c6a657f55 --- /dev/null +++ b/extensions/weave/src/types.ts @@ -0,0 +1,222 @@ +/** + * W&B Weave Plugin Types + */ + +/** + * Plugin configuration schema + */ +export interface WeavePluginConfig { + /** W&B API key for authentication */ + apiKey: string; + /** W&B entity (username or team name) */ + entity: string; + /** Default W&B project name for traces */ + project: string; + /** Automatically trace all agent runs (default: true) */ + autoTrace?: boolean; + /** Log tool calls as child spans (default: true) */ + traceToolCalls?: boolean; + /** Track session lifecycle (default: true) */ + traceSessions?: boolean; + /** Custom W&B server URL */ + baseUrl?: string; + /** Trace sampling rate 0.0 to 1.0 (default: 1.0) */ + sampleRate?: number; + /** Enable debug logging (default: false) */ + debug?: boolean; +} + +/** + * Trace context for tracking spans across hooks + */ +export interface TraceContext { + /** Unique trace ID */ + traceId: string; + /** Session key (channel:user:account) */ + sessionKey: string; + /** Root span reference */ + rootSpan: WeaveSpan | null; + /** Current active span */ + activeSpan: WeaveSpan | null; + /** Start timestamp */ + startTime: number; + /** Metadata collected during trace */ + metadata: Record; +} + +/** + * Weave span representation + */ +export interface WeaveSpan { + /** Span ID */ + id: string; + /** Span name */ + name: string; + /** Parent span ID (null for root) */ + parentId: string | null; + /** Start timestamp */ + startTime: number; + /** End timestamp (null if still running) */ + endTime: number | null; + /** Span inputs */ + inputs: Record; + /** Span outputs */ + outputs: Record | null; + /** Span attributes/metadata */ + attributes: Record; + /** Span status */ + status: 'running' | 'success' | 'error'; + /** Error message if status is error */ + error?: string; +} + +/** + * Tool call data for tracing + */ +export interface ToolCallData { + /** Tool name */ + name: string; + /** Tool inputs */ + inputs: Record; + /** Tool outputs */ + outputs?: unknown; + /** Execution duration in ms */ + duration?: number; + /** Whether the call succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; +} + +/** + * Agent run data for tracing + */ +export interface AgentRunData { + /** Agent ID */ + agentId: string; + /** Session key */ + sessionKey: string; + /** Message channel */ + channel: string; + /** User message */ + userMessage: string; + /** Agent response */ + response?: string; + /** Model used */ + model?: string; + /** Token usage */ + tokenUsage?: { + inputTokens: number; + outputTokens: number; + totalTokens: number; + }; + /** Tool calls made during the run */ + toolCalls: ToolCallData[]; + /** Run duration in ms */ + duration?: number; + /** Whether the run succeeded */ + success: boolean; + /** Error message if failed */ + error?: string; +} + +/** + * Session data for tracing + */ +export interface SessionData { + /** Session key */ + sessionKey: string; + /** Channel */ + channel: string; + /** User ID */ + userId: string; + /** Account ID */ + accountId: string; + /** Session start time */ + startTime: number; + /** Session end time */ + endTime?: number; + /** Number of messages in session */ + messageCount: number; + /** Total token usage in session */ + totalTokens: number; +} + +/** + * Weave log entry for custom logging + */ +export interface WeaveLogEntry { + /** Log name/key */ + name: string; + /** Log value (metrics, data, etc.) */ + value: unknown; + /** Optional metadata */ + metadata?: Record; + /** Timestamp */ + timestamp?: number; +} + +/** + * Weave feedback entry + */ +export interface WeaveFeedback { + /** Trace/call ID to annotate */ + callId: string; + /** Feedback type */ + type: 'thumbs_up' | 'thumbs_down' | 'correction' | 'note' | 'custom'; + /** Feedback value */ + value: unknown; + /** Optional note */ + note?: string; +} + +/** + * Weave query parameters + */ +export interface WeaveQuery { + /** Project to query */ + project?: string; + /** Filter by trace name */ + traceName?: string; + /** Filter by time range (ISO timestamps) */ + timeRange?: { + start: string; + end: string; + }; + /** Filter by metadata */ + filters?: Record; + /** Maximum results */ + limit?: number; + /** Sort order */ + sortBy?: 'time' | 'duration' | 'tokens'; + /** Sort direction */ + sortOrder?: 'asc' | 'desc'; +} + +/** + * Weave evaluation configuration + */ +export interface WeaveEvalConfig { + /** Dataset name or ID */ + dataset: string; + /** Scorers to use */ + scorers: string[]; + /** Model/function to evaluate */ + model?: string; + /** Evaluation name */ + name?: string; + /** Number of trials per example */ + trials?: number; +} + +/** + * Weave dataset configuration + */ +export interface WeaveDatasetConfig { + /** Dataset name */ + name: string; + /** Dataset description */ + description?: string; + /** Dataset rows */ + rows: Record[]; +} diff --git a/extensions/weave/tsconfig.json b/extensions/weave/tsconfig.json new file mode 100644 index 000000000..4b56e9544 --- /dev/null +++ b/extensions/weave/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": ".", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "resolveJsonModule": true + }, + "include": ["index.ts", "src/**/*.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 223537e85..2c4752443 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -172,13 +172,6 @@ importers: zod: specifier: ^4.3.6 version: 4.3.6 - optionalDependencies: - '@napi-rs/canvas': - specifier: ^0.1.88 - version: 0.1.88 - node-llama-cpp: - specifier: 3.15.0 - version: 3.15.0(typescript@5.9.3) devDependencies: '@grammyjs/types': specifier: ^3.23.0 @@ -261,6 +254,13 @@ importers: wireit: specifier: ^0.14.12 version: 0.14.12 + optionalDependencies: + '@napi-rs/canvas': + specifier: ^0.1.88 + version: 0.1.88 + node-llama-cpp: + specifier: 3.15.0 + version: 3.15.0(typescript@5.9.3) extensions/bluebubbles: {} @@ -357,7 +357,7 @@ importers: extensions/memory-core: dependencies: clawdbot: - specifier: '>=2026.1.25' + specifier: workspace:* version: link:../.. extensions/memory-lancedb: @@ -455,6 +455,12 @@ importers: specifier: ^4.3.6 version: 4.3.6 + extensions/weave: + dependencies: + weave: + specifier: ^0.11.0 + version: 0.11.0(ws@8.19.0)(zod@3.25.76) + extensions/whatsapp: {} extensions/zalo: @@ -1316,6 +1322,7 @@ packages: '@lancedb/lancedb@0.23.0': resolution: {integrity: sha512-aYrIoEG24AC+wILCL57Ius/Y4yU+xFHDPKLvmjzzN4byAjzeIGF0TC86S5RBt4Ji+dxS7yIWV5Q/gE5/fybIFQ==} engines: {node: '>= 18'} + cpu: [x64, arm64] os: [darwin, linux, win32] peerDependencies: apache-arrow: '>=15.0.0 <=18.1.0' @@ -2724,9 +2731,15 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + '@types/node@10.17.60': resolution: {integrity: sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==} + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@20.19.30': resolution: {integrity: sha512-WJtwWJu7UdlvzEAUm484QNg5eAoq5QR08KDNx7g45Usrs2NtOPiX8ugDqmKdXkyL03rBqU5dYNYVQetEpBHq2g==} @@ -2932,6 +2945,10 @@ packages: resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -3188,6 +3205,9 @@ packages: resolution: {integrity: sha512-Wdy2Igu8OcBpI2pZePZ5oWjPC38tmDVx5WKUXKwlLYkA0ozo85sLsLvkBbBn/sZaSCMFOGZJ14fvW9t5/d7kdA==} engines: {node: '>=8'} + cjs-module-lexer@1.4.3: + resolution: {integrity: sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==} + cjs-module-lexer@2.2.0: resolution: {integrity: sha512-4bHTS2YuzUvtoLjdy+98ykbNB5jS0+07EvFNXerqZQJ89F7DI6ET7OQo/HJuW6K0aVsKA9hj9/RVb2kQVOrPDQ==} @@ -3203,6 +3223,10 @@ packages: engines: {node: '>=8.0.0', npm: '>=5.0.0'} hasBin: true + cli-progress@3.12.0: + resolution: {integrity: sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A==} + engines: {node: '>=4'} + cli-spinners@2.9.2: resolution: {integrity: sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==} engines: {node: '>=6'} @@ -3607,6 +3631,9 @@ packages: forever-agent@0.6.1: resolution: {integrity: sha512-j0KLYPhm6zeac4lz3oJ3o65qvgQCcPubiyotZrXqEaG4hNagNYO8qdlUrX5vwqv9ohqeT/Z3j6+yW067yWWdUw==} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + form-data@2.3.3: resolution: {integrity: sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==} engines: {node: '>= 0.12'} @@ -3615,6 +3642,10 @@ packages: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -3809,6 +3840,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.4.24: resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} engines: {node: '>=0.10.0'} @@ -3827,6 +3861,9 @@ packages: immediate@3.0.6: resolution: {integrity: sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==} + import-in-the-middle@1.15.0: + resolution: {integrity: sha512-bpQy+CrsRmYmoPMAE/0G33iwRqwW4ouqdRg8jgbH3aKuCtOc8lxgmYXg2dMM92CRiGP660EtBcymH/eVUpCSaA==} + import-in-the-middle@2.0.5: resolution: {integrity: sha512-0InH9/4oDCBRzWXhpOqusspLBrVfK1vPvbn9Wxl8DAQ8yyx5fWJRETICSwkiAMaYntjJAMBP1R4B6cQnEUYVEA==} @@ -3836,6 +3873,10 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ini@5.0.0: + resolution: {integrity: sha512-+N0ngpO3e7cRUWOJAS7qw0IZIVc6XPrW4MlFBdD066F2L4k1L6ker3hLqSq7iXxU5tgS4WGkIUElWn5vogAEnw==} + engines: {node: ^18.17.0 || >=20.5.0} + ipaddr.js@1.9.1: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} @@ -4494,6 +4535,18 @@ packages: resolution: {integrity: sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ==} engines: {node: '>=18'} + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openai@6.10.0: resolution: {integrity: sha512-ITxOGo7rO3XRMiKA5l7tQ43iNNu+iXGFAcf2t+aWVzzqRaS0i7m1K2BhxNdaveB+5eENhO0VY1FkiZzhBk4v3A==} hasBin: true @@ -4928,6 +4981,9 @@ packages: selderee@0.11.0: resolution: {integrity: sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA==} + semifies@1.0.0: + resolution: {integrity: sha512-xXR3KGeoxTNWPD4aBvL5NUpMTT7WMANr3EWnaS190QVkY52lqqcVRD7Q05UVbBhiWDGWMlJEUam9m7uFFGVScw==} + semver@7.7.3: resolution: {integrity: sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==} engines: {node: '>=10'} @@ -5287,6 +5343,9 @@ packages: resolution: {integrity: sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A==} engines: {node: '>=18'} + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} @@ -5346,6 +5405,10 @@ packages: resolution: {integrity: sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==} hasBin: true + uuidv7@1.1.0: + resolution: {integrity: sha512-2VNnOC0+XQlwogChUDzy6pe8GQEys9QFZBGOh54l6qVfwoCUwwRvk7rDTgaIsRgsF5GFa5oiNg8LqXE3jofBBg==} + hasBin: true + validate-npm-package-name@6.0.2: resolution: {integrity: sha512-IUoow1YUtvoBBC06dXs8bR8B9vuA3aJfmQNKMoaPG/OFsPmoQvw8xh+6Ye25Gx9DQhoEom3Pcu9MKHerm/NpUQ==} engines: {node: ^18.17.0 || >=20.5.0} @@ -5432,10 +5495,17 @@ packages: jsdom: optional: true + weave@0.11.0: + resolution: {integrity: sha512-sRPB6O0sdn/Ueipt1gOxmgrJgbIDAwnm8hmYjSX4mblEoyuAySKgiHNdFJSSJsLyskjHlOIRsFlymECJYlbQKQ==} + web-streams-polyfill@3.3.3: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -8487,8 +8557,17 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 25.0.10 + form-data: 4.0.5 + '@types/node@10.17.60': {} + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@20.19.30': dependencies: undici-types: 6.21.0 @@ -8747,6 +8826,10 @@ snapshots: agent-base@7.1.4: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + ajv-formats@3.0.1(ajv@8.17.1): optionalDependencies: ajv: 8.17.1 @@ -9032,6 +9115,8 @@ snapshots: ci-info@4.3.1: optional: true + cjs-module-lexer@1.4.3: {} + cjs-module-lexer@2.2.0: {} class-variance-authority@0.7.1: @@ -9052,6 +9137,10 @@ snapshots: parse5-htmlparser2-tree-adapter: 6.0.1 yargs: 16.2.0 + cli-progress@3.12.0: + dependencies: + string-width: 4.2.3 + cli-spinners@2.9.2: optional: true @@ -9512,6 +9601,8 @@ snapshots: forever-agent@0.6.1: {} + form-data-encoder@1.7.2: {} + form-data@2.3.3: dependencies: asynckit: 0.4.0 @@ -9526,6 +9617,11 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -9766,6 +9862,10 @@ snapshots: transitivePeerDependencies: - supports-color + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.4.24: dependencies: safer-buffer: 2.1.2 @@ -9781,6 +9881,13 @@ snapshots: immediate@3.0.6: {} + import-in-the-middle@1.15.0: + dependencies: + acorn: 8.15.0 + acorn-import-attributes: 1.9.5(acorn@8.15.0) + cjs-module-lexer: 1.4.3 + module-details-from-path: 1.0.4 + import-in-the-middle@2.0.5: dependencies: acorn: 8.15.0 @@ -9793,6 +9900,8 @@ snapshots: ini@1.3.8: optional: true + ini@5.0.0: {} + ipaddr.js@1.9.1: {} ipull@3.9.3: @@ -10506,6 +10615,21 @@ snapshots: mimic-function: 5.0.1 optional: true + openai@4.104.0(ws@8.19.0)(zod@3.25.76): + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + ws: 8.19.0 + zod: 3.25.76 + transitivePeerDependencies: + - encoding + openai@6.10.0(ws@8.19.0)(zod@4.3.6): optionalDependencies: ws: 8.19.0 @@ -11040,6 +11164,8 @@ snapshots: dependencies: parseley: 0.12.1 + semifies@1.0.0: {} + semver@7.7.3: {} send@0.19.2: @@ -11453,6 +11579,8 @@ snapshots: uint8array-extras@1.5.0: {} + undici-types@5.26.5: {} + undici-types@6.21.0: {} undici-types@7.16.0: {} @@ -11499,6 +11627,8 @@ snapshots: uuid@8.3.2: {} + uuidv7@1.1.0: {} + validate-npm-package-name@6.0.2: optional: true @@ -11565,8 +11695,26 @@ snapshots: - tsx - yaml + weave@0.11.0(ws@8.19.0)(zod@3.25.76): + dependencies: + cli-progress: 3.12.0 + cross-spawn: 7.0.6 + form-data: 4.0.5 + import-in-the-middle: 1.15.0 + ini: 5.0.0 + module-details-from-path: 1.0.4 + openai: 4.104.0(ws@8.19.0)(zod@3.25.76) + semifies: 1.0.0 + uuidv7: 1.1.0 + transitivePeerDependencies: + - encoding + - ws + - zod + web-streams-polyfill@3.3.3: {} + web-streams-polyfill@4.0.0-beta.3: {} + webidl-conversions@3.0.1: {} whatwg-fetch@3.6.20: {} diff --git a/src/agents/llm-request-hook.ts b/src/agents/llm-request-hook.ts new file mode 100644 index 000000000..0870eb67b --- /dev/null +++ b/src/agents/llm-request-hook.ts @@ -0,0 +1,124 @@ +/** + * LLM Request Hook Wrapper + * + * Wraps the streamFn to intercept LLM API payloads and emit them + * to plugins via the llm_request hook. + * + * This enables plugins (like W&B Weave) to capture the complete + * request payload including system prompts for observability. + */ + +import crypto from "node:crypto"; + +import type { StreamFn } from "@mariozechner/pi-agent-core"; + +import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; +import type { PluginHookLlmRequestContext, PluginHookLlmRequestEvent } from "../plugins/hooks.js"; + +/** + * Context for the LLM request hook wrapper + */ +export type LlmRequestHookContext = { + runId?: string; + sessionId?: string; + sessionKey?: string; + agentId?: string; + provider?: string; + modelId?: string; + modelApi?: string | null; + workspaceDir?: string; +}; + +/** + * Safely stringify a value to JSON, handling special types + */ +function safeJsonStringify(value: unknown): string | null { + try { + return JSON.stringify(value, (_key, val) => { + if (typeof val === "bigint") return val.toString(); + if (typeof val === "function") return "[Function]"; + if (val instanceof Error) { + return { name: val.name, message: val.message, stack: val.stack }; + } + if (val instanceof Uint8Array) { + return { type: "Uint8Array", data: Buffer.from(val).toString("base64") }; + } + return val; + }); + } catch { + return null; + } +} + +/** + * Compute SHA256 digest of a value + */ +function digest(value: unknown): string | undefined { + const serialized = safeJsonStringify(value); + if (!serialized) return undefined; + return crypto.createHash("sha256").update(serialized).digest("hex"); +} + +/** + * Create a wrapper for streamFn that emits llm_request hooks + * + * Supports ALL LLM providers that use pi-ai's streaming functions, + * as onPayload callback is supported by all APIs: + * - anthropic-messages (Claude) + * - openai-completions (OpenAI, Mistral, xAI, Cerebras, Groq, etc.) + * - openai-responses (OpenAI newer API) + * - google-generative-ai (Gemini) + * - bedrock-converse-stream (AWS Bedrock) + * - google-vertex (Vertex AI) + */ +export function createLlmRequestHookWrapper(ctx: LlmRequestHookContext): { + wrapStreamFn: (streamFn: StreamFn) => StreamFn; +} | null { + const hookRunner = getGlobalHookRunner(); + + // If no hook runner or no hooks registered, skip wrapping + if (!hookRunner || !hookRunner.hasHooks("llm_request")) { + return null; + } + + const hookCtx: PluginHookLlmRequestContext = { + runId: ctx.runId, + sessionId: ctx.sessionId, + sessionKey: ctx.sessionKey, + agentId: ctx.agentId, + provider: ctx.provider, + modelId: ctx.modelId, + modelApi: ctx.modelApi, + workspaceDir: ctx.workspaceDir, + }; + + const wrapStreamFn = (streamFn: StreamFn): StreamFn => { + const wrapped: StreamFn = (model, context, options) => { + const nextOnPayload = (payload: unknown) => { + // Emit the llm_request hook with the full payload + const event: PluginHookLlmRequestEvent = { + payload: payload as PluginHookLlmRequestEvent["payload"], + payloadDigest: digest(payload), + timestamp: new Date().toISOString(), + }; + + // Fire hook asynchronously (don't block the request) + hookRunner.runLlmRequest(event, hookCtx).catch((err) => { + console.error("[llm-request-hook] Hook execution failed:", err); + }); + + // Pass through to any existing onPayload handler + options?.onPayload?.(payload); + }; + + return streamFn(model, context, { + ...options, + onPayload: nextOnPayload, + }); + }; + + return wrapped; + }; + + return { wrapStreamFn }; +} diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index f1c487470..42a0ce00d 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -22,6 +22,7 @@ import { isSubagentSessionKey } from "../../../routing/session-key.js"; import { resolveUserPath } from "../../../utils.js"; import { createCacheTrace } from "../../cache-trace.js"; import { createAnthropicPayloadLogger } from "../../anthropic-payload-log.js"; +import { createLlmRequestHookWrapper } from "../../llm-request-hook.js"; import { resolveClawdbotAgentDir } from "../../agent-paths.js"; import { resolveSessionAgentIds } from "../../agent-scope.js"; import { makeBootstrapWarn, resolveBootstrapContextForRun } from "../../bootstrap-files.js"; @@ -510,6 +511,21 @@ export async function runEmbeddedAttempt( ); } + // LLM Request Hook - emits llm_request hook for plugins to observe full payloads + const llmRequestHook = createLlmRequestHookWrapper({ + runId: params.runId, + sessionId: activeSession.sessionId, + sessionKey: params.sessionKey, + agentId: params.sessionKey?.split(":")[0] ?? "main", + provider: params.provider, + modelId: params.modelId, + modelApi: params.model.api, + workspaceDir: params.workspaceDir, + }); + if (llmRequestHook) { + activeSession.agent.streamFn = llmRequestHook.wrapStreamFn(activeSession.agent.streamFn); + } + try { const prior = await sanitizeSessionHistory({ messages: activeSession.messages, diff --git a/src/plugins/hooks.ts b/src/plugins/hooks.ts index 41fd501f7..cc3530a93 100644 --- a/src/plugins/hooks.ts +++ b/src/plugins/hooks.ts @@ -19,6 +19,8 @@ import type { PluginHookGatewayContext, PluginHookGatewayStartEvent, PluginHookGatewayStopEvent, + PluginHookLlmRequestContext, + PluginHookLlmRequestEvent, PluginHookMessageContext, PluginHookMessageReceivedEvent, PluginHookMessageSendingEvent, @@ -41,6 +43,8 @@ export type { PluginHookBeforeAgentStartEvent, PluginHookBeforeAgentStartResult, PluginHookAgentEndEvent, + PluginHookLlmRequestContext, + PluginHookLlmRequestEvent, PluginHookBeforeCompactionEvent, PluginHookAfterCompactionEvent, PluginHookMessageContext, @@ -206,6 +210,19 @@ export function createHookRunner(registry: PluginRegistry, options: HookRunnerOp return runVoidHook("agent_end", event, ctx); } + /** + * Run llm_request hook. + * Called when an LLM API request is about to be sent. + * Provides the complete request payload including system prompt. + * Runs in parallel (fire-and-forget, observability only). + */ + async function runLlmRequest( + event: PluginHookLlmRequestEvent, + ctx: PluginHookLlmRequestContext, + ): Promise { + return runVoidHook("llm_request", event, ctx); + } + /** * Run before_compaction hook. */ @@ -435,6 +452,7 @@ export function createHookRunner(registry: PluginRegistry, options: HookRunnerOp // Agent hooks runBeforeAgentStart, runAgentEnd, + runLlmRequest, runBeforeCompaction, runAfterCompaction, // Message hooks diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 1ce9731ea..a619d764d 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -289,6 +289,7 @@ export type PluginDiagnostic = { export type PluginHookName = | "before_agent_start" | "agent_end" + | "llm_request" | "before_compaction" | "after_compaction" | "message_received" @@ -462,6 +463,42 @@ export type PluginHookGatewayStopEvent = { reason?: string; }; +// llm_request hook - captures the full LLM API request payload +export type PluginHookLlmRequestEvent = { + /** The complete request payload sent to the LLM API */ + payload: { + /** System prompt (if any) */ + system?: unknown; + /** Messages array */ + messages?: unknown[]; + /** Model ID */ + model?: string; + /** Max tokens */ + max_tokens?: number; + /** Temperature */ + temperature?: number; + /** Tools/functions available */ + tools?: unknown[]; + /** Any other API parameters */ + [key: string]: unknown; + }; + /** SHA256 hash of the payload for deduplication */ + payloadDigest?: string; + /** Timestamp of the request */ + timestamp: string; +}; + +export type PluginHookLlmRequestContext = { + agentId?: string; + sessionKey?: string; + sessionId?: string; + runId?: string; + provider?: string; + modelId?: string; + modelApi?: string | null; + workspaceDir?: string; +}; + // Hook handler types mapped by hook name export type PluginHookHandlerMap = { before_agent_start: ( @@ -469,6 +506,10 @@ export type PluginHookHandlerMap = { ctx: PluginHookAgentContext, ) => Promise | PluginHookBeforeAgentStartResult | void; agent_end: (event: PluginHookAgentEndEvent, ctx: PluginHookAgentContext) => Promise | void; + llm_request: ( + event: PluginHookLlmRequestEvent, + ctx: PluginHookLlmRequestContext, + ) => Promise | void; before_compaction: ( event: PluginHookBeforeCompactionEvent, ctx: PluginHookAgentContext,