diff --git a/docs/concepts/self-healing.md b/docs/concepts/self-healing.md new file mode 100644 index 000000000..0a8e587a6 --- /dev/null +++ b/docs/concepts/self-healing.md @@ -0,0 +1,294 @@ +# Self-Healing Behaviors + +Clawdbot includes built-in resilience mechanisms that automatically recover from transient failures without operator intervention. This document describes each self-healing behavior. + +## Exponential Backoff Reconnection + +All network connections use exponential backoff for automatic reconnection: + +### Backoff Policy + +```typescript +type BackoffPolicy = { + initialMs: number; // First retry delay + maxMs: number; // Maximum delay cap + factor: number; // Multiplier per attempt (typically 2) + jitter: number; // Random variance (0.1 = 10%) +}; +``` + +The backoff formula: `delay = min(maxMs, initialMs * factor^attempt + random * jitter)` + +### Channel Reconnection + +Channels automatically reconnect when connections drop: + +| Channel | Initial Delay | Max Delay | Behavior | +|---------|--------------|-----------|----------| +| Signal SSE | 1s | 10s | Reconnects on stream end or error | +| Discord WebSocket | 500ms | 30s | Reconnects on rate limit (429) | +| Telegram polling | 400ms | 30s | Reconnects on transient errors | + +When a connection succeeds, the attempt counter resets to zero. + +### Configuration + +Per-provider retry settings in `config.yaml`: + +```yaml +channels: + telegram: + retry: + attempts: 3 + minDelayMs: 400 + maxDelayMs: 30000 + jitter: 0.1 + + discord: + retry: + attempts: 3 + minDelayMs: 500 + maxDelayMs: 30000 + jitter: 0.1 +``` + +## Model Failover Cascade + +When a model request fails, Clawdbot automatically tries fallback models. + +### How It Works + +1. **Primary model attempt** - Try the configured primary model +2. **Check cooldowns** - Skip providers where all auth profiles are in cooldown +3. **Fallback cascade** - Try each configured fallback in order +4. **Error aggregation** - Collect errors from all attempts for debugging + +### Configuration + +```yaml +agents: + defaults: + model: + primary: anthropic/claude-sonnet-4-20250514 + fallbacks: + - anthropic/claude-3-5-haiku-latest + - openai/gpt-4o +``` + +### Failover Conditions + +Failover triggers on: +- Rate limit errors (HTTP 429) +- Server errors (HTTP 5xx) +- Timeout errors +- Authentication errors (credential issues) +- Model unavailable errors + +Failover does **not** trigger on: +- User abort/cancel +- Invalid request errors (client bugs) +- Context overflow (not recoverable by switching models) + +### Image Model Failover + +Image generation has separate fallback configuration: + +```yaml +agents: + defaults: + imageModel: + primary: anthropic/claude-sonnet-4-20250514 + fallbacks: + - openai/dall-e-3 +``` + +## Auth Profile Cooldown + +When API requests fail due to rate limiting or billing issues, auth profiles enter a cooldown period. + +### Cooldown Progression + +For rate limit/transient errors: +- 1st failure: 1 minute cooldown +- 2nd failure: 5 minutes +- 3rd failure: 25 minutes +- Maximum: 1 hour + +For billing errors (longer backoff): +- Default base: 5 hours +- Maximum: 24 hours +- Uses exponential growth: `baseMs * 2^(failures-1)` + +### Cooldown Behavior + +- **Automatic recovery**: Cooldown clears automatically after the timeout +- **Success clears cooldown**: A successful request resets error count to zero +- **Provider skipping**: Model failover skips providers where all profiles are in cooldown +- **Failure window**: Error count resets if 24 hours pass without new failures + +### Configuration + +```yaml +auth: + cooldowns: + billingBackoffHours: 5 # Base delay for billing errors + billingMaxHours: 24 # Maximum billing cooldown + failureWindowHours: 24 # Reset window for error count + billingBackoffHoursByProvider: + openai: 12 # Provider-specific override +``` + +### Manual Reset + +Clear cooldown for a specific profile: + +```bash +clawdbot auth profiles --clear-cooldown +``` + +## Token Bucket Rate Limiting + +The gateway uses token bucket rate limiting to prevent abuse while allowing burst traffic. + +### How Token Bucket Works + +1. Each client has a bucket with a maximum token capacity +2. Tokens refill continuously at a fixed rate (tokens per minute) +3. Each request consumes one token +4. If no tokens available, request is rate-limited + +### Default Limits + +| Client Type | Rate | Burst | +|-------------|------|-------| +| Unauthenticated | 60/min | 2x (120 tokens) | +| Authenticated | Unlimited | - | +| Channel messages | 200/min | 2x (400 tokens) | + +### Auth Failure Backoff + +After repeated authentication failures, clients are temporarily blocked: + +- **Threshold**: 5 failures before backoff starts +- **Base delay**: 1 second +- **Growth**: Exponential (1s, 2s, 4s, 8s...) +- **Maximum**: 1 minute +- **Reset**: 10 minutes of inactivity clears failure count + +### Configuration + +```yaml +gateway: + rateLimit: + enabled: true + unauthenticated: 60 # Requests per minute + authenticated: 0 # 0 = unlimited + channelMessages: 200 # Per channel + burstMultiplier: 2 # Allow 2x burst + authFailuresBeforeBackoff: 5 + authBackoffBaseMs: 1000 + authBackoffMaxMs: 60000 +``` + +## Session Stuck Detection + +The diagnostic system monitors for sessions that appear stuck in a particular state. + +### How It Works + +The gateway emits `session.stuck` diagnostic events when a session remains in `processing` or `waiting` state longer than expected. + +```typescript +type DiagnosticSessionStuckEvent = { + type: "session.stuck"; + sessionKey?: string; + sessionId?: string; + state: "idle" | "processing" | "waiting"; + ageMs: number; // How long in this state + queueDepth?: number; // Pending messages +}; +``` + +### What Triggers Detection + +- Session in `processing` state for extended period +- Session in `waiting` state with no progress +- High queue depth combined with state staleness + +### Monitoring Integration + +Subscribe to stuck session events: + +```typescript +import { onDiagnosticEvent } from 'clawdbot/diagnostic-events'; + +onDiagnosticEvent((event) => { + if (event.type === 'session.stuck') { + // Alert ops team + alertChannel.send(`Session ${event.sessionKey} stuck for ${event.ageMs}ms`); + } +}); +``` + +### Prometheus Alert + +```yaml +- alert: SessionStuck + expr: clawdbot_session_stuck_total > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Session appears stuck" +``` + +## Diagnostic Heartbeat + +The gateway emits periodic heartbeat events summarizing system health: + +```typescript +type DiagnosticHeartbeatEvent = { + type: "diagnostic.heartbeat"; + webhooks: { + received: number; // Total webhooks received + processed: number; // Successfully processed + errors: number; // Errors encountered + }; + active: number; // Sessions currently processing + waiting: number; // Sessions waiting for user + queued: number; // Messages in queue +}; +``` + +Use heartbeats to: +- Verify gateway is alive and processing +- Monitor queue backlog growth +- Track error rates over time + +## Recovery Patterns Summary + +| Failure Type | Self-Healing Mechanism | Time to Recover | +|--------------|------------------------|-----------------| +| Network disconnect | Exponential backoff reconnect | 1s - 30s | +| Model rate limit | Failover to backup model | Immediate | +| Model unavailable | Failover cascade | Immediate | +| Auth profile rate limit | Profile cooldown + rotation | 1min - 1hr | +| Billing error | Extended cooldown | 5hr - 24hr | +| Gateway overload | Token bucket + queue | Immediate backpressure | +| Brute-force auth | Auth failure backoff | 1s - 60s | +| Stuck session | Diagnostic event + alert | Requires operator | + +## Best Practices + +1. **Configure fallback models** - Always have at least one fallback for critical workflows +2. **Monitor diagnostic events** - Set up alerts for `session.stuck` and high error rates +3. **Use multiple auth profiles** - Distribute load across profiles to avoid single-profile rate limits +4. **Review cooldown settings** - Tune for your provider's rate limit behavior +5. **Enable rate limiting** - Protect against accidental or malicious overload + +## Related Documentation + +- [Model Failover](/concepts/model-failover) - Detailed model configuration +- [Retry Policy](/concepts/retry) - Per-provider retry settings +- [Observability](/enterprise/observability) - Metrics and alerting +- [Security Hardening](/enterprise/security-hardening) - Rate limit configuration diff --git a/docs/enterprise/deployment.md b/docs/enterprise/deployment.md new file mode 100644 index 000000000..a5442730f --- /dev/null +++ b/docs/enterprise/deployment.md @@ -0,0 +1,348 @@ +# Enterprise Deployment Guide + +This guide covers deployment patterns for enterprise environments with multiple users, teams, and security requirements. + +## Deployment Patterns + +### Single-Tenant (Recommended for Teams) + +A single gateway serves one team with shared configuration: + +``` +┌─────────────────────────────────────────┐ +│ Gateway Host │ +│ ┌─────────────────────────────────┐ │ +│ │ Clawdbot Gateway │ │ +│ │ - All team channels │ │ +│ │ - Shared agent config │ │ +│ │ - Team-wide RBAC │ │ +│ └─────────────────────────────────┘ │ +└─────────────────────────────────────────┘ + │ │ │ + ┌────┴───┐ ┌────┴───┐ ┌────┴───┐ + │ User A │ │ User B │ │ User C │ + └────────┘ └────────┘ └────────┘ +``` + +**Configuration:** + +```yaml +# config.yaml +gateway: + port: 18789 + auth: + mode: token + token: ${GATEWAY_TOKEN} + +rbac: + enabled: true + defaultRole: user + assignments: + "admin@company.com": admin + "dev@company.com": operator +``` + +### Multi-Tenant (Isolated Teams) + +Separate gateway instances per team for full isolation: + +``` +┌──────────────────┐ ┌──────────────────┐ +│ Team A Host │ │ Team B Host │ +│ ┌──────────────┐ │ │ ┌──────────────┐ │ +│ │ Gateway A │ │ │ │ Gateway B │ │ +│ │ Port 18789 │ │ │ │ Port 18790 │ │ +│ └──────────────┘ │ │ └──────────────┘ │ +└──────────────────┘ └──────────────────┘ + │ │ + Team A Users Team B Users +``` + +**Benefits:** +- Complete data isolation between teams +- Independent configuration and credentials +- Separate audit logs per team +- Different RBAC policies per team + +**Setup:** + +```bash +# Team A gateway +clawdbot config set gateway.port 18789 +clawdbot config set gateway.auth.token "team-a-token" + +# Team B gateway (different host or port) +clawdbot config set gateway.port 18790 +clawdbot config set gateway.auth.token "team-b-token" +``` + +### Multi-Agent Routing + +Route different users to different agents based on channel or identity: + +```yaml +# config.yaml +agents: + main: + model: claude-sonnet-4-20250514 + system: "You are a general assistant." + + devops: + model: claude-sonnet-4-20250514 + system: "You are a DevOps specialist." + tools: + policy: elevated + + support: + model: claude-sonnet-4-20250514 + system: "You are a customer support agent." + tools: + policy: read-only + +bindings: + - channel: slack + channelId: "devops-channel" + agentId: devops + + - channel: telegram + senderId: "support-team" + agentId: support +``` + +### Per-User Agent Isolation + +Give each user their own agent instance: + +```yaml +# config.yaml +agents: + template: + model: claude-sonnet-4-20250514 + session: + mode: persistent + dir: ~/.clawdbot/sessions/${senderId} + +bindings: + - channel: "*" + agentId: template + sessionKey: "${channel}:${senderId}" +``` + +## Container Deployment + +### Docker + +```dockerfile +FROM node:22-alpine +RUN npm install -g clawdbot@latest +EXPOSE 18789 +VOLUME /root/.clawdbot +CMD ["clawdbot", "gateway", "run", "--bind", "0.0.0.0"] +``` + +```yaml +# docker-compose.yaml +services: + clawdbot: + image: clawdbot:latest + ports: + - "18789:18789" + volumes: + - ./config:/root/.clawdbot + environment: + - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} + - GATEWAY_TOKEN=${GATEWAY_TOKEN} + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:18789/health"] + interval: 30s + timeout: 10s + retries: 3 +``` + +### Kubernetes + +```yaml +# deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: clawdbot-gateway +spec: + replicas: 1 + selector: + matchLabels: + app: clawdbot + template: + metadata: + labels: + app: clawdbot + spec: + containers: + - name: gateway + image: clawdbot:latest + ports: + - containerPort: 18789 + env: + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: + name: clawdbot-secrets + key: anthropic-api-key + livenessProbe: + httpGet: + path: /health + port: 18789 + initialDelaySeconds: 10 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /ready + port: 18789 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /root/.clawdbot + volumes: + - name: config + configMap: + name: clawdbot-config +``` + +```yaml +# service.yaml +apiVersion: v1 +kind: Service +metadata: + name: clawdbot + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "18789" + prometheus.io/path: "/metrics" +spec: + selector: + app: clawdbot + ports: + - port: 18789 + targetPort: 18789 +``` + +## High Availability + +### Load Balancing Considerations + +Clawdbot gateway is stateful (WebSocket connections, session state). For HA: + +1. **Sticky sessions** - Route users to the same instance +2. **Shared state** - Use external session storage (future feature) +3. **Active-passive** - Run standby instance for failover + +### Health Endpoints + +Configure your load balancer to use: + +- **Liveness:** `GET /health` - Returns 200 if process is alive +- **Readiness:** `GET /ready` - Returns 200 if channels are connected + +### Graceful Shutdown + +The gateway handles SIGTERM gracefully: +1. Stops accepting new connections +2. Completes in-flight requests +3. Closes WebSocket connections cleanly +4. Flushes audit logs + +## Network Configuration + +### Firewall Rules + +| Port | Direction | Purpose | +|------|-----------|---------| +| 18789 | Inbound | Gateway WebSocket/HTTP | +| 443 | Outbound | Anthropic API, channel APIs | + +### Reverse Proxy + +With nginx: + +```nginx +upstream clawdbot { + server localhost:18789; +} + +server { + listen 443 ssl; + server_name gateway.company.com; + + ssl_certificate /etc/ssl/certs/gateway.crt; + ssl_certificate_key /etc/ssl/private/gateway.key; + + location / { + proxy_pass http://clawdbot; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 86400; + } +} +``` + +Configure trusted proxies in Clawdbot: + +```yaml +gateway: + trustedProxies: + - "10.0.0.0/8" + - "172.16.0.0/12" + - "192.168.0.0/16" +``` + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `ANTHROPIC_API_KEY` | API key for Claude models | +| `GATEWAY_TOKEN` | Gateway authentication token | +| `CLAWDBOT_CONFIG_PATH` | Custom config file path | +| `CLAWDBOT_STATE_DIR` | State directory (sessions, logs) | +| `LOG_LEVEL` | Logging verbosity (debug, info, warn, error) | + +## Backup and Recovery + +### Critical Data + +| Path | Description | Backup Priority | +|------|-------------|-----------------| +| `~/.clawdbot/config.yaml` | Configuration | High | +| `~/.clawdbot/credentials/` | Channel tokens, pairing | High | +| `~/.clawdbot/sessions/` | Session history | Medium | +| `~/.clawdbot/audit.jsonl` | Audit logs | High | + +### Backup Script + +```bash +#!/bin/bash +BACKUP_DIR="/backups/clawdbot/$(date +%Y%m%d)" +mkdir -p "$BACKUP_DIR" + +# Config and credentials (encrypted) +tar -czf "$BACKUP_DIR/config.tar.gz" \ + ~/.clawdbot/config.yaml \ + ~/.clawdbot/credentials/ + +# Audit logs +cp ~/.clawdbot/audit*.jsonl "$BACKUP_DIR/" + +# Retain 30 days +find /backups/clawdbot -mtime +30 -delete +``` + +## Next Steps + +- [Security Hardening](/enterprise/security-hardening) - TLS, rate limiting, audit configuration +- [Observability](/enterprise/observability) - Metrics, tracing, dashboards +- [Gateway Configuration](/gateway/configuration) - Full configuration reference diff --git a/docs/enterprise/observability.md b/docs/enterprise/observability.md new file mode 100644 index 000000000..4f0171b9f --- /dev/null +++ b/docs/enterprise/observability.md @@ -0,0 +1,465 @@ +# Observability Guide + +This guide covers monitoring, metrics, tracing, and alerting for enterprise deployments. + +## Health Endpoints + +The gateway exposes HTTP health endpoints for container orchestration: + +### Liveness Probe + +``` +GET /health +``` + +Returns `200 OK` if the gateway process is alive: + +```json +{ + "status": "healthy", + "version": "2024.1.15", + "uptimeMs": 3600000 +} +``` + +### Readiness Probe + +``` +GET /ready +``` + +Returns `200 OK` if channels are connected and ready: + +```json +{ + "status": "healthy", + "checks": { + "channels": { + "telegram": "connected", + "discord": "connected", + "slack": "degraded" + } + } +} +``` + +Returns `503 Service Unavailable` if critical channels are down. + +### Deep Health Check + +``` +GET /health/deep +Authorization: Bearer +``` + +Returns detailed system status (requires authentication): + +```json +{ + "status": "healthy", + "version": "2024.1.15", + "uptimeMs": 3600000, + "checks": { + "channels": { + "telegram": { "status": "connected", "latencyMs": 45 }, + "discord": { "status": "connected", "latencyMs": 62 } + }, + "memory": { + "heapUsed": 128000000, + "heapTotal": 256000000, + "rss": 312000000 + } + } +} +``` + +### Kubernetes Configuration + +```yaml +livenessProbe: + httpGet: + path: /health + port: 18789 + initialDelaySeconds: 10 + periodSeconds: 30 + failureThreshold: 3 + +readinessProbe: + httpGet: + path: /ready + port: 18789 + initialDelaySeconds: 5 + periodSeconds: 10 + failureThreshold: 2 +``` + +## Prometheus Metrics + +The gateway exposes Prometheus metrics at `/metrics`: + +``` +GET /metrics +Authorization: Bearer +``` + +### Available Metrics + +#### Token Usage + +```prometheus +# Total tokens used +clawdbot_tokens_total{type="input"} 125000 +clawdbot_tokens_total{type="output"} 45000 +clawdbot_tokens_total{type="cache_read"} 80000 +clawdbot_tokens_total{type="cache_write"} 15000 +``` + +#### Cost Tracking + +```prometheus +# Total cost in USD +clawdbot_cost_usd_total{channel="telegram",provider="anthropic",model="claude-sonnet-4-20250514"} 1.25 +``` + +#### Message Processing + +```prometheus +# Messages processed +clawdbot_message_processed_total{channel="telegram",outcome="completed"} 500 +clawdbot_message_processed_total{channel="telegram",outcome="error"} 5 + +# Message processing duration +clawdbot_message_duration_seconds_bucket{channel="telegram",le="1"} 450 +clawdbot_message_duration_seconds_bucket{channel="telegram",le="5"} 495 +clawdbot_message_duration_seconds_bucket{channel="telegram",le="30"} 500 +``` + +#### Webhooks + +```prometheus +# Webhook events +clawdbot_webhook_received_total{channel="telegram"} 1000 +clawdbot_webhook_processed_total{channel="telegram"} 995 +clawdbot_webhook_error_total{channel="telegram"} 5 +``` + +#### Sessions + +```prometheus +# Active sessions by state +clawdbot_session_state{state="idle"} 10 +clawdbot_session_state{state="processing"} 2 +clawdbot_session_state{state="waiting"} 1 +``` + +#### Queue + +```prometheus +# Queue depth +clawdbot_queue_depth{lane="default"} 3 +``` + +#### System + +```prometheus +# Gateway uptime +clawdbot_gateway_uptime_seconds 3600 +``` + +### Prometheus Configuration + +```yaml +# prometheus.yml +scrape_configs: + - job_name: 'clawdbot' + static_configs: + - targets: ['gateway-host:18789'] + bearer_token: '' + metrics_path: '/metrics' +``` + +### Grafana Dashboard + +Import this dashboard for key metrics: + +```json +{ + "title": "Clawdbot Gateway", + "panels": [ + { + "title": "Token Usage Rate", + "type": "graph", + "targets": [ + { + "expr": "rate(clawdbot_tokens_total[5m])", + "legendFormat": "{{type}}" + } + ] + }, + { + "title": "Message Throughput", + "type": "graph", + "targets": [ + { + "expr": "rate(clawdbot_message_processed_total[5m])", + "legendFormat": "{{channel}} - {{outcome}}" + } + ] + }, + { + "title": "Cost per Hour", + "type": "stat", + "targets": [ + { + "expr": "increase(clawdbot_cost_usd_total[1h])" + } + ] + }, + { + "title": "Active Sessions", + "type": "gauge", + "targets": [ + { + "expr": "sum(clawdbot_session_state)" + } + ] + } + ] +} +``` + +## Distributed Tracing + +The gateway supports W3C Trace Context for distributed tracing. + +### Trace ID Propagation + +All requests include a trace ID that flows through: +- Audit logs (`traceId` field) +- Diagnostic events +- Console/file logs +- Outgoing HTTP requests + +### Log Correlation + +Trace IDs appear in logs for correlation: + +``` +[gateway] [a1b2c3d4] Processing message from telegram +[agent] [a1b2c3d4] Invoking claude-sonnet-4-20250514 +[gateway] [a1b2c3d4] Message completed in 2.3s +``` + +### Audit Log Correlation + +Query audit logs by trace ID: + +```bash +cat ~/.clawdbot/audit.jsonl | jq 'select(.traceId == "4bf92f3577b34da6a3ce929d0e0e4736")' +``` + +### OpenTelemetry Integration + +For full APM integration, use the diagnostics-otel extension: + +```yaml +plugins: + diagnostics-otel: + enabled: true + endpoint: "http://otel-collector:4317" +``` + +This exports: +- Traces to your APM (Jaeger, Zipkin, Datadog, etc.) +- Metrics in OTLP format +- Logs with trace correlation + +## Logging + +### Log Levels + +Configure log verbosity: + +```yaml +logging: + level: info # trace, debug, info, warn, error, silent + console: + level: warn # Separate console level + style: pretty # pretty, compact, json +``` + +### Structured Logging + +JSON log format for log aggregation: + +```yaml +logging: + console: + style: json +``` + +Output: + +```json +{"time":"2024-01-15T10:30:00.000Z","level":"info","subsystem":"gateway","message":"Gateway started","traceId":"a1b2c3d4"} +``` + +### Log Files + +Logs are written to `~/.clawdbot/logs/`: + +``` +~/.clawdbot/logs/ +├── clawdbot.log # Current log +├── clawdbot.1.log # Rotated logs +└── clawdbot.2.log +``` + +### Log Aggregation + +Ship logs to your aggregation system: + +**Filebeat:** +```yaml +filebeat.inputs: + - type: log + paths: + - /root/.clawdbot/logs/*.log + json.keys_under_root: true + json.add_error_key: true + +output.elasticsearch: + hosts: ["elasticsearch:9200"] +``` + +**Fluentd:** +``` + + @type tail + path /root/.clawdbot/logs/*.log + pos_file /var/log/fluentd/clawdbot.pos + tag clawdbot + + @type json + + +``` + +## Alerting + +### Prometheus Alerting Rules + +```yaml +# alerts.yml +groups: + - name: clawdbot + rules: + - alert: ClawdbotDown + expr: up{job="clawdbot"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Clawdbot gateway is down" + + - alert: HighErrorRate + expr: rate(clawdbot_message_processed_total{outcome="error"}[5m]) > 0.1 + for: 5m + labels: + severity: warning + annotations: + summary: "High message error rate" + + - alert: ChannelDisconnected + expr: clawdbot_channel_connected == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Channel {{ $labels.channel }} disconnected" + + - alert: HighCostRate + expr: increase(clawdbot_cost_usd_total[1h]) > 10 + for: 1m + labels: + severity: info + annotations: + summary: "High API cost in last hour: ${{ $value }}" +``` + +### Key Metrics to Monitor + +| Metric | Alert Threshold | Severity | +|--------|-----------------|----------| +| Gateway uptime | Down > 1m | Critical | +| Message error rate | > 10% | Warning | +| Channel disconnected | > 5m | Warning | +| Hourly cost | > $10 | Info | +| Queue depth | > 100 | Warning | +| Processing latency p99 | > 30s | Warning | + +## Diagnostic Events + +The gateway emits real-time diagnostic events for monitoring: + +```typescript +// Subscribe to diagnostic events +import { onDiagnosticEvent } from 'clawdbot/diagnostic-events'; + +onDiagnosticEvent((event) => { + console.log(event.type, event); +}); +``` + +### Event Types + +| Event | Description | +|-------|-------------| +| `model.usage` | Token usage and cost | +| `webhook.received` | Incoming webhook | +| `webhook.processed` | Webhook processing complete | +| `webhook.error` | Webhook processing error | +| `message.queued` | Message added to queue | +| `message.processed` | Message processing complete | +| `session.state` | Session state change | +| `session.stuck` | Session stuck detection | +| `diagnostic.heartbeat` | Periodic status summary | + +## Troubleshooting + +### Common Issues + +**No metrics at /metrics:** +- Check authentication header +- Verify `gateway.metrics.enabled: true` + +**Missing trace IDs:** +- Ensure requests flow through the gateway +- Check log format includes trace context + +**High memory usage:** +- Review session retention settings +- Check for stuck sessions +- Monitor queue depth + +### Debug Commands + +```bash +# Check gateway health +curl -s http://localhost:18789/health | jq + +# View recent audit events +tail -100 ~/.clawdbot/audit.jsonl | jq + +# Check channel status +clawdbot channels status --deep + +# View diagnostic events +clawdbot gateway events --follow +``` + +## Next Steps + +- [Enterprise Deployment](/enterprise/deployment) - Deployment patterns +- [Security Hardening](/enterprise/security-hardening) - Security configuration +- [Gateway Troubleshooting](/gateway/troubleshooting) - Detailed troubleshooting diff --git a/docs/enterprise/security-hardening.md b/docs/enterprise/security-hardening.md new file mode 100644 index 000000000..d60764828 --- /dev/null +++ b/docs/enterprise/security-hardening.md @@ -0,0 +1,366 @@ +# Security Hardening Guide + +This guide covers security configuration for enterprise deployments. + +## Authentication + +### Gateway Authentication + +Always enable authentication for production deployments: + +```yaml +# config.yaml +gateway: + auth: + mode: token # or "password" + token: ${GATEWAY_TOKEN} # Use environment variable +``` + +**Token vs Password:** +- **Token:** Preferred for API clients and automation +- **Password:** Suitable for interactive users + +Generate a secure token: + +```bash +openssl rand -base64 32 +``` + +### Device Pairing + +For remote access, use device pairing with cryptographic identity: + +```yaml +gateway: + auth: + mode: token + token: ${GATEWAY_TOKEN} +``` + +Paired devices receive a device token after approval, eliminating the need to share the gateway token. + +### Tailscale Authentication + +For zero-trust networking with Tailscale: + +```yaml +gateway: + auth: + mode: tailscale + allowTailscale: true +``` + +## Role-Based Access Control (RBAC) + +Enable RBAC to restrict user permissions: + +```yaml +rbac: + enabled: true + defaultRole: user # Fallback for unassigned users + + roles: + # Custom role for DevOps team + devops: + name: "DevOps Engineer" + permissions: + - exec # Basic command execution + - exec.elevated # Sudo/admin commands + - exec.approve # Can approve exec requests + agents: + - main + - deploy + + # Restricted role for support + support: + name: "Support Agent" + permissions: + - exec + tools: + deny: + - bash + - write + + assignments: + "admin@company.com": admin + "devops@company.com": devops + "support@company.com": support +``` + +### Permission Levels + +| Permission | Description | +|------------|-------------| +| `exec` | Execute basic commands | +| `exec.elevated` | Execute sudo/admin commands | +| `exec.approve` | Approve exec requests from agents | +| `admin` | Full access (grants all permissions) | +| `read-only` | View-only access, no tool execution | + +### Tool Restrictions + +Restrict specific tools per role: + +```yaml +roles: + limited: + name: "Limited User" + permissions: [exec] + tools: + allow: + - read + - search + - glob + deny: + - bash + - write + - edit +``` + +## Audit Logging + +All security-relevant events are logged to `~/.clawdbot/audit.jsonl`: + +```yaml +# Audit logging is enabled by default +# Configure retention in gateway settings +gateway: + audit: + enabled: true + retentionDays: 30 # Keep logs for 30 days +``` + +### Audited Events + +| Event Type | Description | +|------------|-------------| +| `auth.login` | Successful authentication | +| `auth.failure` | Failed authentication attempt | +| `pairing.request` | Device pairing request | +| `pairing.approve` | Device pairing approved | +| `pairing.reject` | Device pairing rejected | +| `exec.request` | Command execution requested | +| `exec.approve` | Command execution approved | +| `exec.reject` | Command execution rejected | +| `rbac.denied` | RBAC permission denied | +| `config.change` | Configuration modified | + +### Audit Log Format + +Each entry is a JSON line: + +```json +{ + "ts": "2024-01-15T10:30:00.000Z", + "eventId": "550e8400-e29b-41d4-a716-446655440000", + "type": "auth.login", + "actor": { + "type": "device", + "id": "device-abc123", + "remoteIp": "192.168.1.100" + }, + "outcome": "success", + "traceId": "4bf92f3577b34da6a3ce929d0e0e4736", + "metadata": { + "method": "device-token" + } +} +``` + +### Querying Audit Logs + +```bash +# Find all failed auth attempts +cat ~/.clawdbot/audit.jsonl | jq 'select(.type == "auth.failure")' + +# Find RBAC denials for a user +cat ~/.clawdbot/audit.jsonl | jq 'select(.type == "rbac.denied" and .actor.id == "user@company.com")' + +# Count events by type +cat ~/.clawdbot/audit.jsonl | jq -s 'group_by(.type) | map({type: .[0].type, count: length})' +``` + +## Rate Limiting + +The gateway includes built-in rate limiting to prevent abuse: + +```yaml +gateway: + rateLimit: + enabled: true + windowMs: 60000 # 1 minute window + maxRequests: 100 # Max requests per window + maxConnections: 50 # Max concurrent WebSocket connections +``` + +### Pairing Rate Limits + +Pairing attempts are rate-limited to prevent brute-force attacks: +- Maximum 10 attempts per minute per channel +- Automatic backoff on repeated failures + +## TLS Configuration + +### With Reverse Proxy (Recommended) + +Terminate TLS at your reverse proxy (nginx, Caddy, Traefik): + +```nginx +server { + listen 443 ssl http2; + server_name gateway.company.com; + + ssl_certificate /etc/ssl/certs/gateway.crt; + ssl_certificate_key /etc/ssl/private/gateway.key; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + + # HSTS + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + + location / { + proxy_pass http://localhost:18789; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header X-Forwarded-Proto $scheme; + } +} +``` + +### Trusted Proxies + +When behind a proxy, configure trusted proxy addresses: + +```yaml +gateway: + trustedProxies: + - "127.0.0.1" + - "10.0.0.0/8" + - "172.16.0.0/12" +``` + +This ensures client IP addresses are correctly extracted from `X-Forwarded-For` headers. + +## Command Execution Security + +### Exec Approvals + +Require human approval for dangerous commands: + +```yaml +approvals: + exec: + enabled: true + requireApproval: + - elevated # Require approval for sudo commands + - destructive # Require approval for rm, delete, etc. + timeoutMs: 120000 # 2 minute timeout +``` + +### Sandbox Mode + +Run commands in a restricted sandbox: + +```yaml +tools: + policy: sandbox # Restrict file system access +``` + +### Command Blocklist + +Block dangerous command patterns: + +```yaml +tools: + exec: + blocklist: + - "rm -rf /" + - ":(){ :|:& };:" # Fork bomb + - "dd if=/dev/zero" +``` + +## Secrets Management + +### Environment Variables + +Store secrets in environment variables, not config files: + +```yaml +# config.yaml - reference environment variables +gateway: + auth: + token: ${GATEWAY_TOKEN} + +models: + anthropic: + apiKey: ${ANTHROPIC_API_KEY} +``` + +### Credential Storage + +Channel credentials are stored with restricted permissions: +- Location: `~/.clawdbot/credentials/` +- Permissions: `0600` (owner read/write only) + +Run security checks: + +```bash +clawdbot doctor --check credentials +``` + +## Network Security + +### Bind Address + +Restrict which interfaces accept connections: + +```yaml +gateway: + # Localhost only (most secure) + bind: "127.0.0.1" + + # All interfaces (for remote access) + bind: "0.0.0.0" +``` + +### Firewall Rules + +Minimal required firewall rules: + +```bash +# Allow gateway port from trusted networks +iptables -A INPUT -p tcp --dport 18789 -s 10.0.0.0/8 -j ACCEPT +iptables -A INPUT -p tcp --dport 18789 -j DROP + +# Allow outbound HTTPS for APIs +iptables -A OUTPUT -p tcp --dport 443 -j ACCEPT +``` + +## Security Checklist + +### Pre-Production + +- [ ] Enable gateway authentication (`gateway.auth.mode`) +- [ ] Configure RBAC with least-privilege roles +- [ ] Set up TLS termination +- [ ] Configure trusted proxies +- [ ] Enable audit logging +- [ ] Review rate limiting settings +- [ ] Remove default/test credentials + +### Ongoing + +- [ ] Rotate gateway tokens quarterly +- [ ] Review audit logs for anomalies +- [ ] Update to latest Clawdbot version +- [ ] Review RBAC assignments when team changes +- [ ] Test backup and recovery procedures + +## Next Steps + +- [Enterprise Deployment](/enterprise/deployment) - Deployment patterns +- [Observability](/enterprise/observability) - Monitoring and alerting +- [Gateway Security Reference](/gateway/security) - Detailed security documentation diff --git a/load-tests/config.ts b/load-tests/config.ts new file mode 100644 index 000000000..db59bfdd8 --- /dev/null +++ b/load-tests/config.ts @@ -0,0 +1,282 @@ +/** + * Load Test Configuration + * + * Configuration types and defaults for gateway load testing. + */ + +export type LoadTestScenario = "connections" | "chat" | "auth-stress"; + +export type LoadTestConfig = { + /** Gateway URL to test against */ + gatewayUrl: string; + + /** Authentication token */ + token?: string; + + /** Test scenario to run */ + scenario: LoadTestScenario; + + /** Test duration in seconds */ + durationSeconds: number; + + /** Number of virtual users / concurrent connections */ + concurrency: number; + + /** Ramp-up time in seconds (0 = instant) */ + rampUpSeconds: number; + + /** Requests per second per virtual user (for throughput tests) */ + rpsPerUser: number; + + /** Enable verbose logging */ + verbose: boolean; + + /** Output format for results */ + outputFormat: "console" | "json"; +}; + +export const DEFAULT_CONFIG: LoadTestConfig = { + gatewayUrl: "ws://127.0.0.1:18789", + scenario: "connections", + durationSeconds: 60, + concurrency: 50, + rampUpSeconds: 10, + rpsPerUser: 1, + verbose: false, + outputFormat: "console", +}; + +export type LoadTestMetrics = { + scenario: LoadTestScenario; + startTime: number; + endTime: number; + durationMs: number; + + // Connection metrics + connectionsAttempted: number; + connectionsSucceeded: number; + connectionsFailed: number; + connectionsPeak: number; + + // Request metrics + requestsTotal: number; + requestsSucceeded: number; + requestsFailed: number; + requestsTimedOut: number; + + // Latency metrics (in milliseconds) + latencyP50: number; + latencyP95: number; + latencyP99: number; + latencyMin: number; + latencyMax: number; + latencyMean: number; + + // Rate limiting + rateLimitHits: number; + authFailures: number; + + // Errors + errors: ErrorSummary[]; +}; + +export type ErrorSummary = { + message: string; + count: number; + firstSeen: number; + lastSeen: number; +}; + +/** + * Calculate percentile from sorted array of values. + */ +export function percentile(sortedValues: number[], p: number): number { + if (sortedValues.length === 0) return 0; + const index = Math.ceil((p / 100) * sortedValues.length) - 1; + return sortedValues[Math.max(0, Math.min(index, sortedValues.length - 1))] ?? 0; +} + +/** + * Calculate mean from array of values. + */ +export function mean(values: number[]): number { + if (values.length === 0) return 0; + return values.reduce((sum, v) => sum + v, 0) / values.length; +} + +/** + * Aggregate error messages into summaries. + */ +export function aggregateErrors( + errors: Array<{ message: string; timestamp: number }>, +): ErrorSummary[] { + const byMessage = new Map(); + + for (const err of errors) { + const existing = byMessage.get(err.message); + if (existing) { + existing.count += 1; + existing.lastSeen = err.timestamp; + } else { + byMessage.set(err.message, { + message: err.message, + count: 1, + firstSeen: err.timestamp, + lastSeen: err.timestamp, + }); + } + } + + return Array.from(byMessage.values()).sort((a, b) => b.count - a.count); +} + +/** + * Format metrics for console output. + */ +export function formatMetricsConsole(metrics: LoadTestMetrics): string { + const lines: string[] = [ + "", + "═══════════════════════════════════════════════════════════════════", + ` Load Test Results: ${metrics.scenario}`, + "═══════════════════════════════════════════════════════════════════", + "", + ` Duration: ${(metrics.durationMs / 1000).toFixed(1)}s`, + "", + " Connections", + " ───────────────────────────────────────────────────────────────────", + ` Attempted: ${metrics.connectionsAttempted}`, + ` Succeeded: ${metrics.connectionsSucceeded}`, + ` Failed: ${metrics.connectionsFailed}`, + ` Peak: ${metrics.connectionsPeak}`, + "", + " Requests", + " ───────────────────────────────────────────────────────────────────", + ` Total: ${metrics.requestsTotal}`, + ` Succeeded: ${metrics.requestsSucceeded}`, + ` Failed: ${metrics.requestsFailed}`, + ` Timed Out: ${metrics.requestsTimedOut}`, + ` Rate: ${(metrics.requestsTotal / (metrics.durationMs / 1000)).toFixed(1)} req/s`, + "", + " Latency (ms)", + " ───────────────────────────────────────────────────────────────────", + ` Min: ${metrics.latencyMin.toFixed(1)}`, + ` Mean: ${metrics.latencyMean.toFixed(1)}`, + ` P50: ${metrics.latencyP50.toFixed(1)}`, + ` P95: ${metrics.latencyP95.toFixed(1)}`, + ` P99: ${metrics.latencyP99.toFixed(1)}`, + ` Max: ${metrics.latencyMax.toFixed(1)}`, + "", + " Rate Limiting", + " ───────────────────────────────────────────────────────────────────", + ` Rate Limit Hits: ${metrics.rateLimitHits}`, + ` Auth Failures: ${metrics.authFailures}`, + "", + ]; + + if (metrics.errors.length > 0) { + lines.push(" Errors (Top 5)"); + lines.push(" ───────────────────────────────────────────────────────────────────"); + for (const err of metrics.errors.slice(0, 5)) { + lines.push(` [${err.count}x] ${err.message.slice(0, 60)}`); + } + lines.push(""); + } + + lines.push("═══════════════════════════════════════════════════════════════════"); + lines.push(""); + + return lines.join("\n"); +} + +/** + * Parse command-line arguments into config. + */ +export function parseArgs(args: string[]): Partial { + const config: Partial = {}; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + const nextArg = args[i + 1]; + + switch (arg) { + case "--url": + case "-u": + config.gatewayUrl = nextArg; + i++; + break; + case "--token": + case "-t": + config.token = nextArg; + i++; + break; + case "--scenario": + case "-s": + config.scenario = nextArg as LoadTestScenario; + i++; + break; + case "--duration": + case "-d": + config.durationSeconds = parseInt(nextArg ?? "60", 10); + i++; + break; + case "--concurrency": + case "-c": + config.concurrency = parseInt(nextArg ?? "50", 10); + i++; + break; + case "--ramp-up": + case "-r": + config.rampUpSeconds = parseInt(nextArg ?? "10", 10); + i++; + break; + case "--rps": + config.rpsPerUser = parseFloat(nextArg ?? "1"); + i++; + break; + case "--verbose": + case "-v": + config.verbose = true; + break; + case "--json": + config.outputFormat = "json"; + break; + case "--help": + case "-h": + printUsage(); + process.exit(0); + } + } + + return config; +} + +function printUsage(): void { + console.log(` +Gateway Load Test + +Usage: + bun load-tests/run.ts [options] + +Options: + -u, --url Gateway WebSocket URL (default: ws://127.0.0.1:18789) + -t, --token Authentication token + -s, --scenario Test scenario: connections, chat, auth-stress (default: connections) + -d, --duration Test duration in seconds (default: 60) + -c, --concurrency Number of concurrent connections (default: 50) + -r, --ramp-up Ramp-up time in seconds (default: 10) + --rps Requests per second per user (default: 1) + -v, --verbose Enable verbose output + --json Output results as JSON + -h, --help Show this help + +Scenarios: + connections WebSocket connection stress test + chat Chat message throughput test + auth-stress Authentication rate limit verification + +Examples: + bun load-tests/run.ts --scenario connections --concurrency 100 --duration 30 + bun load-tests/run.ts --scenario chat --concurrency 20 --rps 5 --duration 60 + bun load-tests/run.ts --scenario auth-stress --concurrency 10 --duration 30 +`); +} diff --git a/load-tests/gateway-auth.ts b/load-tests/gateway-auth.ts new file mode 100644 index 000000000..fccccf7b2 --- /dev/null +++ b/load-tests/gateway-auth.ts @@ -0,0 +1,333 @@ +/** + * Gateway Authentication Stress Test + * + * Tests the gateway's rate limiting and auth backoff mechanisms. + * Verifies that the gateway correctly limits failed auth attempts + * and applies exponential backoff. + */ + +import { randomUUID } from "node:crypto"; +import { WebSocket } from "ws"; +import type { LoadTestConfig, LoadTestMetrics } from "./config.js"; +import { aggregateErrors, mean, percentile } from "./config.js"; + +type AuthAttempt = { + startMs: number; + latencyMs: number; + success: boolean; + error?: string; + rateLimited: boolean; +}; + +/** + * Run the authentication stress test. + */ +export async function runAuthStressTest(config: LoadTestConfig): Promise { + const startTime = Date.now(); + const attempts: AuthAttempt[] = []; + const errors: Array<{ message: string; timestamp: number }> = []; + + let connectionsAttempted = 0; + let connectionsSucceeded = 0; + let connectionsFailed = 0; + let rateLimitHits = 0; + let authFailures = 0; + + const log = config.verbose + ? (msg: string) => console.log(`[${new Date().toISOString()}] ${msg}`) + : () => {}; + + log(`Starting auth stress test: ${config.concurrency} concurrent attempts`); + + // Phase 1: Rapid auth attempts with invalid tokens + log("Phase 1: Testing invalid token rejection..."); + const invalidTokenAttempts = await runAuthAttempts({ + config, + count: 20, + concurrency: 5, + token: "invalid-token-" + randomUUID(), + log, + }); + + for (const attempt of invalidTokenAttempts) { + attempts.push(attempt); + connectionsAttempted++; + if (attempt.success) { + connectionsSucceeded++; + } else { + connectionsFailed++; + authFailures++; + if (attempt.error) { + errors.push({ message: attempt.error, timestamp: attempt.startMs }); + } + } + if (attempt.rateLimited) { + rateLimitHits++; + } + } + + log(`Phase 1 complete: ${authFailures} auth failures, ${rateLimitHits} rate limits`); + + // Phase 2: Verify backoff is applied + log("Phase 2: Verifying rate limit backoff..."); + await sleep(1000); + + const backoffAttempts = await runAuthAttempts({ + config, + count: 10, + concurrency: 1, + token: "invalid-token-" + randomUUID(), + log, + }); + + for (const attempt of backoffAttempts) { + attempts.push(attempt); + connectionsAttempted++; + if (attempt.success) { + connectionsSucceeded++; + } else { + connectionsFailed++; + authFailures++; + if (attempt.error) { + errors.push({ message: attempt.error, timestamp: attempt.startMs }); + } + } + if (attempt.rateLimited) { + rateLimitHits++; + } + } + + log(`Phase 2 complete: ${rateLimitHits} total rate limits observed`); + + // Phase 3: Valid token should still work (if provided) + if (config.token) { + log("Phase 3: Verifying valid token still works..."); + await sleep(2000); + + const validTokenAttempts = await runAuthAttempts({ + config, + count: 5, + concurrency: 1, + token: config.token, + log, + }); + + for (const attempt of validTokenAttempts) { + attempts.push(attempt); + connectionsAttempted++; + if (attempt.success) { + connectionsSucceeded++; + log("Valid token accepted"); + } else { + connectionsFailed++; + if (attempt.error) { + errors.push({ message: attempt.error, timestamp: attempt.startMs }); + } + if (attempt.rateLimited) { + rateLimitHits++; + log("Valid token rate limited (expected during cooldown)"); + } else { + authFailures++; + } + } + } + } + + const endTime = Date.now(); + + // Calculate latency metrics + const latencies = attempts.map((a) => a.latencyMs).sort((a, b) => a - b); + + const metrics: LoadTestMetrics = { + scenario: "auth-stress", + startTime, + endTime, + durationMs: endTime - startTime, + + connectionsAttempted, + connectionsSucceeded, + connectionsFailed, + connectionsPeak: config.concurrency, + + requestsTotal: attempts.length, + requestsSucceeded: attempts.filter((a) => a.success).length, + requestsFailed: attempts.filter((a) => !a.success).length, + requestsTimedOut: attempts.filter((a) => a.error?.includes("timed out")).length, + + latencyP50: percentile(latencies, 50), + latencyP95: percentile(latencies, 95), + latencyP99: percentile(latencies, 99), + latencyMin: latencies[0] ?? 0, + latencyMax: latencies[latencies.length - 1] ?? 0, + latencyMean: mean(latencies), + + rateLimitHits, + authFailures, + errors: aggregateErrors(errors), + }; + + return metrics; +} + +async function runAuthAttempts(params: { + config: LoadTestConfig; + count: number; + concurrency: number; + token: string; + log: (msg: string) => void; +}): Promise { + const { config, count, concurrency, token, log } = params; + const attempts: AuthAttempt[] = []; + const pending: Promise[] = []; + + for (let i = 0; i < count; i++) { + const attemptPromise = runSingleAuthAttempt(config, token, log); + pending.push(attemptPromise); + + if (pending.length >= concurrency) { + const completed = await Promise.race(pending.map((p, idx) => p.then((r) => ({ idx, result: r })))); + void pending.splice(completed.idx, 1); + attempts.push(completed.result); + } + } + + // Wait for remaining + const remaining = await Promise.all(pending); + attempts.push(...remaining); + + return attempts; +} + +async function runSingleAuthAttempt( + config: LoadTestConfig, + token: string, + _log: (msg: string) => void, +): Promise { + const startMs = Date.now(); + let ws: WebSocket | null = null; + + try { + ws = new WebSocket(config.gatewayUrl, { + maxPayload: 10 * 1024 * 1024, + }); + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Connection timeout")); + }, 10000); + + ws?.once("open", () => { + clearTimeout(timeout); + resolve(); + }); + + ws?.once("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + }); + + // Send connect request + const result = await sendAuthRequest(ws, token); + const latencyMs = Date.now() - startMs; + + if (result.success) { + return { startMs, latencyMs, success: true, rateLimited: false }; + } else { + const isRateLimited = + result.error?.includes("rate") || + result.error?.includes("429") || + result.error?.includes("backoff") || + result.error?.includes("too many"); + + return { + startMs, + latencyMs, + success: false, + error: result.error, + rateLimited: isRateLimited, + }; + } + } catch (err) { + const latencyMs = Date.now() - startMs; + const errMsg = err instanceof Error ? err.message : String(err); + const isRateLimited = + errMsg.includes("rate") || + errMsg.includes("429") || + errMsg.includes("backoff") || + errMsg.includes("too many"); + + return { + startMs, + latencyMs, + success: false, + error: errMsg, + rateLimited: isRateLimited, + }; + } finally { + ws?.close(); + } +} + +async function sendAuthRequest( + ws: WebSocket, + token: string, +): Promise<{ success: boolean; error?: string }> { + return new Promise((resolve) => { + if (ws.readyState !== WebSocket.OPEN) { + resolve({ success: false, error: "WebSocket not open" }); + return; + } + + const requestId = randomUUID(); + const timeout = setTimeout(() => { + ws.off("message", handler); + resolve({ success: false, error: "Auth request timed out" }); + }, 10000); + + const handler = (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.type === "res" && msg.id === requestId) { + clearTimeout(timeout); + ws.off("message", handler); + if (msg.ok) { + resolve({ success: true }); + } else { + resolve({ success: false, error: msg.error?.message ?? "Auth failed" }); + } + } + } catch { + // Ignore parse errors + } + }; + + ws.on("message", handler); + + ws.send( + JSON.stringify({ + type: "req", + id: requestId, + method: "connect", + params: { + minProtocol: 1, + maxProtocol: 1, + client: { + id: "load-test-auth", + version: "1.0.0", + platform: "test", + mode: "test", + instanceId: randomUUID(), + }, + caps: [], + role: "operator", + auth: { token }, + }, + }), + ); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/load-tests/gateway-chat.ts b/load-tests/gateway-chat.ts new file mode 100644 index 000000000..22113e874 --- /dev/null +++ b/load-tests/gateway-chat.ts @@ -0,0 +1,324 @@ +/** + * Gateway Chat Message Throughput Test + * + * Tests the gateway's ability to handle sustained message throughput. + * Measures message processing latency and throughput under load. + */ + +import { randomUUID } from "node:crypto"; +import { WebSocket } from "ws"; +import type { LoadTestConfig, LoadTestMetrics } from "./config.js"; +import { aggregateErrors, mean, percentile } from "./config.js"; + +type VirtualUser = { + id: string; + ws: WebSocket | null; + connected: boolean; + requestsTotal: number; + requestsSucceeded: number; + requestsFailed: number; + requestsTimedOut: number; + latencies: number[]; + errors: Array<{ message: string; timestamp: number }>; +}; + +/** + * Run the chat message throughput test. + */ +export async function runChatTest(config: LoadTestConfig): Promise { + const startTime = Date.now(); + const users: VirtualUser[] = []; + let rateLimitHits = 0; + let authFailures = 0; + + const log = config.verbose + ? (msg: string) => console.log(`[${new Date().toISOString()}] ${msg}`) + : () => {}; + + log(`Starting chat test: ${config.concurrency} users, ${config.rpsPerUser} rps/user`); + + // Create and connect virtual users + const rampUpDelayMs = + config.rampUpSeconds > 0 ? (config.rampUpSeconds * 1000) / config.concurrency : 0; + + for (let i = 0; i < config.concurrency; i++) { + const user = await createAndConnectUser(config, i, log); + users.push(user); + + if (user.connected) { + log(`User ${i} connected`); + } else { + log(`User ${i} failed to connect`); + for (const err of user.errors) { + if (err.message.includes("rate") || err.message.includes("429")) { + rateLimitHits++; + } + if (err.message.includes("auth") || err.message.includes("unauthorized")) { + authFailures++; + } + } + } + + if (rampUpDelayMs > 0 && i < config.concurrency - 1) { + await sleep(rampUpDelayMs); + } + } + + const connectedUsers = users.filter((u) => u.connected); + log(`${connectedUsers.length}/${config.concurrency} users connected, starting message load`); + + // Start sending messages at the configured rate + const testDurationMs = config.durationSeconds * 1000; + const intervalMs = 1000 / config.rpsPerUser; + const endTime = startTime + testDurationMs + (config.rampUpSeconds * 1000); + + const messageLoops = connectedUsers.map((user) => + runMessageLoop(user, config, intervalMs, endTime, log), + ); + + // Wait for message loops to complete + await Promise.all(messageLoops); + + // Close all connections + log("Closing connections..."); + for (const user of users) { + user.ws?.close(); + } + + const finalTime = Date.now(); + + // Aggregate metrics + const allLatencies = users.flatMap((u) => u.latencies).sort((a, b) => a - b); + const allErrors = users.flatMap((u) => u.errors); + + // Count rate limit hits and auth failures from errors + for (const err of allErrors) { + if (err.message.includes("rate") || err.message.includes("429")) { + rateLimitHits++; + } + if (err.message.includes("auth") || err.message.includes("unauthorized")) { + authFailures++; + } + } + + const metrics: LoadTestMetrics = { + scenario: "chat", + startTime, + endTime: finalTime, + durationMs: finalTime - startTime, + + connectionsAttempted: config.concurrency, + connectionsSucceeded: connectedUsers.length, + connectionsFailed: config.concurrency - connectedUsers.length, + connectionsPeak: connectedUsers.length, + + requestsTotal: users.reduce((sum, u) => sum + u.requestsTotal, 0), + requestsSucceeded: users.reduce((sum, u) => sum + u.requestsSucceeded, 0), + requestsFailed: users.reduce((sum, u) => sum + u.requestsFailed, 0), + requestsTimedOut: users.reduce((sum, u) => sum + u.requestsTimedOut, 0), + + latencyP50: percentile(allLatencies, 50), + latencyP95: percentile(allLatencies, 95), + latencyP99: percentile(allLatencies, 99), + latencyMin: allLatencies[0] ?? 0, + latencyMax: allLatencies[allLatencies.length - 1] ?? 0, + latencyMean: mean(allLatencies), + + rateLimitHits, + authFailures, + errors: aggregateErrors(allErrors), + }; + + return metrics; +} + +async function createAndConnectUser( + config: LoadTestConfig, + index: number, + log: (msg: string) => void, +): Promise { + const user: VirtualUser = { + id: randomUUID(), + ws: null, + connected: false, + requestsTotal: 0, + requestsSucceeded: 0, + requestsFailed: 0, + requestsTimedOut: 0, + latencies: [], + errors: [], + }; + + try { + user.ws = new WebSocket(config.gatewayUrl, { + maxPayload: 10 * 1024 * 1024, + }); + + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Connection timeout")); + }, 10000); + + user.ws?.once("open", () => { + clearTimeout(timeout); + resolve(); + }); + + user.ws?.once("error", (err) => { + clearTimeout(timeout); + reject(err); + }); + }); + + // Send connect request + await sendConnectRequest(user, config); + user.connected = true; + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + user.errors.push({ message: errMsg, timestamp: Date.now() }); + log(`User ${index} connection error: ${errMsg}`); + } + + return user; +} + +async function sendConnectRequest(user: VirtualUser, config: LoadTestConfig): Promise { + return new Promise((resolve, reject) => { + if (!user.ws || user.ws.readyState !== WebSocket.OPEN) { + reject(new Error("WebSocket not open")); + return; + } + + const requestId = randomUUID(); + const timeout = setTimeout(() => { + user.ws?.off("message", handler); + reject(new Error("Connect request timed out")); + }, 10000); + + const handler = (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.type === "res" && msg.id === requestId) { + clearTimeout(timeout); + user.ws?.off("message", handler); + if (msg.ok) { + resolve(); + } else { + reject(new Error(msg.error?.message ?? "Connect failed")); + } + } + } catch { + // Ignore parse errors + } + }; + + user.ws.on("message", handler); + + user.ws.send( + JSON.stringify({ + type: "req", + id: requestId, + method: "connect", + params: { + minProtocol: 1, + maxProtocol: 1, + client: { + id: "load-test-chat", + version: "1.0.0", + platform: "test", + mode: "test", + instanceId: user.id, + }, + caps: [], + role: "operator", + auth: config.token ? { token: config.token } : undefined, + }, + }), + ); + }); +} + +async function runMessageLoop( + user: VirtualUser, + config: LoadTestConfig, + intervalMs: number, + endTime: number, + _log: (msg: string) => void, +): Promise { + while (Date.now() < endTime && user.connected && user.ws?.readyState === WebSocket.OPEN) { + const startMs = Date.now(); + user.requestsTotal++; + + try { + await sendChatMessage(user, config); + const latency = Date.now() - startMs; + user.requestsSucceeded++; + user.latencies.push(latency); + } catch (err) { + const errMsg = err instanceof Error ? err.message : String(err); + user.errors.push({ message: errMsg, timestamp: Date.now() }); + + if (errMsg.includes("timed out")) { + user.requestsTimedOut++; + } else { + user.requestsFailed++; + } + } + + // Wait for next interval + const elapsed = Date.now() - startMs; + const waitMs = Math.max(0, intervalMs - elapsed); + if (waitMs > 0) { + await sleep(waitMs); + } + } +} + +async function sendChatMessage(user: VirtualUser, _config: LoadTestConfig): Promise { + return new Promise((resolve, reject) => { + if (!user.ws || user.ws.readyState !== WebSocket.OPEN) { + reject(new Error("WebSocket not open")); + return; + } + + const requestId = randomUUID(); + const timeout = setTimeout(() => { + user.ws?.off("message", handler); + reject(new Error("Chat message timed out")); + }, 30000); + + const handler = (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.type === "res" && msg.id === requestId) { + clearTimeout(timeout); + user.ws?.off("message", handler); + if (msg.ok) { + resolve(); + } else { + reject(new Error(msg.error?.message ?? "Chat message failed")); + } + } + } catch { + // Ignore parse errors + } + }; + + user.ws.on("message", handler); + + // Send a simple echo-style message via the sessions.list RPC + // (a lightweight operation that exercises the request path) + user.ws.send( + JSON.stringify({ + type: "req", + id: requestId, + method: "sessions.list", + params: {}, + }), + ); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/load-tests/gateway-ws.ts b/load-tests/gateway-ws.ts new file mode 100644 index 000000000..9b32f5c36 --- /dev/null +++ b/load-tests/gateway-ws.ts @@ -0,0 +1,306 @@ +/** + * Gateway WebSocket Connection Stress Test + * + * Tests the gateway's ability to handle many concurrent WebSocket connections. + * Measures connection establishment time, stability, and resource usage. + */ + +import { randomUUID } from "node:crypto"; +import { WebSocket } from "ws"; +import type { LoadTestConfig, LoadTestMetrics } from "./config.js"; +import { aggregateErrors, mean, percentile } from "./config.js"; + +type ConnectionState = { + id: string; + ws: WebSocket | null; + connected: boolean; + connectStartMs: number; + connectLatencyMs: number | null; + error: string | null; + requestLatencies: number[]; +}; + +type RawError = { + message: string; + timestamp: number; +}; + +/** + * Run the WebSocket connection stress test. + */ +export async function runConnectionsTest(config: LoadTestConfig): Promise { + const startTime = Date.now(); + const connections: ConnectionState[] = []; + const errors: RawError[] = []; + + let connectionsAttempted = 0; + let connectionsSucceeded = 0; + let connectionsFailed = 0; + let connectionsPeak = 0; + let rateLimitHits = 0; + let authFailures = 0; + + const log = config.verbose + ? (msg: string) => console.log(`[${new Date().toISOString()}] ${msg}`) + : () => {}; + + log(`Starting connections test: ${config.concurrency} connections over ${config.rampUpSeconds}s`); + + // Calculate ramp-up delay between connections + const rampUpDelayMs = + config.rampUpSeconds > 0 ? (config.rampUpSeconds * 1000) / config.concurrency : 0; + + // Create connections with ramp-up + for (let i = 0; i < config.concurrency; i++) { + const conn = createConnection(config, i); + connections.push(conn); + connectionsAttempted++; + + conn.ws?.on("open", () => { + conn.connectLatencyMs = Date.now() - conn.connectStartMs; + log(`Connection ${i} opened in ${conn.connectLatencyMs}ms`); + + // Send connect request + sendConnectRequest(conn, config) + .then(() => { + conn.connected = true; + connectionsSucceeded++; + const activeCount = connections.filter((c) => c.connected).length; + connectionsPeak = Math.max(connectionsPeak, activeCount); + log(`Connection ${i} authenticated (active: ${activeCount})`); + }) + .catch((err) => { + const errMsg = err instanceof Error ? err.message : String(err); + conn.error = errMsg; + connectionsFailed++; + errors.push({ message: errMsg, timestamp: Date.now() }); + + if (errMsg.includes("rate") || errMsg.includes("429")) { + rateLimitHits++; + } + if (errMsg.includes("auth") || errMsg.includes("unauthorized")) { + authFailures++; + } + + log(`Connection ${i} auth failed: ${errMsg}`); + }); + }); + + conn.ws?.on("error", (err) => { + const errMsg = err instanceof Error ? err.message : String(err); + if (!conn.error) { + conn.error = errMsg; + connectionsFailed++; + errors.push({ message: errMsg, timestamp: Date.now() }); + } + log(`Connection ${i} error: ${errMsg}`); + }); + + conn.ws?.on("close", (code, reason) => { + if (conn.connected) { + conn.connected = false; + log(`Connection ${i} closed: ${code} ${reason.toString()}`); + } + }); + + if (rampUpDelayMs > 0 && i < config.concurrency - 1) { + await sleep(rampUpDelayMs); + } + } + + // Wait for all connections to establish + await sleep(3000); + + // Hold connections open for the test duration + const holdDurationMs = config.durationSeconds * 1000 - (Date.now() - startTime); + if (holdDurationMs > 0) { + log(`Holding ${connectionsPeak} connections for ${(holdDurationMs / 1000).toFixed(1)}s`); + + // Periodically send ping requests to verify connections are alive + const pingInterval = setInterval(() => { + for (const conn of connections) { + if (conn.connected && conn.ws?.readyState === WebSocket.OPEN) { + const pingStart = Date.now(); + sendPingRequest(conn) + .then(() => { + conn.requestLatencies.push(Date.now() - pingStart); + }) + .catch(() => { + // Connection may have dropped + }); + } + } + }, 5000); + + await sleep(holdDurationMs); + clearInterval(pingInterval); + } + + // Close all connections + log("Closing connections..."); + for (const conn of connections) { + conn.ws?.close(); + } + + const endTime = Date.now(); + + // Calculate latency metrics + const connectLatencies = connections + .map((c) => c.connectLatencyMs) + .filter((l): l is number => l !== null) + .sort((a, b) => a - b); + + const allRequestLatencies = connections + .flatMap((c) => c.requestLatencies) + .sort((a, b) => a - b); + + const metrics: LoadTestMetrics = { + scenario: "connections", + startTime, + endTime, + durationMs: endTime - startTime, + + connectionsAttempted, + connectionsSucceeded, + connectionsFailed, + connectionsPeak, + + requestsTotal: allRequestLatencies.length, + requestsSucceeded: allRequestLatencies.length, + requestsFailed: 0, + requestsTimedOut: 0, + + latencyP50: percentile(connectLatencies, 50), + latencyP95: percentile(connectLatencies, 95), + latencyP99: percentile(connectLatencies, 99), + latencyMin: connectLatencies[0] ?? 0, + latencyMax: connectLatencies[connectLatencies.length - 1] ?? 0, + latencyMean: mean(connectLatencies), + + rateLimitHits, + authFailures, + errors: aggregateErrors(errors), + }; + + return metrics; +} + +function createConnection(config: LoadTestConfig, _index: number): ConnectionState { + const id = randomUUID(); + const ws = new WebSocket(config.gatewayUrl, { + maxPayload: 10 * 1024 * 1024, + }); + + return { + id, + ws, + connected: false, + connectStartMs: Date.now(), + connectLatencyMs: null, + error: null, + requestLatencies: [], + }; +} + +async function sendConnectRequest(conn: ConnectionState, config: LoadTestConfig): Promise { + return new Promise((resolve, reject) => { + if (!conn.ws || conn.ws.readyState !== WebSocket.OPEN) { + reject(new Error("WebSocket not open")); + return; + } + + const requestId = randomUUID(); + const timeout = setTimeout(() => { + conn.ws?.off("message", handler); + reject(new Error("Connect request timed out")); + }, 10000); + + const handler = (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.type === "res" && msg.id === requestId) { + clearTimeout(timeout); + conn.ws?.off("message", handler); + if (msg.ok) { + resolve(); + } else { + reject(new Error(msg.error?.message ?? "Connect failed")); + } + } + } catch { + // Ignore parse errors + } + }; + + conn.ws.on("message", handler); + + conn.ws.send( + JSON.stringify({ + type: "req", + id: requestId, + method: "connect", + params: { + minProtocol: 1, + maxProtocol: 1, + client: { + id: "load-test", + version: "1.0.0", + platform: "test", + mode: "test", + instanceId: conn.id, + }, + caps: [], + role: "operator", + auth: config.token ? { token: config.token } : undefined, + }, + }), + ); + }); +} + +async function sendPingRequest(conn: ConnectionState): Promise { + return new Promise((resolve, reject) => { + if (!conn.ws || conn.ws.readyState !== WebSocket.OPEN) { + reject(new Error("WebSocket not open")); + return; + } + + const requestId = randomUUID(); + const timeout = setTimeout(() => { + conn.ws?.off("message", handler); + reject(new Error("Ping request timed out")); + }, 5000); + + const handler = (data: Buffer) => { + try { + const msg = JSON.parse(data.toString()); + if (msg.type === "res" && msg.id === requestId) { + clearTimeout(timeout); + conn.ws?.off("message", handler); + if (msg.ok) { + resolve(); + } else { + reject(new Error(msg.error?.message ?? "Ping failed")); + } + } + } catch { + // Ignore parse errors + } + }; + + conn.ws.on("message", handler); + + conn.ws.send( + JSON.stringify({ + type: "req", + id: requestId, + method: "ping", + params: {}, + }), + ); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/load-tests/run.ts b/load-tests/run.ts new file mode 100755 index 000000000..49386cffe --- /dev/null +++ b/load-tests/run.ts @@ -0,0 +1,102 @@ +#!/usr/bin/env bun +/** + * Gateway Load Test Runner + * + * Entry point for running gateway load tests. + * + * Usage: + * bun load-tests/run.ts --scenario connections --concurrency 100 --duration 30 + * bun load-tests/run.ts --scenario chat --concurrency 20 --rps 5 --duration 60 + * bun load-tests/run.ts --scenario auth-stress --concurrency 10 --duration 30 + */ + +import { DEFAULT_CONFIG, formatMetricsConsole, parseArgs } from "./config.js"; +import type { LoadTestConfig, LoadTestMetrics } from "./config.js"; +import { runConnectionsTest } from "./gateway-ws.js"; +import { runChatTest } from "./gateway-chat.js"; +import { runAuthStressTest } from "./gateway-auth.js"; + +async function main(): Promise { + const args = process.argv.slice(2); + const parsedArgs = parseArgs(args); + + const config: LoadTestConfig = { + ...DEFAULT_CONFIG, + ...parsedArgs, + }; + + // Validate config + if (!config.gatewayUrl) { + console.error("Error: Gateway URL is required (--url or -u)"); + process.exit(1); + } + + if (!["connections", "chat", "auth-stress"].includes(config.scenario)) { + console.error(`Error: Invalid scenario "${config.scenario}". Use: connections, chat, auth-stress`); + process.exit(1); + } + + console.log(""); + console.log("╔══════════════════════════════════════════════════════════════════╗"); + console.log("║ Clawdbot Gateway Load Test ║"); + console.log("╚══════════════════════════════════════════════════════════════════╝"); + console.log(""); + console.log(` Scenario: ${config.scenario}`); + console.log(` Gateway: ${config.gatewayUrl}`); + console.log(` Concurrency: ${config.concurrency}`); + console.log(` Duration: ${config.durationSeconds}s`); + console.log(` Ramp-up: ${config.rampUpSeconds}s`); + if (config.scenario === "chat") { + console.log(` RPS/user: ${config.rpsPerUser}`); + } + console.log(` Auth: ${config.token ? "token provided" : "none"}`); + console.log(""); + + let metrics: LoadTestMetrics; + + try { + switch (config.scenario) { + case "connections": + metrics = await runConnectionsTest(config); + break; + case "chat": + metrics = await runChatTest(config); + break; + case "auth-stress": + metrics = await runAuthStressTest(config); + break; + } + } catch (err) { + console.error(`\nLoad test failed: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); + } + + // Output results + if (config.outputFormat === "json") { + console.log(JSON.stringify(metrics, null, 2)); + } else { + console.log(formatMetricsConsole(metrics)); + } + + // Exit with error code if significant failures + const failureRate = metrics.requestsTotal > 0 + ? metrics.requestsFailed / metrics.requestsTotal + : 0; + + if (failureRate > 0.1) { + console.log("⚠️ Warning: Failure rate > 10%"); + process.exit(1); + } + + if (metrics.connectionsFailed > 0 && metrics.connectionsSucceeded === 0) { + console.log("⚠️ Warning: All connections failed"); + process.exit(1); + } + + console.log("✓ Load test completed successfully"); +} + +main().catch((err) => { + console.error(`Fatal error: ${err instanceof Error ? err.message : String(err)}`); + process.exit(1); +}); diff --git a/package.json b/package.json index 04322f3af..6516f834a 100644 --- a/package.json +++ b/package.json @@ -197,6 +197,7 @@ "osc-progress": "^0.3.0", "pdfjs-dist": "^5.4.530", "playwright-core": "1.58.0", + "prom-client": "^15.1.3", "proper-lockfile": "^4.1.2", "qrcode-terminal": "^0.12.0", "sharp": "^0.34.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9c0f99928..91b16a185 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -142,6 +142,9 @@ importers: playwright-core: specifier: 1.58.0 version: 1.58.0 + prom-client: + specifier: ^15.1.3 + version: 15.1.3 proper-lockfile: specifier: ^4.1.2 version: 4.1.2 @@ -3101,6 +3104,9 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} + bintrees@1.0.2: + resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} + bluebird@3.7.2: resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==} @@ -4761,6 +4767,10 @@ packages: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} + prom-client@15.1.3: + resolution: {integrity: sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==} + engines: {node: ^16 || ^18 || >=20} + proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} @@ -5194,6 +5204,9 @@ packages: resolution: {integrity: sha512-AN04xbWGrSTDmVwlI4/GTlIIwMFk/XEv7uL8aa57zuvRy6s4hdBed+lVq2fAZ89XDa7Us3ANXcE3Tvqvja1kTA==} engines: {node: '>=18'} + tdigest@0.1.2: + resolution: {integrity: sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -8958,6 +8971,8 @@ snapshots: binary-extensions@2.3.0: {} + bintrees@1.0.2: {} + bluebird@3.7.2: {} body-parser@1.20.4: @@ -10848,6 +10863,11 @@ snapshots: process@0.11.10: {} + prom-client@15.1.3: + dependencies: + '@opentelemetry/api': 1.9.0 + tdigest: 0.1.2 + proper-lockfile@4.1.2: dependencies: graceful-fs: 4.2.11 @@ -11478,6 +11498,10 @@ snapshots: minizlib: 3.1.0 yallist: 5.0.0 + tdigest@0.1.2: + dependencies: + bintrees: 1.0.2 + thenify-all@1.6.0: dependencies: thenify: 3.3.1 diff --git a/scripts/load-test.sh b/scripts/load-test.sh new file mode 100755 index 000000000..91e4da130 --- /dev/null +++ b/scripts/load-test.sh @@ -0,0 +1,138 @@ +#!/bin/bash +# +# Gateway Load Test Runner +# +# Usage: +# ./scripts/load-test.sh [options] +# +# Examples: +# ./scripts/load-test.sh --scenario connections --concurrency 100 +# ./scripts/load-test.sh --scenario chat --rps 5 +# ./scripts/load-test.sh --scenario auth-stress +# +# Environment variables: +# GATEWAY_URL Gateway WebSocket URL (default: ws://127.0.0.1:18789) +# GATEWAY_TOKEN Authentication token +# + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_DIR="$(dirname "$SCRIPT_DIR")" + +# Default values +SCENARIO="${SCENARIO:-connections}" +DURATION="${DURATION:-60}" +CONCURRENCY="${CONCURRENCY:-50}" +RAMP_UP="${RAMP_UP:-10}" +RPS="${RPS:-1}" +VERBOSE="${VERBOSE:-}" + +# Build args +ARGS=() + +if [ -n "$GATEWAY_URL" ]; then + ARGS+=("--url" "$GATEWAY_URL") +fi + +if [ -n "$GATEWAY_TOKEN" ]; then + ARGS+=("--token" "$GATEWAY_TOKEN") +fi + +# Parse command line args +while [[ $# -gt 0 ]]; do + case $1 in + -s|--scenario) + SCENARIO="$2" + shift 2 + ;; + -d|--duration) + DURATION="$2" + shift 2 + ;; + -c|--concurrency) + CONCURRENCY="$2" + shift 2 + ;; + -r|--ramp-up) + RAMP_UP="$2" + shift 2 + ;; + --rps) + RPS="$2" + shift 2 + ;; + -v|--verbose) + VERBOSE="1" + shift + ;; + --json) + ARGS+=("--json") + shift + ;; + -u|--url) + ARGS+=("--url" "$2") + shift 2 + ;; + -t|--token) + ARGS+=("--token" "$2") + shift 2 + ;; + -h|--help) + echo "Usage: $0 [options]" + echo "" + echo "Options:" + echo " -s, --scenario Test scenario: connections, chat, auth-stress" + echo " -d, --duration Test duration in seconds (default: 60)" + echo " -c, --concurrency Number of concurrent connections (default: 50)" + echo " -r, --ramp-up Ramp-up time in seconds (default: 10)" + echo " --rps Requests per second per user (default: 1)" + echo " -v, --verbose Enable verbose output" + echo " --json Output results as JSON" + echo " -u, --url Gateway WebSocket URL" + echo " -t, --token Authentication token" + echo " -h, --help Show this help" + echo "" + echo "Scenarios:" + echo " connections WebSocket connection stress test" + echo " chat Chat message throughput test" + echo " auth-stress Authentication rate limit verification" + echo "" + echo "Environment variables:" + echo " GATEWAY_URL Gateway WebSocket URL" + echo " GATEWAY_TOKEN Authentication token" + exit 0 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + +ARGS+=("--scenario" "$SCENARIO") +ARGS+=("--duration" "$DURATION") +ARGS+=("--concurrency" "$CONCURRENCY") +ARGS+=("--ramp-up" "$RAMP_UP") +ARGS+=("--rps" "$RPS") + +if [ -n "$VERBOSE" ]; then + ARGS+=("--verbose") +fi + +cd "$PROJECT_DIR" + +# Check if bun is available +if command -v bun &> /dev/null; then + exec bun load-tests/run.ts "${ARGS[@]}" +else + # Fall back to tsx or node with ts-node + if command -v tsx &> /dev/null; then + exec tsx load-tests/run.ts "${ARGS[@]}" + elif command -v npx &> /dev/null; then + exec npx tsx load-tests/run.ts "${ARGS[@]}" + else + echo "Error: bun, tsx, or npx required to run load tests" + exit 1 + fi +fi diff --git a/src/config/types.clawdbot.ts b/src/config/types.clawdbot.ts index c2b2f05e4..392e1d767 100644 --- a/src/config/types.clawdbot.ts +++ b/src/config/types.clawdbot.ts @@ -21,6 +21,7 @@ import type { import type { ModelsConfig } from "./types.models.js"; import type { NodeHostConfig } from "./types.node-host.js"; import type { PluginsConfig } from "./types.plugins.js"; +import type { RbacConfig } from "./types.rbac.js"; import type { SkillsConfig } from "./types.skills.js"; import type { ToolsConfig } from "./types.tools.js"; @@ -95,6 +96,7 @@ export type MoltbotConfig = { canvasHost?: CanvasHostConfig; talk?: TalkConfig; gateway?: GatewayConfig; + rbac?: RbacConfig; }; export type ConfigValidationIssue = { diff --git a/src/config/types.gateway.ts b/src/config/types.gateway.ts index 019126e1f..b329494f8 100644 --- a/src/config/types.gateway.ts +++ b/src/config/types.gateway.ts @@ -191,6 +191,32 @@ export type GatewayHttpConfig = { endpoints?: GatewayHttpEndpointsConfig; }; +/** + * Health endpoint configuration for container orchestration. + * Provides K8s-style liveness/readiness probes. + */ +export type GatewayHealthConfig = { + /** Enable health endpoints (default: true). */ + enabled?: boolean; + /** Base path for health endpoints (default: ""). */ + basePath?: string; + /** Require auth for /health/deep (default: true). */ + deepAuthRequired?: boolean; +}; + +/** + * Prometheus metrics endpoint configuration. + * Exposes metrics in Prometheus exposition format. + */ +export type GatewayMetricsConfig = { + /** Enable metrics endpoint (default: false). */ + enabled?: boolean; + /** Path for metrics endpoint (default: /metrics). */ + path?: string; + /** Require auth for metrics endpoint (default: true). */ + authRequired?: boolean; +}; + export type GatewayNodesConfig = { /** Browser routing policy for node-hosted browser proxies. */ browser?: { @@ -286,4 +312,8 @@ export type GatewayConfig = { * `x-real-ip`) to determine the client IP for local pairing and HTTP checks. */ trustedProxies?: string[]; + /** Health endpoint configuration for container orchestration. */ + health?: GatewayHealthConfig; + /** Prometheus metrics endpoint configuration. */ + metrics?: GatewayMetricsConfig; }; diff --git a/src/config/types.rbac.ts b/src/config/types.rbac.ts new file mode 100644 index 000000000..2e7283fc6 --- /dev/null +++ b/src/config/types.rbac.ts @@ -0,0 +1,48 @@ +/** + * RBAC (Role-Based Access Control) configuration types. + * + * Provides flat role-based permission boundaries for enterprise deployments. + * When enabled, restricts what actions users can perform based on their assigned role. + */ + +/** Tool access control for a role. */ +export type RbacToolAccess = { + /** Tools explicitly allowed (if set, only these tools are available). */ + allow?: string[]; + /** Tools explicitly denied (takes precedence over allow). */ + deny?: string[]; +}; + +/** Permission levels for RBAC. */ +export type RbacPermission = + | "exec" // Can execute basic commands + | "exec.elevated" // Can execute elevated/sudo commands + | "exec.approve" // Can approve exec requests from others + | "admin" // Full administrative access + | "read-only"; // Can only view, not modify + +/** Definition of a single role. */ +export type RbacRoleDefinition = { + /** Human-readable role name. */ + name: string; + /** Permissions granted to this role. */ + permissions: RbacPermission[]; + /** Tool access restrictions (optional). */ + tools?: RbacToolAccess; + /** Allowed agent IDs this role can interact with (optional, all if not set). */ + agents?: string[]; + /** Description of this role's purpose. */ + description?: string; +}; + +/** RBAC configuration. */ +export type RbacConfig = { + /** Enable RBAC enforcement (default: false). */ + enabled?: boolean; + /** Role definitions keyed by role ID. */ + roles?: Record; + /** User assignments: senderId -> roleId. */ + assignments?: Record; + /** Default role for users not explicitly assigned (optional). */ + defaultRole?: string; +}; diff --git a/src/config/types.ts b/src/config/types.ts index 424ccaf26..7b3b9594f 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -27,4 +27,5 @@ export * from "./types.slack.js"; export * from "./types.telegram.js"; export * from "./types.tts.js"; export * from "./types.tools.js"; +export * from "./types.rbac.js"; export * from "./types.whatsapp.js"; diff --git a/src/gateway/exec-approval-manager.ts b/src/gateway/exec-approval-manager.ts index 5e51f2abc..3ae8d3a69 100644 --- a/src/gateway/exec-approval-manager.ts +++ b/src/gateway/exec-approval-manager.ts @@ -1,6 +1,7 @@ import { randomUUID } from "node:crypto"; import type { ExecApprovalDecision } from "../infra/exec-approvals.js"; +import { auditExecApprove, auditExecReject, auditExecRequest } from "../security/audit-log.js"; export type ExecApprovalRequestPayload = { command: string; @@ -46,6 +47,14 @@ export class ExecApprovalManager { createdAtMs: now, expiresAtMs: now + timeoutMs, }; + + // Audit: record exec request + auditExecRequest({ + actor: { type: "agent", id: request.agentId ?? "unknown" }, + target: { type: "session", id: request.sessionKey ?? resolvedId }, + command: request.command, + }); + return record; } @@ -71,6 +80,20 @@ export class ExecApprovalManager { pending.record.resolvedBy = resolvedBy ?? null; this.pending.delete(recordId); pending.resolve(decision); + + // Audit: record exec approval or rejection + const request = pending.record.request; + const auditParams = { + actor: { type: "user" as const, id: resolvedBy ?? "unknown" }, + target: { type: "session", id: request.sessionKey ?? recordId }, + command: request.command, + }; + if (decision === "allow-once" || decision === "allow-always") { + auditExecApprove(auditParams); + } else { + auditExecReject({ ...auditParams, reason: decision }); + } + return true; } diff --git a/src/gateway/http-health.test.ts b/src/gateway/http-health.test.ts new file mode 100644 index 000000000..e632c4e22 --- /dev/null +++ b/src/gateway/http-health.test.ts @@ -0,0 +1,194 @@ +import { describe, expect, it, afterEach } from "vitest"; +import { createServer, type Server, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createHealthEndpointsHandler } from "./http-health.js"; +import type { ResolvedGatewayAuth } from "./auth.js"; + +const mockAuth: ResolvedGatewayAuth = { + mode: "token", + token: "test-token-123", + allowTailscale: false, +}; + +function createTestServer( + handler: (req: IncomingMessage, res: ServerResponse) => Promise, +) { + const server = createServer(async (req, res) => { + const handled = await handler(req, res); + if (!handled) { + res.statusCode = 404; + res.end("Not Found"); + } + }); + return server; +} + +async function startServer(server: Server): Promise { + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${addr.port}`); + }); + }); +} + +async function stopServer(server: Server): Promise { + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +describe("http-health", () => { + let server: Server; + let baseUrl: string; + + afterEach(async () => { + if (server) { + await stopServer(server); + } + }); + + describe("GET /health", () => { + it("returns 200 with healthy status when enabled", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health`); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.status).toBe("healthy"); + expect(body.version).toBeDefined(); + expect(body.uptimeMs).toBeGreaterThanOrEqual(0); + expect(body.timestamp).toBeDefined(); + }); + + it("returns 404 when disabled", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health`); + expect(res.status).toBe(404); + }); + + it("respects custom basePath", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true, basePath: "/api/v1" }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + // Original path should 404 + const res1 = await fetch(`${baseUrl}/health`); + expect(res1.status).toBe(404); + + // Custom path should work + const res2 = await fetch(`${baseUrl}/api/v1/health`); + expect(res2.status).toBe(200); + }); + }); + + describe("GET /ready", () => { + it("returns 200 when healthy", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/ready`); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.status).toBe("healthy"); + }); + }); + + describe("GET /health/deep", () => { + it("returns 401 without auth when deepAuthRequired is true", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true, deepAuthRequired: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health/deep`); + expect(res.status).toBe(401); + + const body = await res.json(); + expect(body.error).toBe("Unauthorized"); + }); + + it("returns 200 with valid auth", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true, deepAuthRequired: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health/deep`, { + headers: { + Authorization: `Bearer ${mockAuth.token}`, + }, + }); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.status).toBeDefined(); + expect(body.checks).toBeDefined(); + }); + + it("returns 200 without auth when deepAuthRequired is false", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true, deepAuthRequired: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health/deep`); + expect(res.status).toBe(200); + + const body = await res.json(); + expect(body.checks).toBeDefined(); + }); + }); + + describe("response format", () => { + it("includes Cache-Control: no-store header", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health`); + expect(res.headers.get("Cache-Control")).toBe("no-store"); + }); + + it("includes Content-Type: application/json header", async () => { + const handler = createHealthEndpointsHandler({ + config: { enabled: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/health`); + expect(res.headers.get("Content-Type")).toContain("application/json"); + }); + }); +}); diff --git a/src/gateway/http-health.ts b/src/gateway/http-health.ts new file mode 100644 index 000000000..ea94295e6 --- /dev/null +++ b/src/gateway/http-health.ts @@ -0,0 +1,203 @@ +/** + * HTTP health endpoints for container orchestration (K8s liveness/readiness). + * + * - GET /health - Liveness probe: 200 if process alive + * - GET /ready - Readiness probe: 200 if channels ready, 503 if degraded + * - GET /health/deep - Detailed status (auth-protected) + */ + +import type { IncomingMessage, ServerResponse } from "node:http"; +import { VERSION } from "../version.js"; +import type { HealthSummary } from "../commands/health.js"; +import type { GatewayHealthConfig } from "../config/types.gateway.js"; +import { authorizeGatewayConnect, type ResolvedGatewayAuth } from "./auth.js"; +import { getBearerToken } from "./http-utils.js"; +import { getHealthCache, refreshGatewayHealthSnapshot } from "./server/health-state.js"; + +export type HealthStatus = "healthy" | "degraded" | "unhealthy"; + +export type HealthResponse = { + status: HealthStatus; + version: string; + uptimeMs: number; + timestamp: string; + checks?: { + channels?: { + total: number; + configured: number; + healthy: number; + degraded: string[]; + }; + agents?: { + total: number; + default: string; + }; + }; +}; + +const startTime = Date.now(); + +function sendJson(res: ServerResponse, status: number, body: unknown) { + res.statusCode = status; + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader("Cache-Control", "no-store"); + res.end(JSON.stringify(body)); +} + +/** + * Determine overall health status from channel health summaries. + * - healthy: all configured channels have successful probes + * - degraded: some channels have failed probes + * - unhealthy: all channels have failed probes or critical failure + */ +function determineHealthStatus(health: HealthSummary | null): { + status: HealthStatus; + degradedChannels: string[]; +} { + if (!health) { + return { status: "healthy", degradedChannels: [] }; + } + + const channels = health.channels ?? {}; + const channelIds = Object.keys(channels); + const degradedChannels: string[] = []; + + for (const channelId of channelIds) { + const channelSummary = channels[channelId]; + if (!channelSummary) continue; + + // Skip unconfigured channels + if (channelSummary.configured === false) continue; + + // Check probe status + const probe = channelSummary.probe as { ok?: boolean } | undefined; + if (probe && probe.ok === false) { + degradedChannels.push(channelId); + } + + // Check account-level probes + const accounts = channelSummary.accounts ?? {}; + for (const [accountId, accountSummary] of Object.entries(accounts)) { + const accountProbe = accountSummary.probe as { ok?: boolean } | undefined; + if (accountProbe && accountProbe.ok === false) { + const key = `${channelId}:${accountId}`; + if (!degradedChannels.includes(key)) { + degradedChannels.push(key); + } + } + } + } + + const configuredCount = channelIds.filter((id) => channels[id]?.configured !== false).length; + + if (degradedChannels.length === 0) { + return { status: "healthy", degradedChannels }; + } + if (degradedChannels.length >= configuredCount && configuredCount > 0) { + return { status: "unhealthy", degradedChannels }; + } + return { status: "degraded", degradedChannels }; +} + +function buildHealthResponse(health: HealthSummary | null, includeChecks: boolean): HealthResponse { + const uptimeMs = Date.now() - startTime; + const { status, degradedChannels } = determineHealthStatus(health); + + const response: HealthResponse = { + status, + version: VERSION, + uptimeMs, + timestamp: new Date().toISOString(), + }; + + if (includeChecks && health) { + const channels = health.channels ?? {}; + const channelIds = Object.keys(channels); + const configuredCount = channelIds.filter((id) => channels[id]?.configured !== false).length; + const healthyCount = configuredCount - degradedChannels.length; + + response.checks = { + channels: { + total: channelIds.length, + configured: configuredCount, + healthy: healthyCount, + degraded: degradedChannels, + }, + agents: { + total: health.agents?.length ?? 0, + default: health.defaultAgentId ?? "pi", + }, + }; + } + + return response; +} + +export type HealthEndpointsHandler = ( + req: IncomingMessage, + res: ServerResponse, +) => Promise; + +export function createHealthEndpointsHandler(opts: { + config?: GatewayHealthConfig; + resolvedAuth: ResolvedGatewayAuth; + trustedProxies?: string[]; +}): HealthEndpointsHandler { + const { config, resolvedAuth, trustedProxies } = opts; + const basePath = config?.basePath ?? ""; + const deepAuthRequired = config?.deepAuthRequired !== false; + + return async (req, res) => { + if (config?.enabled === false) return false; + + const url = new URL(req.url ?? "/", "http://localhost"); + const path = url.pathname; + + // GET /health - Liveness probe (always 200 if process is running) + if (path === `${basePath}/health` && req.method === "GET") { + const health = getHealthCache(); + const response = buildHealthResponse(health, false); + sendJson(res, 200, response); + return true; + } + + // GET /ready - Readiness probe (503 if degraded/unhealthy) + if (path === `${basePath}/ready` && req.method === "GET") { + const health = getHealthCache(); + const { status } = determineHealthStatus(health); + const response = buildHealthResponse(health, false); + + const httpStatus = status === "healthy" ? 200 : 503; + sendJson(res, httpStatus, response); + return true; + } + + // GET /health/deep - Detailed health with auth + if (path === `${basePath}/health/deep` && req.method === "GET") { + if (deepAuthRequired) { + const token = getBearerToken(req); + const authResult = await authorizeGatewayConnect({ + auth: resolvedAuth, + connectAuth: token ? { token, password: token } : null, + req, + trustedProxies, + }); + if (!authResult.ok) { + sendJson(res, 401, { + error: "Unauthorized", + reason: authResult.reason ?? "Authentication required", + }); + return true; + } + } + + // Refresh health snapshot for deep check + const health = await refreshGatewayHealthSnapshot({ probe: true }); + const response = buildHealthResponse(health, true); + sendJson(res, 200, response); + return true; + } + + return false; + }; +} diff --git a/src/gateway/http-metrics.test.ts b/src/gateway/http-metrics.test.ts new file mode 100644 index 000000000..061071145 --- /dev/null +++ b/src/gateway/http-metrics.test.ts @@ -0,0 +1,176 @@ +import { describe, expect, it, afterEach } from "vitest"; +import { createServer, type Server, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createMetricsEndpointsHandler } from "./http-metrics.js"; +import type { ResolvedGatewayAuth } from "./auth.js"; + +const mockAuth: ResolvedGatewayAuth = { + mode: "token", + token: "test-token-123", + allowTailscale: false, +}; + +function createTestServer( + handler: (req: IncomingMessage, res: ServerResponse) => Promise, +) { + const server = createServer(async (req, res) => { + const handled = await handler(req, res); + if (!handled) { + res.statusCode = 404; + res.end("Not Found"); + } + }); + return server; +} + +async function startServer(server: Server): Promise { + return new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + const addr = server.address() as AddressInfo; + resolve(`http://127.0.0.1:${addr.port}`); + }); + }); +} + +async function stopServer(server: Server): Promise { + return new Promise((resolve) => { + server.close(() => resolve()); + }); +} + +describe("http-metrics", () => { + let server: Server; + let baseUrl: string; + + afterEach(async () => { + if (server) { + await stopServer(server); + } + }); + + describe("GET /metrics", () => { + it("returns 404 when metrics are disabled (default)", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + expect(res.status).toBe(404); + }); + + it("returns 401 without auth when enabled and authRequired is true", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + expect(res.status).toBe(401); + + const body = await res.json(); + expect(body.error).toBe("Unauthorized"); + }); + + it("returns 200 with valid auth when enabled", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: true }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`, { + headers: { + Authorization: `Bearer ${mockAuth.token}`, + }, + }); + expect(res.status).toBe(200); + + const body = await res.text(); + // Should contain Prometheus metrics format + expect(body).toContain("# HELP"); + expect(body).toContain("# TYPE"); + }); + + it("returns 200 without auth when authRequired is false", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + expect(res.status).toBe(200); + }); + + it("respects custom path", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: false, path: "/custom/metrics" }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + // Default path should 404 + const res1 = await fetch(`${baseUrl}/metrics`); + expect(res1.status).toBe(404); + + // Custom path should work + const res2 = await fetch(`${baseUrl}/custom/metrics`); + expect(res2.status).toBe(200); + }); + }); + + describe("response format", () => { + it("includes correct Content-Type header", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + const contentType = res.headers.get("Content-Type"); + // Prometheus text format + expect(contentType).toMatch(/text\/plain|application\/openmetrics-text/); + }); + + it("includes clawdbot metrics", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + const body = await res.text(); + + // Check for clawdbot-specific metrics + expect(body).toContain("clawdbot_gateway_uptime_seconds"); + }); + + it("includes default Node.js metrics", async () => { + const handler = createMetricsEndpointsHandler({ + config: { enabled: true, authRequired: false }, + resolvedAuth: mockAuth, + }); + server = createTestServer(handler); + baseUrl = await startServer(server); + + const res = await fetch(`${baseUrl}/metrics`); + const body = await res.text(); + + // Check for default Node.js metrics + expect(body).toContain("nodejs_"); + expect(body).toContain("process_"); + }); + }); +}); diff --git a/src/gateway/http-metrics.ts b/src/gateway/http-metrics.ts new file mode 100644 index 000000000..010052250 --- /dev/null +++ b/src/gateway/http-metrics.ts @@ -0,0 +1,90 @@ +/** + * Prometheus metrics endpoint for observability. + * + * - GET /metrics - Prometheus exposition format metrics + */ + +import type { IncomingMessage, ServerResponse } from "node:http"; +import type { GatewayMetricsConfig } from "../config/types.gateway.js"; +import { authorizeGatewayConnect, type ResolvedGatewayAuth } from "./auth.js"; +import { getBearerToken } from "./http-utils.js"; +import { + getMetricsText, + getMetricsContentType, + startMetricsCollection, +} from "../observability/metrics-registry.js"; + +export type MetricsEndpointsHandler = ( + req: IncomingMessage, + res: ServerResponse, +) => Promise; + +function sendText(res: ServerResponse, status: number, body: string, contentType: string) { + res.statusCode = status; + res.setHeader("Content-Type", contentType); + res.setHeader("Cache-Control", "no-store"); + res.end(body); +} + +function sendJson(res: ServerResponse, status: number, body: unknown) { + res.statusCode = status; + res.setHeader("Content-Type", "application/json; charset=utf-8"); + res.setHeader("Cache-Control", "no-store"); + res.end(JSON.stringify(body)); +} + +export function createMetricsEndpointsHandler(opts: { + config?: GatewayMetricsConfig; + resolvedAuth: ResolvedGatewayAuth; + trustedProxies?: string[]; +}): MetricsEndpointsHandler { + const { config, resolvedAuth, trustedProxies } = opts; + const metricsPath = config?.path ?? "/metrics"; + const authRequired = config?.authRequired !== false; + + // Start metrics collection when handler is created (if enabled) + if (config?.enabled !== false) { + startMetricsCollection(); + } + + return async (req, res) => { + // Metrics disabled by default (must be explicitly enabled) + if (config?.enabled !== true) return false; + + const url = new URL(req.url ?? "/", "http://localhost"); + const path = url.pathname; + + // GET /metrics - Prometheus exposition format + if (path === metricsPath && req.method === "GET") { + if (authRequired) { + const token = getBearerToken(req); + const authResult = await authorizeGatewayConnect({ + auth: resolvedAuth, + connectAuth: token ? { token, password: token } : null, + req, + trustedProxies, + }); + if (!authResult.ok) { + sendJson(res, 401, { + error: "Unauthorized", + reason: authResult.reason ?? "Authentication required", + }); + return true; + } + } + + try { + const metricsText = await getMetricsText(); + sendText(res, 200, metricsText, getMetricsContentType()); + } catch (err) { + sendJson(res, 500, { + error: "Internal Server Error", + message: err instanceof Error ? err.message : "Failed to collect metrics", + }); + } + return true; + } + + return false; + }; +} diff --git a/src/gateway/server-http.ts b/src/gateway/server-http.ts index f08dc811c..e48ce2864 100644 --- a/src/gateway/server-http.ts +++ b/src/gateway/server-http.ts @@ -10,10 +10,13 @@ import type { WebSocketServer } from "ws"; import { handleA2uiHttpRequest } from "../canvas-host/a2ui.js"; import type { CanvasHostHandler } from "../canvas-host/server.js"; import { loadConfig } from "../config/config.js"; +import type { GatewayHealthConfig, GatewayMetricsConfig } from "../config/types.gateway.js"; import type { createSubsystemLogger } from "../logging/subsystem.js"; import { handleSlackHttpRequest } from "../slack/http/index.js"; import { resolveAgentAvatar } from "../agents/identity-avatar.js"; import { handleControlUiAvatarRequest, handleControlUiHttpRequest } from "./control-ui.js"; +import { createHealthEndpointsHandler, type HealthEndpointsHandler } from "./http-health.js"; +import { createMetricsEndpointsHandler, type MetricsEndpointsHandler } from "./http-metrics.js"; import { extractHookToken, getHookChannelError, @@ -211,6 +214,9 @@ export function createGatewayHttpServer(opts: { handlePluginRequest?: HooksRequestHandler; resolvedAuth: import("./auth.js").ResolvedGatewayAuth; tlsOptions?: TlsOptions; + healthConfig?: GatewayHealthConfig; + metricsConfig?: GatewayMetricsConfig; + trustedProxies?: string[]; }): HttpServer { const { canvasHost, @@ -222,7 +228,24 @@ export function createGatewayHttpServer(opts: { handleHooksRequest, handlePluginRequest, resolvedAuth, + healthConfig, + metricsConfig, + trustedProxies, } = opts; + + // Health endpoints for container orchestration (enabled by default) + const handleHealthRequest: HealthEndpointsHandler = createHealthEndpointsHandler({ + config: healthConfig, + resolvedAuth, + trustedProxies, + }); + + // Metrics endpoint for Prometheus (disabled by default) + const handleMetricsRequest: MetricsEndpointsHandler = createMetricsEndpointsHandler({ + config: metricsConfig, + resolvedAuth, + trustedProxies, + }); const httpServer: HttpServer = opts.tlsOptions ? createHttpsServer(opts.tlsOptions, (req, res) => { void handleRequest(req, res); @@ -237,12 +260,17 @@ export function createGatewayHttpServer(opts: { try { const configSnapshot = loadConfig(); - const trustedProxies = configSnapshot.gateway?.trustedProxies ?? []; + const configTrustedProxies = configSnapshot.gateway?.trustedProxies ?? []; + + // Health and metrics endpoints (highest priority for ops tooling) + if (await handleHealthRequest(req, res)) return; + if (await handleMetricsRequest(req, res)) return; + if (await handleHooksRequest(req, res)) return; if ( await handleToolsInvokeHttpRequest(req, res, { auth: resolvedAuth, - trustedProxies, + trustedProxies: configTrustedProxies, }) ) return; @@ -253,7 +281,7 @@ export function createGatewayHttpServer(opts: { await handleOpenResponsesHttpRequest(req, res, { auth: resolvedAuth, config: openResponsesConfig, - trustedProxies, + trustedProxies: configTrustedProxies, }) ) return; @@ -262,7 +290,7 @@ export function createGatewayHttpServer(opts: { if ( await handleOpenAiHttpRequest(req, res, { auth: resolvedAuth, - trustedProxies, + trustedProxies: configTrustedProxies, }) ) return; diff --git a/src/gateway/server-runtime-state.ts b/src/gateway/server-runtime-state.ts index 02b060a3b..8b2e2a156 100644 --- a/src/gateway/server-runtime-state.ts +++ b/src/gateway/server-runtime-state.ts @@ -119,6 +119,9 @@ export async function createGatewayRuntimeState(params: { handlePluginRequest, resolvedAuth: params.resolvedAuth, tlsOptions: params.gatewayTls?.enabled ? params.gatewayTls.tlsOptions : undefined, + healthConfig: params.cfg.gateway?.health, + metricsConfig: params.cfg.gateway?.metrics, + trustedProxies: params.cfg.gateway?.trustedProxies, }); try { await listenGatewayHttpServer({ diff --git a/src/gateway/server/ws-connection/message-handler.ts b/src/gateway/server/ws-connection/message-handler.ts index d1f6ae511..b03045eec 100644 --- a/src/gateway/server/ws-connection/message-handler.ts +++ b/src/gateway/server/ws-connection/message-handler.ts @@ -7,6 +7,7 @@ import { normalizeDevicePublicKeyBase64Url, verifyDeviceSignature, } from "../../../infra/device-identity.js"; +import { auditAuthFailure, auditAuthLogin } from "../../../security/audit-log.js"; import { approveDevicePairing, ensureDeviceToken, @@ -601,6 +602,18 @@ export function attachGatewayWsMessageHandler(params: { reason: authResult.reason, client: connectParams.client, }); + + // Audit: record auth failure + auditAuthFailure({ + actor: { + type: "user", + id: device?.id ?? connectParams.client.id ?? "unknown", + remoteIp: clientIp ?? remoteAddr, + channel: "gateway", + }, + reason: authResult.reason ?? "unauthorized", + }); + setCloseCause("unauthorized", { authMode: resolvedAuth.mode, authProvided, @@ -755,6 +768,18 @@ export function attachGatewayWsMessageHandler(params: { auth: authMethod, }); + // Audit: record successful auth login + auditAuthLogin({ + actor: { + type: device ? "device" : "user", + id: device?.id ?? connectParams.client.id ?? "unknown", + remoteIp: clientIp ?? remoteAddr, + channel: "gateway", + deviceId: device?.id, + }, + method: authMethod, + }); + if (isWebchatConnect(connectParams)) { logWsControl.info( `webchat connected conn=${connId} remote=${remoteAddr ?? "?"} client=${clientLabel} ${connectParams.client.mode} v${connectParams.client.version}`, diff --git a/src/infra/diagnostic-events.ts b/src/infra/diagnostic-events.ts index 586177b0b..04309175a 100644 --- a/src/infra/diagnostic-events.ts +++ b/src/infra/diagnostic-events.ts @@ -1,10 +1,13 @@ import type { MoltbotConfig } from "../config/config.js"; +import { getCurrentTraceId } from "../observability/trace-context.js"; export type DiagnosticSessionState = "idle" | "processing" | "waiting"; type DiagnosticBaseEvent = { ts: number; seq: number; + /** W3C trace ID for distributed tracing correlation. */ + traceId?: string; }; export type DiagnosticUsageEvent = DiagnosticBaseEvent & { @@ -154,10 +157,12 @@ export function isDiagnosticsEnabled(config?: MoltbotConfig): boolean { } export function emitDiagnosticEvent(event: DiagnosticEventInput) { + const traceId = getCurrentTraceId(); const enriched = { ...event, seq: (seq += 1), ts: Date.now(), + ...(traceId !== "no-trace" ? { traceId } : {}), } satisfies DiagnosticEventPayload; for (const listener of listeners) { try { diff --git a/src/logging/subsystem.ts b/src/logging/subsystem.ts index a156fd8f3..dc6a7a37a 100644 --- a/src/logging/subsystem.ts +++ b/src/logging/subsystem.ts @@ -9,6 +9,7 @@ import { type LogLevel, levelToMinLevel } from "./levels.js"; import { getChildLogger } from "./logger.js"; import { loggingState } from "./state.js"; import { clearActiveProgressLine } from "../terminal/progress-line.js"; +import { getShortTraceId } from "../observability/trace-context.js"; type LogObj = { date?: Date } & Record; @@ -126,6 +127,7 @@ function formatConsoleLine(opts: { message: string; style: "pretty" | "compact" | "json"; meta?: Record; + traceId?: string; }): string { const displaySubsystem = opts.style === "json" ? opts.subsystem : formatSubsystemForConsole(opts.subsystem); @@ -135,6 +137,7 @@ function formatConsoleLine(opts: { level: opts.level, subsystem: displaySubsystem, message: opts.message, + ...(opts.traceId && opts.traceId !== "no-trace" ? { traceId: opts.traceId } : {}), ...opts.meta, }); } @@ -160,7 +163,10 @@ function formatConsoleLine(opts: { return ""; })(); const prefixToken = prefixColor(prefix); - const head = [time, prefixToken].filter(Boolean).join(" "); + // Include short trace ID in the prefix if available (for distributed tracing correlation) + const traceToken = + opts.traceId && opts.traceId !== "no-trace" ? color.gray(`[${opts.traceId}]`) : ""; + const head = [time, prefixToken, traceToken].filter(Boolean).join(" "); return `${head} ${levelColor(displayMessage)}`; } @@ -218,7 +224,12 @@ export function createSubsystemLogger(subsystem: string): SubsystemLogger { } fileMeta = Object.keys(rest).length > 0 ? rest : undefined; } - logToFile(getFileLogger(), level, message, fileMeta); + // Get trace ID for correlation + const traceId = getShortTraceId(); + // Include trace ID in file log metadata + const fileMetaWithTrace = + traceId && traceId !== "no-trace" ? { ...fileMeta, traceId } : fileMeta; + logToFile(getFileLogger(), level, message, fileMetaWithTrace); if (!shouldLogToConsole(level, { level: consoleSettings.level })) return; if (!shouldLogSubsystemToConsole(subsystem)) return; const consoleMessage = consoleMessageOverride ?? message; @@ -235,6 +246,7 @@ export function createSubsystemLogger(subsystem: string): SubsystemLogger { message: consoleSettings.style === "json" ? message : consoleMessage, style: consoleSettings.style, meta: fileMeta, + traceId, }); writeConsoleLine(level, line); }; diff --git a/src/observability/metrics-registry.ts b/src/observability/metrics-registry.ts new file mode 100644 index 000000000..0d89fff00 --- /dev/null +++ b/src/observability/metrics-registry.ts @@ -0,0 +1,347 @@ +/** + * Prometheus metrics registry for observability. + * + * This module provides a shared metrics registry that mirrors the OTel metrics + * defined in the diagnostics-otel extension. The metrics are updated by + * subscribing to diagnostic events. + */ + +import { Counter, Gauge, Histogram, Registry, collectDefaultMetrics } from "prom-client"; +import { onDiagnosticEvent, type DiagnosticEventPayload } from "../infra/diagnostic-events.js"; +import { VERSION } from "../version.js"; + +const registry = new Registry(); + +// Set default labels +registry.setDefaultLabels({ + app: "clawdbot", + version: VERSION, +}); + +// Collect default Node.js metrics (memory, CPU, event loop, etc.) +collectDefaultMetrics({ register: registry }); + +// Gateway uptime gauge +const gatewayUptime = new Gauge({ + name: "clawdbot_gateway_uptime_seconds", + help: "Gateway uptime in seconds", + registers: [registry], +}); + +const startTime = Date.now(); +gatewayUptime.set(0); + +// Token usage counter +const tokensTotal = new Counter({ + name: "clawdbot_tokens_total", + help: "Total token usage by type", + labelNames: ["type", "channel", "provider", "model"], + registers: [registry], +}); + +// Cost counter +const costTotal = new Counter({ + name: "clawdbot_cost_usd_total", + help: "Total estimated model cost in USD", + labelNames: ["channel", "provider", "model"], + registers: [registry], +}); + +// Run duration histogram +const runDuration = new Histogram({ + name: "clawdbot_run_duration_seconds", + help: "Agent run duration in seconds", + labelNames: ["channel", "provider", "model"], + buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60, 120], + registers: [registry], +}); + +// Context window histogram +const contextTokens = new Histogram({ + name: "clawdbot_context_tokens", + help: "Context window token usage", + labelNames: ["type", "channel", "provider", "model"], + buckets: [1000, 5000, 10000, 25000, 50000, 100000, 200000], + registers: [registry], +}); + +// Webhook counters +const webhookReceived = new Counter({ + name: "clawdbot_webhook_received_total", + help: "Total webhook requests received", + labelNames: ["channel", "update_type"], + registers: [registry], +}); + +const webhookError = new Counter({ + name: "clawdbot_webhook_error_total", + help: "Total webhook processing errors", + labelNames: ["channel", "update_type"], + registers: [registry], +}); + +const webhookDuration = new Histogram({ + name: "clawdbot_webhook_duration_seconds", + help: "Webhook processing duration in seconds", + labelNames: ["channel", "update_type"], + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5], + registers: [registry], +}); + +// Message processing counters +const messageQueued = new Counter({ + name: "clawdbot_message_queued_total", + help: "Total messages queued for processing", + labelNames: ["channel", "source"], + registers: [registry], +}); + +const messageProcessed = new Counter({ + name: "clawdbot_message_processed_total", + help: "Total messages processed", + labelNames: ["channel", "outcome"], + registers: [registry], +}); + +const messageDuration = new Histogram({ + name: "clawdbot_message_duration_seconds", + help: "Message processing duration in seconds", + labelNames: ["channel", "outcome"], + buckets: [0.1, 0.5, 1, 2, 5, 10, 30, 60], + registers: [registry], +}); + +// Queue metrics +const queueDepth = new Gauge({ + name: "clawdbot_queue_depth", + help: "Current queue depth", + labelNames: ["lane"], + registers: [registry], +}); + +const queueWait = new Histogram({ + name: "clawdbot_queue_wait_seconds", + help: "Queue wait time before execution in seconds", + labelNames: ["lane"], + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10], + registers: [registry], +}); + +// Session state gauge +const sessionState = new Gauge({ + name: "clawdbot_session_state", + help: "Current session state count", + labelNames: ["state"], + registers: [registry], +}); + +// Session stuck counter +const sessionStuck = new Counter({ + name: "clawdbot_session_stuck_total", + help: "Total sessions stuck in processing", + labelNames: ["state"], + registers: [registry], +}); + +// Run attempt counter +const runAttempt = new Counter({ + name: "clawdbot_run_attempt_total", + help: "Total run attempts", + labelNames: ["attempt"], + registers: [registry], +}); + +// Track session state counts +const sessionStateCounts = new Map(); + +/** + * Handle diagnostic events and update metrics. + */ +function handleDiagnosticEvent(evt: DiagnosticEventPayload): void { + switch (evt.type) { + case "model.usage": { + const labels = { + channel: evt.channel ?? "unknown", + provider: evt.provider ?? "unknown", + model: evt.model ?? "unknown", + }; + + if (evt.usage.input) { + tokensTotal.inc({ ...labels, type: "input" }, evt.usage.input); + } + if (evt.usage.output) { + tokensTotal.inc({ ...labels, type: "output" }, evt.usage.output); + } + if (evt.usage.cacheRead) { + tokensTotal.inc({ ...labels, type: "cache_read" }, evt.usage.cacheRead); + } + if (evt.usage.cacheWrite) { + tokensTotal.inc({ ...labels, type: "cache_write" }, evt.usage.cacheWrite); + } + if (evt.usage.total) { + tokensTotal.inc({ ...labels, type: "total" }, evt.usage.total); + } + + if (evt.costUsd) { + costTotal.inc(labels, evt.costUsd); + } + if (evt.durationMs) { + runDuration.observe(labels, evt.durationMs / 1000); + } + if (evt.context?.limit) { + contextTokens.observe({ ...labels, type: "limit" }, evt.context.limit); + } + if (evt.context?.used) { + contextTokens.observe({ ...labels, type: "used" }, evt.context.used); + } + break; + } + + case "webhook.received": { + webhookReceived.inc({ + channel: evt.channel ?? "unknown", + update_type: evt.updateType ?? "unknown", + }); + break; + } + + case "webhook.processed": { + if (typeof evt.durationMs === "number") { + webhookDuration.observe( + { + channel: evt.channel ?? "unknown", + update_type: evt.updateType ?? "unknown", + }, + evt.durationMs / 1000, + ); + } + break; + } + + case "webhook.error": { + webhookError.inc({ + channel: evt.channel ?? "unknown", + update_type: evt.updateType ?? "unknown", + }); + break; + } + + case "message.queued": { + messageQueued.inc({ + channel: evt.channel ?? "unknown", + source: evt.source ?? "unknown", + }); + break; + } + + case "message.processed": { + const labels = { + channel: evt.channel ?? "unknown", + outcome: evt.outcome ?? "unknown", + }; + messageProcessed.inc(labels); + if (typeof evt.durationMs === "number") { + messageDuration.observe(labels, evt.durationMs / 1000); + } + break; + } + + case "queue.lane.enqueue": { + queueDepth.set({ lane: evt.lane }, evt.queueSize); + break; + } + + case "queue.lane.dequeue": { + queueDepth.set({ lane: evt.lane }, evt.queueSize); + if (typeof evt.waitMs === "number") { + queueWait.observe({ lane: evt.lane }, evt.waitMs / 1000); + } + break; + } + + case "session.state": { + // Decrement previous state count + if (evt.prevState) { + const prevCount = sessionStateCounts.get(evt.prevState) ?? 0; + if (prevCount > 0) { + sessionStateCounts.set(evt.prevState, prevCount - 1); + sessionState.set({ state: evt.prevState }, prevCount - 1); + } + } + // Increment new state count + const newCount = (sessionStateCounts.get(evt.state) ?? 0) + 1; + sessionStateCounts.set(evt.state, newCount); + sessionState.set({ state: evt.state }, newCount); + break; + } + + case "session.stuck": { + sessionStuck.inc({ state: evt.state }); + break; + } + + case "run.attempt": { + runAttempt.inc({ attempt: String(evt.attempt) }); + break; + } + + case "diagnostic.heartbeat": { + // Update queue depth from heartbeat + queueDepth.set({ lane: "global" }, evt.queued); + break; + } + } +} + +let unsubscribe: (() => void) | null = null; +let uptimeInterval: ReturnType | null = null; + +/** + * Start the metrics collection by subscribing to diagnostic events. + */ +export function startMetricsCollection(): void { + if (unsubscribe) return; + unsubscribe = onDiagnosticEvent(handleDiagnosticEvent); + + // Update uptime every second + uptimeInterval = setInterval(() => { + gatewayUptime.set((Date.now() - startTime) / 1000); + }, 1000); +} + +/** + * Stop the metrics collection. + */ +export function stopMetricsCollection(): void { + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + if (uptimeInterval) { + clearInterval(uptimeInterval); + uptimeInterval = null; + } +} + +/** + * Get the metrics registry. + */ +export function getMetricsRegistry(): Registry { + return registry; +} + +/** + * Get metrics in Prometheus exposition format. + */ +export async function getMetricsText(): Promise { + // Update uptime before returning + gatewayUptime.set((Date.now() - startTime) / 1000); + return registry.metrics(); +} + +/** + * Get the content type for Prometheus metrics. + */ +export function getMetricsContentType(): string { + return registry.contentType; +} diff --git a/src/observability/trace-context.test.ts b/src/observability/trace-context.test.ts new file mode 100644 index 000000000..f7bf8adea --- /dev/null +++ b/src/observability/trace-context.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it } from "vitest"; +import { + generateTraceId, + generateSpanId, + parseTraceparent, + formatTraceparent, + createTraceContext, + getCurrentTraceContext, + getCurrentTraceId, + getShortTraceId, + withTraceContext, + withTraceContextAsync, + extractOrCreateTraceContext, + createOutgoingTraceparent, + traceLogMeta, +} from "./trace-context.js"; + +describe("trace-context", () => { + describe("generateTraceId", () => { + it("generates a 32 character hex string", () => { + const traceId = generateTraceId(); + expect(traceId).toMatch(/^[a-f0-9]{32}$/); + }); + + it("generates unique IDs", () => { + const ids = new Set(); + for (let i = 0; i < 100; i++) { + ids.add(generateTraceId()); + } + expect(ids.size).toBe(100); + }); + }); + + describe("generateSpanId", () => { + it("generates a 16 character hex string", () => { + const spanId = generateSpanId(); + expect(spanId).toMatch(/^[a-f0-9]{16}$/); + }); + + it("generates unique IDs", () => { + const ids = new Set(); + for (let i = 0; i < 100; i++) { + ids.add(generateSpanId()); + } + expect(ids.size).toBe(100); + }); + }); + + describe("parseTraceparent", () => { + it("parses a valid traceparent header", () => { + const header = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"; + const ctx = parseTraceparent(header); + expect(ctx).toEqual({ + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + sampled: true, + parentSpanId: "00f067aa0ba902b7", + }); + }); + + it("handles unsampled traces", () => { + const header = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00"; + const ctx = parseTraceparent(header); + expect(ctx?.sampled).toBe(false); + }); + + it("returns null for undefined input", () => { + expect(parseTraceparent(undefined)).toBeNull(); + }); + + it("returns null for empty string", () => { + expect(parseTraceparent("")).toBeNull(); + }); + + it("returns null for invalid format", () => { + expect(parseTraceparent("invalid")).toBeNull(); + expect(parseTraceparent("00-invalid")).toBeNull(); + expect( + parseTraceparent("01-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"), + ).toBeNull(); + }); + + it("returns null for all-zero trace ID", () => { + expect( + parseTraceparent("00-00000000000000000000000000000000-00f067aa0ba902b7-01"), + ).toBeNull(); + }); + + it("returns null for all-zero span ID", () => { + expect( + parseTraceparent("00-4bf92f3577b34da6a3ce929d0e0e4736-0000000000000000-01"), + ).toBeNull(); + }); + + it("normalizes to lowercase", () => { + const header = "00-4BF92F3577B34DA6A3CE929D0E0E4736-00F067AA0BA902B7-01"; + const ctx = parseTraceparent(header); + expect(ctx?.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736"); + }); + }); + + describe("formatTraceparent", () => { + it("formats a trace context as a traceparent header", () => { + const ctx = { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + sampled: true, + }; + expect(formatTraceparent(ctx)).toBe( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + ); + }); + + it("formats unsampled traces correctly", () => { + const ctx = { + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + sampled: false, + }; + expect(formatTraceparent(ctx)).toBe( + "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-00", + ); + }); + }); + + describe("createTraceContext", () => { + it("creates a new trace context without parent", () => { + const ctx = createTraceContext(); + expect(ctx.traceId).toMatch(/^[a-f0-9]{32}$/); + expect(ctx.spanId).toMatch(/^[a-f0-9]{16}$/); + expect(ctx.sampled).toBe(true); + expect(ctx.parentSpanId).toBeUndefined(); + }); + + it("creates a child span from parent", () => { + const parent = createTraceContext(); + const child = createTraceContext(parent); + + expect(child.traceId).toBe(parent.traceId); + expect(child.spanId).not.toBe(parent.spanId); + expect(child.sampled).toBe(parent.sampled); + expect(child.parentSpanId).toBe(parent.spanId); + }); + }); + + describe("AsyncLocalStorage", () => { + it("returns null when not in a traced context", () => { + expect(getCurrentTraceContext()).toBeNull(); + expect(getCurrentTraceId()).toBe("no-trace"); + expect(getShortTraceId()).toBe("no-trace"); + }); + + it("withTraceContext provides context synchronously", () => { + const ctx = createTraceContext(); + const result = withTraceContext(ctx, () => { + const current = getCurrentTraceContext(); + expect(current).toEqual(ctx); + return getCurrentTraceId(); + }); + expect(result).toBe(ctx.traceId); + }); + + it("withTraceContextAsync provides context asynchronously", async () => { + const ctx = createTraceContext(); + const result = await withTraceContextAsync(ctx, async () => { + await new Promise((resolve) => setTimeout(resolve, 10)); + const current = getCurrentTraceContext(); + expect(current).toEqual(ctx); + return getCurrentTraceId(); + }); + expect(result).toBe(ctx.traceId); + }); + + it("getShortTraceId returns first 8 chars", () => { + const ctx = createTraceContext(); + withTraceContext(ctx, () => { + expect(getShortTraceId()).toBe(ctx.traceId.slice(0, 8)); + }); + }); + }); + + describe("extractOrCreateTraceContext", () => { + it("creates new context when no traceparent header", () => { + const ctx = extractOrCreateTraceContext({}); + expect(ctx.traceId).toMatch(/^[a-f0-9]{32}$/); + expect(ctx.parentSpanId).toBeUndefined(); + }); + + it("extracts context from valid traceparent header", () => { + const ctx = extractOrCreateTraceContext({ + traceparent: "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + }); + expect(ctx.traceId).toBe("4bf92f3577b34da6a3ce929d0e0e4736"); + expect(ctx.parentSpanId).toBe("00f067aa0ba902b7"); + expect(ctx.spanId).not.toBe("00f067aa0ba902b7"); // New span created + }); + + it("creates new context for invalid traceparent header", () => { + const ctx = extractOrCreateTraceContext({ + traceparent: "invalid", + }); + expect(ctx.traceId).toMatch(/^[a-f0-9]{32}$/); + expect(ctx.parentSpanId).toBeUndefined(); + }); + }); + + describe("createOutgoingTraceparent", () => { + it("creates new trace when not in context", () => { + const header = createOutgoingTraceparent(); + expect(header).toMatch(/^00-[a-f0-9]{32}-[a-f0-9]{16}-01$/); + }); + + it("creates child span when in context", () => { + const ctx = createTraceContext(); + withTraceContext(ctx, () => { + const header = createOutgoingTraceparent(); + expect(header).toContain(ctx.traceId); + expect(header).not.toContain(ctx.spanId); + }); + }); + }); + + describe("traceLogMeta", () => { + it("returns empty object when not in context", () => { + expect(traceLogMeta()).toEqual({}); + }); + + it("returns trace metadata when in context", () => { + const parent = createTraceContext(); + const child = createTraceContext(parent); + withTraceContext(child, () => { + const meta = traceLogMeta(); + expect(meta.traceId).toBe(child.traceId); + expect(meta.spanId).toBe(child.spanId); + expect(meta.parentSpanId).toBe(parent.spanId); + }); + }); + }); +}); diff --git a/src/observability/trace-context.ts b/src/observability/trace-context.ts new file mode 100644 index 000000000..0751f6ca5 --- /dev/null +++ b/src/observability/trace-context.ts @@ -0,0 +1,180 @@ +/** + * W3C Trace Context utilities for distributed tracing. + * + * Implements the W3C Trace Context specification for trace propagation: + * https://www.w3.org/TR/trace-context/ + * + * Format: 00--- + * - version: 00 (fixed) + * - trace-id: 32 hex chars (16 bytes) + * - span-id: 16 hex chars (8 bytes) + * - trace-flags: 2 hex chars (1 byte, 01 = sampled) + */ + +import { AsyncLocalStorage } from "node:async_hooks"; +import { randomBytes } from "node:crypto"; + +export type TraceContext = { + traceId: string; + spanId: string; + sampled: boolean; + parentSpanId?: string; +}; + +const TRACEPARENT_REGEX = /^00-([a-f0-9]{32})-([a-f0-9]{16})-([a-f0-9]{2})$/; +const INVALID_TRACE_ID = "00000000000000000000000000000000"; +const INVALID_SPAN_ID = "0000000000000000"; + +const traceStore = new AsyncLocalStorage(); + +/** + * Generate a new random trace ID (32 hex chars). + */ +export function generateTraceId(): string { + return randomBytes(16).toString("hex"); +} + +/** + * Generate a new random span ID (16 hex chars). + */ +export function generateSpanId(): string { + return randomBytes(8).toString("hex"); +} + +/** + * Parse a W3C traceparent header value. + * Returns null if the header is invalid or missing. + */ +export function parseTraceparent(header: string | undefined): TraceContext | null { + if (!header) return null; + + const match = header.trim().toLowerCase().match(TRACEPARENT_REGEX); + if (!match) return null; + + const [, traceId, spanId, flags] = match; + + // Reject all-zero trace ID or span ID + if (traceId === INVALID_TRACE_ID || spanId === INVALID_SPAN_ID) { + return null; + } + + return { + traceId, + spanId, + sampled: (Number.parseInt(flags, 16) & 0x01) === 0x01, + parentSpanId: spanId, // The incoming span becomes our parent + }; +} + +/** + * Format a TraceContext into a W3C traceparent header value. + */ +export function formatTraceparent(ctx: TraceContext): string { + const flags = ctx.sampled ? "01" : "00"; + return `00-${ctx.traceId}-${ctx.spanId}-${flags}`; +} + +/** + * Create a new TraceContext, optionally inheriting from a parent. + */ +export function createTraceContext(parent?: TraceContext | null): TraceContext { + if (parent) { + return { + traceId: parent.traceId, + spanId: generateSpanId(), + sampled: parent.sampled, + parentSpanId: parent.spanId, + }; + } + return { + traceId: generateTraceId(), + spanId: generateSpanId(), + sampled: true, // Default to sampled + }; +} + +/** + * Get the current trace context from AsyncLocalStorage. + * Returns null if not in a traced context. + */ +export function getCurrentTraceContext(): TraceContext | null { + return traceStore.getStore() ?? null; +} + +/** + * Get the current trace ID or a short placeholder. + * Useful for logging when you need a trace ID but may not be in a traced context. + */ +export function getCurrentTraceId(): string { + const ctx = getCurrentTraceContext(); + return ctx?.traceId ?? "no-trace"; +} + +/** + * Get a short trace ID for logging (first 8 chars). + */ +export function getShortTraceId(): string { + const ctx = getCurrentTraceContext(); + return ctx?.traceId.slice(0, 8) ?? "no-trace"; +} + +/** + * Run a function within a trace context. + * The context is available via getCurrentTraceContext() within the function. + */ +export function withTraceContext(ctx: TraceContext, fn: () => T): T { + return traceStore.run(ctx, fn); +} + +/** + * Run an async function within a trace context. + */ +export async function withTraceContextAsync( + ctx: TraceContext, + fn: () => Promise, +): Promise { + return traceStore.run(ctx, fn); +} + +/** + * Extract or create a trace context from an HTTP request. + * If the request has a valid traceparent header, it's used to create a child span. + * Otherwise, a new trace is started. + */ +export function extractOrCreateTraceContext(headers: { + traceparent?: string; + [key: string]: unknown; +}): TraceContext { + const traceparent = headers.traceparent; + const parentHeader = typeof traceparent === "string" ? traceparent : undefined; + const parent = parseTraceparent(parentHeader); + return createTraceContext(parent); +} + +/** + * Create a W3C traceparent header for outgoing requests. + * Uses the current trace context if available, otherwise creates a new one. + */ +export function createOutgoingTraceparent(): string { + let ctx = getCurrentTraceContext(); + if (!ctx) { + ctx = createTraceContext(); + } else { + // Create a new span for the outgoing request + ctx = createTraceContext(ctx); + } + return formatTraceparent(ctx); +} + +/** + * Utility to add trace context to log metadata. + */ +export function traceLogMeta(): Record { + const ctx = getCurrentTraceContext(); + if (!ctx) return {}; + return { + traceId: ctx.traceId, + spanId: ctx.spanId, + ...(ctx.parentSpanId ? { parentSpanId: ctx.parentSpanId } : {}), + }; +} diff --git a/src/pairing/pairing-store.ts b/src/pairing/pairing-store.ts index 14fc183a1..1bbb5dab5 100644 --- a/src/pairing/pairing-store.ts +++ b/src/pairing/pairing-store.ts @@ -7,6 +7,7 @@ import lockfile from "proper-lockfile"; import { getPairingAdapter } from "../channels/plugins/pairing.js"; import type { ChannelId, ChannelPairingAdapter } from "../channels/plugins/types.js"; import { resolveOAuthDir, resolveStateDir } from "../config/paths.js"; +import { auditPairingApprove, auditPairingRequest } from "../security/audit-log.js"; // SECURITY: 16 chars × 5 bits/char = 80 bits entropy (increased from 40 bits) const PAIRING_CODE_LENGTH = 16; @@ -509,6 +510,13 @@ export async function upsertChannelPairingRequest(params: { ...(meta ? { meta } : {}), }; await writeSignedPairingStore(filePath, [...reqs, next]); + + // Audit: record pairing request + auditPairingRequest({ + actor: { type: "user", id }, + target: { type: "channel", id: String(params.channel) }, + }); + return { code, created: true }; }, ); @@ -572,6 +580,13 @@ export async function approveChannelPairingCode(params: { entry: entry.id, env, }); + + // Audit: record pairing approval + auditPairingApprove({ + actor: { type: "system", id: "pairing-store" }, + target: { type: "channel-user", id: entry.id, channel: String(params.channel) }, + }); + return { id: entry.id, entry }; }, ); diff --git a/src/security/audit-log.test.ts b/src/security/audit-log.test.ts new file mode 100644 index 000000000..9eb0e8dd4 --- /dev/null +++ b/src/security/audit-log.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import fs from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { + initAuditLog, + stopAuditLog, + emitAuditEvent, + auditAuthLogin, + auditAuthFailure, + auditPairingApprove, + auditExecRequest, + auditRbacDenied, + readRecentAuditEntries, + queryAuditEntries, + rotateAuditLogs, +} from "./audit-log.js"; + +describe("audit-log", () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "audit-test-")); + initAuditLog({ enabled: true, baseDir: tempDir }); + }); + + afterEach(async () => { + stopAuditLog(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + describe("emitAuditEvent", () => { + it("writes an event to the audit log", async () => { + emitAuditEvent({ + type: "auth.login", + actor: { type: "user", id: "test-user" }, + outcome: "success", + }); + + // Wait for async write + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries.length).toBe(1); + expect(entries[0].type).toBe("auth.login"); + expect(entries[0].actor.id).toBe("test-user"); + expect(entries[0].outcome).toBe("success"); + expect(entries[0].eventId).toBeDefined(); + expect(entries[0].ts).toBeDefined(); + }); + + it("writes multiple events sequentially", async () => { + for (let i = 0; i < 5; i++) { + emitAuditEvent({ + type: "auth.login", + actor: { type: "user", id: `user-${i}` }, + outcome: "success", + }); + } + + await new Promise((resolve) => setTimeout(resolve, 100)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries.length).toBe(5); + // Most recent first + expect(entries[0].actor.id).toBe("user-4"); + }); + + it("includes metadata when provided", async () => { + emitAuditEvent({ + type: "config.change", + actor: { type: "user", id: "admin" }, + outcome: "success", + metadata: { key: "gateway.port", oldValue: 18789, newValue: 8080 }, + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].metadata).toEqual({ + key: "gateway.port", + oldValue: 18789, + newValue: 8080, + }); + }); + }); + + describe("convenience functions", () => { + it("auditAuthLogin writes correct event", async () => { + auditAuthLogin({ + actor: { type: "user", id: "user-1", remoteIp: "192.168.1.1" }, + method: "token", + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].type).toBe("auth.login"); + expect(entries[0].metadata?.method).toBe("token"); + }); + + it("auditAuthFailure writes correct event", async () => { + auditAuthFailure({ + actor: { type: "user", id: "unknown", remoteIp: "10.0.0.1" }, + reason: "invalid_token", + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].type).toBe("auth.failure"); + expect(entries[0].outcome).toBe("failure"); + expect(entries[0].metadata?.reason).toBe("invalid_token"); + }); + + it("auditPairingApprove writes correct event", async () => { + auditPairingApprove({ + actor: { type: "user", id: "admin" }, + target: { type: "device", id: "device-123" }, + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].type).toBe("pairing.approve"); + expect(entries[0].target?.id).toBe("device-123"); + }); + + it("auditExecRequest writes correct event", async () => { + auditExecRequest({ + actor: { type: "agent", id: "pi" }, + target: { type: "session", id: "session-abc" }, + command: "ls -la", + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].type).toBe("exec.request"); + expect(entries[0].metadata?.command).toBe("ls -la"); + }); + + it("auditRbacDenied writes correct event", async () => { + auditRbacDenied({ + actor: { type: "user", id: "user-1" }, + action: "exec.elevated", + resource: "/bin/bash", + reason: "role does not have elevated permission", + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + expect(entries[0].type).toBe("rbac.denied"); + expect(entries[0].outcome).toBe("denied"); + expect(entries[0].metadata?.action).toBe("exec.elevated"); + }); + }); + + describe("queryAuditEntries", () => { + beforeEach(async () => { + // Write some test events + emitAuditEvent({ + type: "auth.login", + actor: { type: "user", id: "user-1" }, + outcome: "success", + }); + emitAuditEvent({ + type: "auth.failure", + actor: { type: "user", id: "user-2" }, + outcome: "failure", + }); + emitAuditEvent({ + type: "auth.login", + actor: { type: "user", id: "user-3" }, + outcome: "success", + }); + + await new Promise((resolve) => setTimeout(resolve, 100)); + }); + + it("filters by type", async () => { + const entries = await queryAuditEntries({ type: "auth.login" }, 10, { baseDir: tempDir }); + expect(entries.length).toBe(2); + expect(entries.every((e) => e.type === "auth.login")).toBe(true); + }); + + it("filters by outcome", async () => { + const entries = await queryAuditEntries({ outcome: "failure" }, 10, { baseDir: tempDir }); + expect(entries.length).toBe(1); + expect(entries[0].actor.id).toBe("user-2"); + }); + + it("filters by actor", async () => { + const entries = await queryAuditEntries({ actor: "user-1" }, 10, { baseDir: tempDir }); + expect(entries.length).toBe(1); + expect(entries[0].actor.id).toBe("user-1"); + }); + }); + + describe("disabled state", () => { + it("does not write when disabled", async () => { + stopAuditLog(); + initAuditLog({ enabled: false, baseDir: tempDir }); + + emitAuditEvent({ + type: "auth.login", + actor: { type: "user", id: "test" }, + outcome: "success", + }); + + await new Promise((resolve) => setTimeout(resolve, 50)); + + const entries = await readRecentAuditEntries(10, { baseDir: tempDir }); + // Should only have entries from previous tests, not the new one + expect(entries.filter((e) => e.actor.id === "test").length).toBe(0); + }); + }); + + describe("rotateAuditLogs", () => { + it("removes files older than retention period", async () => { + // Create old audit files + const oldDate = new Date(); + oldDate.setDate(oldDate.getDate() - 10); + const oldDateStr = oldDate.toISOString().slice(0, 10); + const oldFilePath = path.join(tempDir, `audit.${oldDateStr}.jsonl`); + await fs.writeFile(oldFilePath, '{"test": true}\n'); + + // Run rotation with 7-day retention + await rotateAuditLogs({ enabled: true, baseDir: tempDir, retentionDays: 7 }); + + // Old file should be deleted + const exists = await fs + .stat(oldFilePath) + .then(() => true) + .catch(() => false); + expect(exists).toBe(false); + }); + + it("keeps files within retention period", async () => { + // Create recent audit file + const recentDate = new Date(); + recentDate.setDate(recentDate.getDate() - 3); + const recentDateStr = recentDate.toISOString().slice(0, 10); + const recentFilePath = path.join(tempDir, `audit.${recentDateStr}.jsonl`); + await fs.writeFile(recentFilePath, '{"test": true}\n'); + + // Run rotation with 7-day retention + await rotateAuditLogs({ enabled: true, baseDir: tempDir, retentionDays: 7 }); + + // Recent file should still exist + const exists = await fs + .stat(recentFilePath) + .then(() => true) + .catch(() => false); + expect(exists).toBe(true); + }); + }); +}); diff --git a/src/security/audit-log.ts b/src/security/audit-log.ts new file mode 100644 index 000000000..b8d951b1b --- /dev/null +++ b/src/security/audit-log.ts @@ -0,0 +1,451 @@ +/** + * Persistent audit logging for enterprise compliance and debugging. + * + * Audit events are written to a JSONL file at ~/.clawdbot/audit.jsonl + * with automatic daily rotation (keeps 7 days of history). + */ + +import fs from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { resolveStateDir } from "../config/paths.js"; +import { getCurrentTraceId } from "../observability/trace-context.js"; + +/** Audit event types for different security-relevant actions. */ +export type AuditEventType = + | "auth.login" + | "auth.logout" + | "auth.failure" + | "pairing.request" + | "pairing.approve" + | "pairing.reject" + | "pairing.revoke" + | "exec.request" + | "exec.approve" + | "exec.reject" + | "config.change" + | "gateway.start" + | "gateway.stop" + | "rbac.denied" + | "session.create" + | "session.destroy"; + +/** Actor who performed the action. */ +export type AuditActor = { + type: "user" | "system" | "agent" | "device"; + id: string; + channel?: string; + deviceId?: string; + remoteIp?: string; +}; + +/** Target of the action (optional). */ +export type AuditTarget = { + type: string; + id: string; + [key: string]: unknown; +}; + +/** Outcome of the action. */ +export type AuditOutcome = "success" | "failure" | "denied"; + +/** Full audit log entry structure. */ +export type AuditLogEntry = { + /** ISO timestamp of the event. */ + ts: string; + /** Unique event ID. */ + eventId: string; + /** Type of audit event. */ + type: AuditEventType; + /** Who performed the action. */ + actor: AuditActor; + /** What was affected (optional). */ + target?: AuditTarget; + /** Result of the action. */ + outcome: AuditOutcome; + /** Trace ID for distributed tracing correlation. */ + traceId?: string; + /** Additional metadata. */ + metadata?: Record; +}; + +/** Input for creating an audit event (without auto-generated fields). */ +export type AuditEventInput = Omit; + +/** Configuration for the audit logger. */ +export type AuditLogConfig = { + /** Enable audit logging (default: true). */ + enabled?: boolean; + /** Base directory for audit logs (default: ~/.clawdbot). */ + baseDir?: string; + /** Number of days to keep audit logs (default: 7). */ + retentionDays?: number; +}; + +const DEFAULT_RETENTION_DAYS = 7; +const AUDIT_FILENAME = "audit.jsonl"; + +let writeQueue: Promise = Promise.resolve(); + +function resolveAuditPaths(baseDir?: string) { + const root = baseDir ?? resolveStateDir(); + return { + dir: root, + currentPath: path.join(root, AUDIT_FILENAME), + }; +} + +function getDateSuffix(date: Date = new Date()): string { + return date.toISOString().slice(0, 10); // YYYY-MM-DD +} + +function getArchivePath(basePath: string, date: Date): string { + const suffix = getDateSuffix(date); + return basePath.replace(/\.jsonl$/, `.${suffix}.jsonl`); +} + +/** + * Append an audit entry to the log file atomically. + */ +async function appendAuditEntry(entry: AuditLogEntry, baseDir?: string): Promise { + const { dir, currentPath } = resolveAuditPaths(baseDir); + + // Ensure directory exists + await fs.mkdir(dir, { recursive: true }); + + // Serialize entry + const line = JSON.stringify(entry) + "\n"; + + // Append to file (create if doesn't exist) + await fs.appendFile(currentPath, line, { encoding: "utf8", mode: 0o600 }); +} + +/** + * Rotate old audit log files based on date. + * Called periodically to archive yesterday's logs and clean up old files. + */ +export async function rotateAuditLogs(config?: AuditLogConfig): Promise { + const { dir, currentPath } = resolveAuditPaths(config?.baseDir); + const retentionDays = config?.retentionDays ?? DEFAULT_RETENTION_DAYS; + + try { + // Ensure directory exists before listing + await fs.mkdir(dir, { recursive: true }).catch(() => undefined); + + // List all audit files + const files = await fs.readdir(dir).catch(() => [] as string[]); + const auditFiles = files.filter((f) => f.startsWith("audit.") && f.endsWith(".jsonl")); + + // Calculate cutoff date + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - retentionDays); + const cutoffStr = getDateSuffix(cutoffDate); + + // Delete files older than retention period + for (const file of auditFiles) { + const match = file.match(/audit\.(\d{4}-\d{2}-\d{2})\.jsonl/); + if (match && match[1] < cutoffStr) { + const filePath = path.join(dir, file); + await fs.unlink(filePath).catch(() => undefined); + } + } + + // Archive current log if it's from a previous day + const todayStr = getDateSuffix(); + const stats = await fs.stat(currentPath).catch(() => null); + if (!stats) return; // No current log to archive + + const content = await fs.readFile(currentPath, "utf8").catch(() => ""); + if (!content.trim()) return; + + const lines = content.trim().split("\n"); + const firstLine = lines[0]; + if (!firstLine) return; + + try { + const firstEntry = JSON.parse(firstLine) as { ts?: string }; + const firstTs = firstEntry.ts; + if (firstTs && firstTs.slice(0, 10) < todayStr) { + // Archive the old log + const archiveDate = new Date(firstTs); + const archivePath = getArchivePath(currentPath, archiveDate); + await fs.rename(currentPath, archivePath).catch(() => undefined); + } + } catch { + // Ignore parse errors, keep the file + } + } catch { + // Rotation is best-effort + } +} + +/** + * Queue an audit write to ensure sequential writes. + */ +function queueAuditWrite(fn: () => Promise): Promise { + const prev = writeQueue; + let release: (() => void) | undefined; + writeQueue = new Promise((resolve) => { + release = resolve; + }); + return prev.then(fn).finally(() => release?.()); +} + +// Global state +let auditConfig: AuditLogConfig | null = null; +let rotationInterval: ReturnType | null = null; + +/** + * Initialize the audit logger with configuration. + * Should be called at gateway startup. + */ +export function initAuditLog(config?: AuditLogConfig): void { + auditConfig = config ?? { enabled: true }; + + if (auditConfig.enabled === false) return; + + // Run rotation on startup + void rotateAuditLogs(auditConfig); + + // Schedule daily rotation (check every hour) + if (rotationInterval) { + clearInterval(rotationInterval); + } + rotationInterval = setInterval( + () => { + void rotateAuditLogs(auditConfig ?? undefined); + }, + 60 * 60 * 1000, + ); // Every hour +} + +/** + * Stop the audit logger (e.g., on gateway shutdown). + */ +export function stopAuditLog(): void { + if (rotationInterval) { + clearInterval(rotationInterval); + rotationInterval = null; + } +} + +/** + * Emit an audit event. + * This is the main API for logging audit events. + */ +export function emitAuditEvent(event: AuditEventInput): void { + if (auditConfig?.enabled === false) return; + + const traceId = getCurrentTraceId(); + const entry: AuditLogEntry = { + ...event, + ts: new Date().toISOString(), + eventId: randomUUID(), + ...(traceId !== "no-trace" ? { traceId } : {}), + }; + + // Queue the write (fire and forget, but sequential) + void queueAuditWrite(() => appendAuditEntry(entry, auditConfig?.baseDir)); +} + +/** + * Convenience functions for common audit events. + */ + +export function auditAuthLogin(params: { + actor: AuditActor; + method?: string; + outcome?: AuditOutcome; +}): void { + emitAuditEvent({ + type: "auth.login", + actor: params.actor, + outcome: params.outcome ?? "success", + metadata: params.method ? { method: params.method } : undefined, + }); +} + +export function auditAuthFailure(params: { actor: AuditActor; reason: string }): void { + emitAuditEvent({ + type: "auth.failure", + actor: params.actor, + outcome: "failure", + metadata: { reason: params.reason }, + }); +} + +export function auditPairingRequest(params: { actor: AuditActor; target: AuditTarget }): void { + emitAuditEvent({ + type: "pairing.request", + actor: params.actor, + target: params.target, + outcome: "success", + }); +} + +export function auditPairingApprove(params: { actor: AuditActor; target: AuditTarget }): void { + emitAuditEvent({ + type: "pairing.approve", + actor: params.actor, + target: params.target, + outcome: "success", + }); +} + +export function auditPairingReject(params: { + actor: AuditActor; + target: AuditTarget; + reason?: string; +}): void { + emitAuditEvent({ + type: "pairing.reject", + actor: params.actor, + target: params.target, + outcome: "denied", + metadata: params.reason ? { reason: params.reason } : undefined, + }); +} + +export function auditExecRequest(params: { + actor: AuditActor; + target: AuditTarget; + command: string; +}): void { + emitAuditEvent({ + type: "exec.request", + actor: params.actor, + target: params.target, + outcome: "success", + metadata: { command: params.command }, + }); +} + +export function auditExecApprove(params: { + actor: AuditActor; + target: AuditTarget; + command: string; +}): void { + emitAuditEvent({ + type: "exec.approve", + actor: params.actor, + target: params.target, + outcome: "success", + metadata: { command: params.command }, + }); +} + +export function auditExecReject(params: { + actor: AuditActor; + target: AuditTarget; + command: string; + reason?: string; +}): void { + emitAuditEvent({ + type: "exec.reject", + actor: params.actor, + target: params.target, + outcome: "denied", + metadata: { command: params.command, ...(params.reason ? { reason: params.reason } : {}) }, + }); +} + +export function auditConfigChange(params: { + actor: AuditActor; + changes: Record; +}): void { + emitAuditEvent({ + type: "config.change", + actor: params.actor, + outcome: "success", + metadata: { changes: params.changes }, + }); +} + +export function auditGatewayStart(params: { + actor: AuditActor; + metadata?: Record; +}): void { + emitAuditEvent({ + type: "gateway.start", + actor: params.actor, + outcome: "success", + metadata: params.metadata, + }); +} + +export function auditGatewayStop(params: { actor: AuditActor; reason?: string }): void { + emitAuditEvent({ + type: "gateway.stop", + actor: params.actor, + outcome: "success", + metadata: params.reason ? { reason: params.reason } : undefined, + }); +} + +export function auditRbacDenied(params: { + actor: AuditActor; + action: string; + resource?: string; + reason?: string; +}): void { + emitAuditEvent({ + type: "rbac.denied", + actor: params.actor, + outcome: "denied", + metadata: { + action: params.action, + ...(params.resource ? { resource: params.resource } : {}), + ...(params.reason ? { reason: params.reason } : {}), + }, + }); +} + +/** + * Read recent audit entries (for debugging/testing). + * Returns entries from most recent first. + */ +export async function readRecentAuditEntries( + limit: number = 100, + config?: AuditLogConfig, +): Promise { + const { currentPath } = resolveAuditPaths(config?.baseDir); + + try { + const content = await fs.readFile(currentPath, "utf8"); + const lines = content.trim().split("\n").filter(Boolean); + const entries: AuditLogEntry[] = []; + + // Read from end (most recent) + for (let i = lines.length - 1; i >= 0 && entries.length < limit; i--) { + try { + entries.push(JSON.parse(lines[i]) as AuditLogEntry); + } catch { + // Skip malformed lines + } + } + + return entries; + } catch { + return []; + } +} + +/** + * Query audit entries by type (for debugging/testing). + */ +export async function queryAuditEntries( + filter: { type?: AuditEventType; actor?: string; outcome?: AuditOutcome }, + limit: number = 100, + config?: AuditLogConfig, +): Promise { + const entries = await readRecentAuditEntries(limit * 10, config); + + return entries + .filter((entry) => { + if (filter.type && entry.type !== filter.type) return false; + if (filter.actor && entry.actor.id !== filter.actor) return false; + if (filter.outcome && entry.outcome !== filter.outcome) return false; + return true; + }) + .slice(0, limit); +} diff --git a/src/security/rbac.test.ts b/src/security/rbac.test.ts new file mode 100644 index 000000000..7559b05fb --- /dev/null +++ b/src/security/rbac.test.ts @@ -0,0 +1,375 @@ +import { describe, expect, it, beforeEach, afterEach } from "vitest"; +import type { ClawdbotConfig } from "../config/types.js"; +import { + isRbacEnabled, + getRoleForUser, + hasPermission, + canExecuteCommand, + canAccessAgent, + canUseTool, + canApproveExec, + getUserPermissionSummary, +} from "./rbac.js"; +import { initAuditLog, stopAuditLog } from "./audit-log.js"; + +describe("rbac", () => { + beforeEach(() => { + // Initialize audit log in disabled mode to prevent file writes during tests + initAuditLog({ enabled: false }); + }); + + afterEach(() => { + stopAuditLog(); + }); + + describe("isRbacEnabled", () => { + it("returns false when config is undefined", () => { + expect(isRbacEnabled()).toBe(false); + }); + + it("returns false when rbac is not configured", () => { + expect(isRbacEnabled({})).toBe(false); + }); + + it("returns false when rbac.enabled is false", () => { + expect(isRbacEnabled({ rbac: { enabled: false } })).toBe(false); + }); + + it("returns true when rbac.enabled is true", () => { + expect(isRbacEnabled({ rbac: { enabled: true } })).toBe(true); + }); + }); + + describe("getRoleForUser", () => { + it("returns null when RBAC is disabled", () => { + const config: ClawdbotConfig = { rbac: { enabled: false } }; + expect(getRoleForUser("user-1", config)).toBeNull(); + }); + + it("returns assigned role when user is assigned", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "admin" }, + }, + }; + const result = getRoleForUser("user-1", config); + expect(result?.roleId).toBe("admin"); + expect(result?.role.name).toBe("Administrator"); + }); + + it("returns default role when user is not assigned", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + defaultRole: "viewer", + }, + }; + const result = getRoleForUser("user-1", config); + expect(result?.roleId).toBe("viewer"); + }); + + it("returns null when no assignment and no default role", () => { + const config: ClawdbotConfig = { + rbac: { enabled: true }, + }; + expect(getRoleForUser("user-1", config)).toBeNull(); + }); + + it("uses custom role definitions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + roles: { + custom: { + name: "Custom Role", + permissions: ["exec"], + }, + }, + assignments: { "user-1": "custom" }, + }, + }; + const result = getRoleForUser("user-1", config); + expect(result?.roleId).toBe("custom"); + expect(result?.role.name).toBe("Custom Role"); + }); + }); + + describe("hasPermission", () => { + it("allows everything when RBAC is disabled", () => { + const config: ClawdbotConfig = { rbac: { enabled: false } }; + const result = hasPermission("user-1", "exec.elevated", config); + expect(result.allowed).toBe(true); + expect(result.reason).toBe("rbac-disabled"); + }); + + it("denies when user has no role", () => { + const config: ClawdbotConfig = { rbac: { enabled: true } }; + const result = hasPermission("user-1", "exec", config); + expect(result.allowed).toBe(false); + expect(result.reason).toBe("no-role-assigned"); + }); + + it("allows when user has the permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = hasPermission("user-1", "exec", config); + expect(result.allowed).toBe(true); + expect(result.role).toBe("user"); + }); + + it("denies when user lacks the permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = hasPermission("user-1", "exec.elevated", config); + expect(result.allowed).toBe(false); + expect(result.reason).toBe("permission-denied"); + }); + + it("admin role grants all permissions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "admin" }, + }, + }; + expect(hasPermission("user-1", "exec", config).allowed).toBe(true); + expect(hasPermission("user-1", "exec.elevated", config).allowed).toBe(true); + expect(hasPermission("user-1", "exec.approve", config).allowed).toBe(true); + expect(hasPermission("user-1", "admin", config).allowed).toBe(true); + }); + }); + + describe("canExecuteCommand", () => { + it("allows basic commands with exec permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = canExecuteCommand("user-1", "ls -la", config); + expect(result.allowed).toBe(true); + }); + + it("denies elevated commands without exec.elevated permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = canExecuteCommand("user-1", "sudo rm -rf /tmp/test", config); + expect(result.allowed).toBe(false); + }); + + it("allows elevated commands with exec.elevated permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "admin" }, + }, + }; + const result = canExecuteCommand("user-1", "sudo apt update", config); + expect(result.allowed).toBe(true); + }); + + it("detects sudo in various positions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + expect(canExecuteCommand("user-1", "echo test | sudo tee /etc/file", config).allowed).toBe( + false, + ); + expect(canExecuteCommand("user-1", "cd /tmp && sudo rm -rf *", config).allowed).toBe(false); + }); + }); + + describe("canAccessAgent", () => { + it("allows all agents when no restrictions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = canAccessAgent("user-1", "any-agent", config); + expect(result.allowed).toBe(true); + expect(result.reason).toBe("no-agent-restrictions"); + }); + + it("denies access to restricted agents", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + roles: { + restricted: { + name: "Restricted", + permissions: ["exec"], + agents: ["agent-a", "agent-b"], + }, + }, + assignments: { "user-1": "restricted" }, + }, + }; + expect(canAccessAgent("user-1", "agent-a", config).allowed).toBe(true); + expect(canAccessAgent("user-1", "agent-c", config).allowed).toBe(false); + }); + + it("admin bypasses agent restrictions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "admin" }, + }, + }; + const result = canAccessAgent("user-1", "any-agent", config); + expect(result.allowed).toBe(true); + expect(result.reason).toBe("admin-role"); + }); + }); + + describe("canUseTool", () => { + it("allows all tools when no restrictions", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = canUseTool("user-1", "any-tool", config); + expect(result.allowed).toBe(true); + }); + + it("denies tools on deny list", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + roles: { + limited: { + name: "Limited", + permissions: ["exec"], + tools: { deny: ["dangerous-tool"] }, + }, + }, + assignments: { "user-1": "limited" }, + }, + }; + expect(canUseTool("user-1", "safe-tool", config).allowed).toBe(true); + expect(canUseTool("user-1", "dangerous-tool", config).allowed).toBe(false); + }); + + it("only allows tools on allow list when specified", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + roles: { + limited: { + name: "Limited", + permissions: ["exec"], + tools: { allow: ["read", "search"] }, + }, + }, + assignments: { "user-1": "limited" }, + }, + }; + expect(canUseTool("user-1", "read", config).allowed).toBe(true); + expect(canUseTool("user-1", "write", config).allowed).toBe(false); + }); + + it("deny takes precedence over allow", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + roles: { + mixed: { + name: "Mixed", + permissions: ["exec"], + tools: { allow: ["tool-a", "tool-b"], deny: ["tool-b"] }, + }, + }, + assignments: { "user-1": "mixed" }, + }, + }; + expect(canUseTool("user-1", "tool-a", config).allowed).toBe(true); + expect(canUseTool("user-1", "tool-b", config).allowed).toBe(false); + }); + + it("read-only role cannot use tools", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "viewer" }, + }, + }; + const result = canUseTool("user-1", "any-tool", config); + expect(result.allowed).toBe(false); + expect(result.reason).toBe("read-only-role"); + }); + }); + + describe("canApproveExec", () => { + it("allows with exec.approve permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "operator" }, + }, + }; + const result = canApproveExec("user-1", config); + expect(result.allowed).toBe(true); + }); + + it("denies without exec.approve permission", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "user" }, + }, + }; + const result = canApproveExec("user-1", config); + expect(result.allowed).toBe(false); + }); + }); + + describe("getUserPermissionSummary", () => { + it("returns disabled when RBAC is off", () => { + const config: ClawdbotConfig = { rbac: { enabled: false } }; + const summary = getUserPermissionSummary("user-1", config); + expect(summary.enabled).toBe(false); + expect(summary.roleId).toBeUndefined(); + }); + + it("returns role details when RBAC is on", () => { + const config: ClawdbotConfig = { + rbac: { + enabled: true, + assignments: { "user-1": "admin" }, + }, + }; + const summary = getUserPermissionSummary("user-1", config); + expect(summary.enabled).toBe(true); + expect(summary.roleId).toBe("admin"); + expect(summary.roleName).toBe("Administrator"); + expect(summary.permissions).toContain("admin"); + }); + + it("returns enabled with no role when user unassigned", () => { + const config: ClawdbotConfig = { rbac: { enabled: true } }; + const summary = getUserPermissionSummary("user-1", config); + expect(summary.enabled).toBe(true); + expect(summary.roleId).toBeUndefined(); + }); + }); +}); diff --git a/src/security/rbac.ts b/src/security/rbac.ts new file mode 100644 index 000000000..30c547309 --- /dev/null +++ b/src/security/rbac.ts @@ -0,0 +1,345 @@ +/** + * RBAC (Role-Based Access Control) engine for enterprise deployments. + * + * Provides permission checking based on user role assignments. + * When RBAC is disabled, all permissions are granted (backwards compatible). + */ + +import type { ClawdbotConfig } from "../config/types.js"; +import type { RbacConfig, RbacPermission, RbacRoleDefinition } from "../config/types.rbac.js"; +import { auditRbacDenied } from "./audit-log.js"; + +/** Result of a permission check. */ +export type RbacCheckResult = { + allowed: boolean; + role?: string; + reason?: string; +}; + +/** Built-in role definitions for when no custom roles are configured. */ +const DEFAULT_ROLES: Record = { + admin: { + name: "Administrator", + description: "Full access to all features", + permissions: ["exec", "exec.elevated", "exec.approve", "admin"], + }, + operator: { + name: "Operator", + description: "Can execute commands and approve requests", + permissions: ["exec", "exec.approve"], + }, + user: { + name: "User", + description: "Can execute basic commands", + permissions: ["exec"], + }, + viewer: { + name: "Viewer", + description: "Read-only access", + permissions: ["read-only"], + }, +}; + +/** + * Check if RBAC is enabled in the config. + */ +export function isRbacEnabled(config?: ClawdbotConfig): boolean { + return config?.rbac?.enabled === true; +} + +/** + * Get the RBAC config with defaults applied. + */ +function getRbacConfig(config?: ClawdbotConfig): RbacConfig { + return { + enabled: false, + roles: DEFAULT_ROLES, + assignments: {}, + ...config?.rbac, + }; +} + +/** + * Get all role definitions (custom roles merged with defaults). + */ +function getRoles(rbacConfig: RbacConfig): Record { + return { + ...DEFAULT_ROLES, + ...rbacConfig.roles, + }; +} + +/** + * Get the role definition for a user. + * Returns the assigned role, default role, or undefined if no role applies. + */ +export function getRoleForUser( + senderId: string, + config?: ClawdbotConfig, +): { roleId: string; role: RbacRoleDefinition } | null { + const rbacConfig = getRbacConfig(config); + const roles = getRoles(rbacConfig); + + // Check explicit assignment first + const assignedRoleId = rbacConfig.assignments?.[senderId]; + if (assignedRoleId && roles[assignedRoleId]) { + return { roleId: assignedRoleId, role: roles[assignedRoleId] }; + } + + // Fall back to default role + const defaultRoleId = rbacConfig.defaultRole; + if (defaultRoleId && roles[defaultRoleId]) { + return { roleId: defaultRoleId, role: roles[defaultRoleId] }; + } + + return null; +} + +/** + * Check if a user has a specific permission. + */ +export function hasPermission( + senderId: string, + permission: RbacPermission, + config?: ClawdbotConfig, +): RbacCheckResult { + // If RBAC is disabled, allow everything + if (!isRbacEnabled(config)) { + return { allowed: true, reason: "rbac-disabled" }; + } + + const userRole = getRoleForUser(senderId, config); + if (!userRole) { + return { allowed: false, reason: "no-role-assigned" }; + } + + const { roleId, role } = userRole; + + // Admin permission grants all other permissions + if (role.permissions.includes("admin")) { + return { allowed: true, role: roleId, reason: "admin-role" }; + } + + // Check for the specific permission + if (role.permissions.includes(permission)) { + return { allowed: true, role: roleId }; + } + + // Special case: exec.elevated requires explicit permission, but exec allows basic commands + if (permission === "exec" && role.permissions.includes("exec.elevated")) { + return { allowed: true, role: roleId, reason: "elevated-includes-exec" }; + } + + return { allowed: false, role: roleId, reason: "permission-denied" }; +} + +/** + * Check if a user can execute a command. + * Elevated commands (sudo, etc.) require exec.elevated permission. + */ +export function canExecuteCommand( + senderId: string, + command: string, + config?: ClawdbotConfig, +): RbacCheckResult { + // If RBAC is disabled, allow everything + if (!isRbacEnabled(config)) { + return { allowed: true, reason: "rbac-disabled" }; + } + + const isElevated = isElevatedCommand(command); + const requiredPermission: RbacPermission = isElevated ? "exec.elevated" : "exec"; + + const result = hasPermission(senderId, requiredPermission, config); + + if (!result.allowed) { + // Audit the denial + auditRbacDenied({ + actor: { type: "user", id: senderId }, + action: requiredPermission, + resource: command, + reason: result.reason, + }); + } + + return result; +} + +/** + * Check if a command requires elevated permissions. + */ +function isElevatedCommand(command: string): boolean { + const trimmed = command.trim().toLowerCase(); + + // Check for sudo prefix + if (trimmed.startsWith("sudo ")) return true; + + // Check for common elevated operations + const elevatedPatterns = [ + /^su\s/, + /^doas\s/, + /^pkexec\s/, + /^runas\s/, + /\|\s*sudo\s/, + /;\s*sudo\s/, + /&&\s*sudo\s/, + ]; + + return elevatedPatterns.some((pattern) => pattern.test(trimmed)); +} + +/** + * Check if a user can access a specific agent. + */ +export function canAccessAgent( + senderId: string, + agentId: string, + config?: ClawdbotConfig, +): RbacCheckResult { + // If RBAC is disabled, allow everything + if (!isRbacEnabled(config)) { + return { allowed: true, reason: "rbac-disabled" }; + } + + const userRole = getRoleForUser(senderId, config); + if (!userRole) { + return { allowed: false, reason: "no-role-assigned" }; + } + + const { roleId, role } = userRole; + + // Admin can access all agents + if (role.permissions.includes("admin")) { + return { allowed: true, role: roleId, reason: "admin-role" }; + } + + // If no agent restrictions, allow all + if (!role.agents || role.agents.length === 0) { + return { allowed: true, role: roleId, reason: "no-agent-restrictions" }; + } + + // Check if agent is in the allowed list + if (role.agents.includes(agentId)) { + return { allowed: true, role: roleId }; + } + + // Audit the denial + auditRbacDenied({ + actor: { type: "user", id: senderId }, + action: "agent.access", + resource: agentId, + reason: "agent-not-allowed", + }); + + return { allowed: false, role: roleId, reason: "agent-not-allowed" }; +} + +/** + * Check if a user can use a specific tool. + */ +export function canUseTool( + senderId: string, + toolName: string, + config?: ClawdbotConfig, +): RbacCheckResult { + // If RBAC is disabled, allow everything + if (!isRbacEnabled(config)) { + return { allowed: true, reason: "rbac-disabled" }; + } + + const userRole = getRoleForUser(senderId, config); + if (!userRole) { + return { allowed: false, reason: "no-role-assigned" }; + } + + const { roleId, role } = userRole; + + // Admin can use all tools + if (role.permissions.includes("admin")) { + return { allowed: true, role: roleId, reason: "admin-role" }; + } + + // Read-only users cannot use tools that modify state + if (role.permissions.includes("read-only") && !role.permissions.includes("exec")) { + auditRbacDenied({ + actor: { type: "user", id: senderId }, + action: "tool.use", + resource: toolName, + reason: "read-only-role", + }); + return { allowed: false, role: roleId, reason: "read-only-role" }; + } + + const toolAccess = role.tools; + if (!toolAccess) { + // No tool restrictions + return { allowed: true, role: roleId, reason: "no-tool-restrictions" }; + } + + // Check deny list first (takes precedence) + if (toolAccess.deny?.includes(toolName)) { + auditRbacDenied({ + actor: { type: "user", id: senderId }, + action: "tool.use", + resource: toolName, + reason: "tool-denied", + }); + return { allowed: false, role: roleId, reason: "tool-denied" }; + } + + // Check allow list (if specified) + if (toolAccess.allow && toolAccess.allow.length > 0) { + if (!toolAccess.allow.includes(toolName)) { + auditRbacDenied({ + actor: { type: "user", id: senderId }, + action: "tool.use", + resource: toolName, + reason: "tool-not-allowed", + }); + return { allowed: false, role: roleId, reason: "tool-not-allowed" }; + } + } + + return { allowed: true, role: roleId }; +} + +/** + * Check if a user can approve exec requests. + */ +export function canApproveExec(senderId: string, config?: ClawdbotConfig): RbacCheckResult { + return hasPermission(senderId, "exec.approve", config); +} + +/** + * Get a summary of a user's permissions for debugging/display. + */ +export function getUserPermissionSummary( + senderId: string, + config?: ClawdbotConfig, +): { + enabled: boolean; + roleId?: string; + roleName?: string; + permissions?: RbacPermission[]; + agents?: string[]; + tools?: { allow?: string[]; deny?: string[] }; +} { + if (!isRbacEnabled(config)) { + return { enabled: false }; + } + + const userRole = getRoleForUser(senderId, config); + if (!userRole) { + return { enabled: true }; + } + + const { roleId, role } = userRole; + return { + enabled: true, + roleId, + roleName: role.name, + permissions: role.permissions, + agents: role.agents, + tools: role.tools, + }; +}