7a900e90f5
Molt is a self-healing update system for self-hosted Clawdbot instances. It handles nightly updates from upstream with automatic rollback if something breaks. Key features: - Pulls from upstream, installs deps, builds, restarts gateway - Health check with stability window (catches crash loops) - Automatic rollback to pre-update commit on failure - Module manifest to define what *you* care about (personalized health checks) - Changelog generation for morning reports - Agentic recovery philosophy: capture context, let the AI fix edge cases Files: - docs/molt.md: Full PRD with architecture, config schema, and rationale - scripts/molt.sh: Reference implementation (bash) This is an RFC seeking feedback on approach and integration strategy. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
314 lines
9.0 KiB
Bash
Executable File
314 lines
9.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# molt.sh - Self-healing update script for Clawdbot
|
|
#
|
|
# Usage: ./molt.sh [--dry-run]
|
|
#
|
|
# This script:
|
|
# 1. Checks for updates from upstream
|
|
# 2. Pulls, installs, builds, restarts
|
|
# 3. Verifies the gateway comes back healthy
|
|
# 4. Rolls back if something breaks
|
|
#
|
|
# Designed to be run via Clawdbot cron or system cron.
|
|
|
|
set -euo pipefail
|
|
|
|
# === Configuration ===
|
|
MOLT_DIR="${HOME}/.clawdbot/molt"
|
|
CLAWDBOT_DIR="${HOME}/clawd"
|
|
WORKSPACE_DIR="${HOME}/maja-workspace"
|
|
REMOTE="upstream"
|
|
BRANCH="main"
|
|
|
|
# Health check timing
|
|
STARTUP_TIMEOUT=60
|
|
STABILITY_WINDOW=30
|
|
PING_TIMEOUT=5
|
|
|
|
# === Parse args ===
|
|
DRY_RUN=false
|
|
if [[ "${1:-}" == "--dry-run" ]]; then
|
|
DRY_RUN=true
|
|
echo "[molt] DRY RUN MODE - no changes will be made"
|
|
fi
|
|
|
|
# === Helper functions ===
|
|
timestamp() {
|
|
date -u +"%Y-%m-%dT%H:%M:%SZ"
|
|
}
|
|
|
|
log() {
|
|
echo "[molt $(timestamp)] $*"
|
|
}
|
|
|
|
die() {
|
|
log "ERROR: $*"
|
|
exit 1
|
|
}
|
|
|
|
# === Phase 0: Preflight ===
|
|
log "=== Phase 0: Preflight ==="
|
|
|
|
# Acquire lock
|
|
LOCK_DIR="${MOLT_DIR}/lock"
|
|
if ! mkdir "$LOCK_DIR" 2>/dev/null; then
|
|
log "Another molt run in progress (lock exists), exiting"
|
|
exit 0
|
|
fi
|
|
trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT
|
|
|
|
# Change to repo directory
|
|
cd "$CLAWDBOT_DIR" || die "Cannot cd to $CLAWDBOT_DIR"
|
|
|
|
# Fetch from remote
|
|
log "Fetching from ${REMOTE}/${BRANCH}..."
|
|
git fetch "$REMOTE" "$BRANCH" --quiet
|
|
|
|
# Compare HEAD with remote
|
|
CURRENT_HEAD=$(git rev-parse HEAD)
|
|
REMOTE_HEAD=$(git rev-parse "${REMOTE}/${BRANCH}")
|
|
|
|
log "Current HEAD: ${CURRENT_HEAD:0:8}"
|
|
log "Remote HEAD: ${REMOTE_HEAD:0:8}"
|
|
|
|
if [[ "$CURRENT_HEAD" == "$REMOTE_HEAD" ]]; then
|
|
log "Already up to date"
|
|
|
|
# Write minimal changelog
|
|
cat > "${WORKSPACE_DIR}/update-changelog.md" << EOF
|
|
# Clawdbot Update Changelog
|
|
Generated: $(timestamp)
|
|
|
|
## Status
|
|
No changes - already up to date.
|
|
|
|
Current version: ${CURRENT_HEAD:0:8}
|
|
EOF
|
|
|
|
exit 0
|
|
fi
|
|
|
|
# Check for clean workdir (warn but continue if dirty)
|
|
REQUIRE_CLEAN=${REQUIRE_CLEAN_WORKDIR:-false}
|
|
if [[ -n "$(git status --porcelain)" ]]; then
|
|
log "Working directory not clean:"
|
|
git status --short
|
|
if [[ "$REQUIRE_CLEAN" == "true" ]]; then
|
|
die "Aborting - commit or stash your changes first"
|
|
else
|
|
log "WARNING: Continuing with dirty workdir (REQUIRE_CLEAN_WORKDIR=false)"
|
|
log "WARNING: Local changes may be lost during rollback!"
|
|
fi
|
|
fi
|
|
|
|
# Save state for potential rollback
|
|
log "Saving pre-update state..."
|
|
echo "$CURRENT_HEAD" > "${MOLT_DIR}/pre-update-head"
|
|
cp pnpm-lock.yaml "${MOLT_DIR}/pre-update-lock.yaml" 2>/dev/null || true
|
|
git log --oneline -1 > "${MOLT_DIR}/pre-update-info"
|
|
|
|
# Count incoming commits
|
|
COMMIT_COUNT=$(git rev-list --count "${CURRENT_HEAD}..${REMOTE_HEAD}")
|
|
log "Incoming commits: $COMMIT_COUNT"
|
|
|
|
if $DRY_RUN; then
|
|
log "[DRY RUN] Would update from ${CURRENT_HEAD:0:8} to ${REMOTE_HEAD:0:8}"
|
|
log "[DRY RUN] Incoming changes:"
|
|
git log --oneline "${CURRENT_HEAD}..${REMOTE_HEAD}"
|
|
exit 0
|
|
fi
|
|
|
|
# === Phase 1: Update ===
|
|
log "=== Phase 1: Update ==="
|
|
|
|
# Merge (fast-forward only)
|
|
log "Merging ${REMOTE}/${BRANCH}..."
|
|
if ! git merge "${REMOTE}/${BRANCH}" --ff-only; then
|
|
die "Merge failed - history has diverged. Manual intervention required."
|
|
fi
|
|
|
|
NEW_HEAD=$(git rev-parse HEAD)
|
|
log "Updated to: ${NEW_HEAD:0:8}"
|
|
|
|
# Install dependencies
|
|
log "Installing dependencies..."
|
|
if ! pnpm install --frozen-lockfile --prefer-offline 2>&1; then
|
|
log "pnpm install failed, attempting recovery..."
|
|
# Recovery: try without --frozen-lockfile in case lockfile is out of sync
|
|
if ! pnpm install --prefer-offline 2>&1; then
|
|
log "pnpm install still failed, rolling back..."
|
|
source "${MOLT_DIR}/molt.sh" --rollback-internal
|
|
die "pnpm install failed"
|
|
fi
|
|
fi
|
|
|
|
# Build
|
|
log "Building..."
|
|
if ! pnpm build 2>&1; then
|
|
log "Build failed, rolling back..."
|
|
git checkout "$CURRENT_HEAD" --quiet
|
|
pnpm install --frozen-lockfile --prefer-offline 2>&1 || true
|
|
die "Build failed"
|
|
fi
|
|
|
|
# Generate changelog before restart
|
|
log "Generating changelog..."
|
|
cat > "${WORKSPACE_DIR}/update-changelog.md" << EOF
|
|
# Clawdbot Update Changelog
|
|
Generated: $(timestamp)
|
|
|
|
## Summary
|
|
Updated from \`${CURRENT_HEAD:0:8}\` to \`${NEW_HEAD:0:8}\` ($COMMIT_COUNT commits)
|
|
|
|
## Commits
|
|
$(git log --oneline "${CURRENT_HEAD}..${NEW_HEAD}")
|
|
|
|
## Changed Files
|
|
$(git diff --stat "${CURRENT_HEAD}..${NEW_HEAD}" | tail -20)
|
|
EOF
|
|
|
|
# Restart gateway
|
|
log "Restarting gateway..."
|
|
systemctl --user restart clawdbot-gateway.service
|
|
|
|
# === Phase 2: Verify ===
|
|
log "=== Phase 2: Verify ==="
|
|
|
|
# Wait for gateway to respond
|
|
log "Waiting for gateway to come up (timeout: ${STARTUP_TIMEOUT}s)..."
|
|
waited=0
|
|
gateway_up=false
|
|
|
|
while [[ $waited -lt $STARTUP_TIMEOUT ]]; do
|
|
if systemctl --user is-active --quiet clawdbot-gateway.service; then
|
|
# Service is running, check RPC probe via daemon status
|
|
if cd "$CLAWDBOT_DIR" && node dist/entry.js daemon status 2>&1 | grep -q "RPC probe: ok"; then
|
|
gateway_up=true
|
|
break
|
|
fi
|
|
fi
|
|
sleep 5
|
|
waited=$((waited + 5))
|
|
log " ...waiting ($waited/${STARTUP_TIMEOUT}s)"
|
|
done
|
|
|
|
if ! $gateway_up; then
|
|
log "Gateway didn't come up within ${STARTUP_TIMEOUT}s"
|
|
|
|
# Capture logs
|
|
log "Capturing crash logs..."
|
|
journalctl --user -u clawdbot-gateway.service -n 100 --no-pager > "${MOLT_DIR}/crash-log.txt" 2>&1 || true
|
|
|
|
# Attempt rollback
|
|
log "Attempting rollback..."
|
|
git checkout "$CURRENT_HEAD" --quiet
|
|
pnpm install --frozen-lockfile --prefer-offline 2>&1 || pnpm install 2>&1 || true
|
|
pnpm build 2>&1 || true
|
|
systemctl --user restart clawdbot-gateway.service
|
|
|
|
sleep 10
|
|
if systemctl --user is-active --quiet clawdbot-gateway.service; then
|
|
log "Rollback successful - gateway is running on ${CURRENT_HEAD:0:8}"
|
|
|
|
# Update changelog to reflect rollback
|
|
cat >> "${WORKSPACE_DIR}/update-changelog.md" << EOF
|
|
|
|
## ROLLBACK
|
|
Update failed - rolled back to \`${CURRENT_HEAD:0:8}\`
|
|
Gateway didn't come up after update.
|
|
See crash log: ~/.clawdbot/molt/crash-log.txt
|
|
EOF
|
|
|
|
die "Update failed, rolled back successfully"
|
|
else
|
|
log "CRITICAL: Rollback also failed!"
|
|
|
|
# Write recovery doc
|
|
cat > "${MOLT_DIR}/RECOVERY.md" << EOF
|
|
# Molt Recovery Required
|
|
|
|
The nightly update failed and automatic rollback also failed.
|
|
|
|
## What happened
|
|
- Update started at: $(cat "${MOLT_DIR}/pre-update-info" 2>/dev/null || echo "unknown")
|
|
- Old HEAD: ${CURRENT_HEAD}
|
|
- New HEAD: ${NEW_HEAD} (attempted)
|
|
- Error: Gateway didn't start
|
|
|
|
## Crash log
|
|
See: ~/.clawdbot/molt/crash-log.txt
|
|
|
|
## Manual recovery steps
|
|
1. Check the crash log for the root cause
|
|
2. Try: \`cd ~/clawd && git checkout ${CURRENT_HEAD} && pnpm install && pnpm build && systemctl --user restart clawdbot-gateway\`
|
|
3. If that fails, see ~/clawd/CLAUDE.md for nuclear options
|
|
|
|
## Context for AI recovery
|
|
The gateway failed to start after update. Common causes:
|
|
- Missing dependency (check for "Cannot find module" in logs)
|
|
- Syntax error in new code (check for "SyntaxError" in logs)
|
|
- Config incompatibility (check for "Invalid config" in logs)
|
|
EOF
|
|
|
|
die "CRITICAL: Both update and rollback failed. Manual intervention required. See ~/.clawdbot/molt/RECOVERY.md"
|
|
fi
|
|
fi
|
|
|
|
# Stability window
|
|
log "Gateway up! Waiting ${STABILITY_WINDOW}s for stability..."
|
|
sleep "$STABILITY_WINDOW"
|
|
|
|
# Check still running after stability window
|
|
if ! systemctl --user is-active --quiet clawdbot-gateway.service; then
|
|
log "Gateway crashed during stability window"
|
|
journalctl --user -u clawdbot-gateway.service -n 100 --no-pager > "${MOLT_DIR}/crash-log.txt" 2>&1 || true
|
|
|
|
# Rollback
|
|
git checkout "$CURRENT_HEAD" --quiet
|
|
pnpm install --frozen-lockfile --prefer-offline 2>&1 || pnpm install 2>&1 || true
|
|
pnpm build 2>&1 || true
|
|
systemctl --user restart clawdbot-gateway.service
|
|
|
|
cat >> "${WORKSPACE_DIR}/update-changelog.md" << EOF
|
|
|
|
## ROLLBACK
|
|
Update failed - crashed during stability window. Rolled back to \`${CURRENT_HEAD:0:8}\`
|
|
See crash log: ~/.clawdbot/molt/crash-log.txt
|
|
EOF
|
|
|
|
die "Gateway crashed during stability window, rolled back"
|
|
fi
|
|
|
|
# Final health check
|
|
if ! (cd "$CLAWDBOT_DIR" && node dist/entry.js daemon status 2>&1 | grep -q "RPC probe: ok"); then
|
|
log "Gateway running but not responding to RPC probe"
|
|
# Don't rollback for this - it's running, just maybe slow
|
|
log "Warning: RPC probe failed, but service is active"
|
|
fi
|
|
|
|
# === Phase 3: Report ===
|
|
log "=== Phase 3: Success ==="
|
|
|
|
# Update last-good
|
|
echo "$NEW_HEAD" > "${MOLT_DIR}/last-good"
|
|
|
|
# Log to history
|
|
echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"success\"}" >> "${MOLT_DIR}/history.jsonl"
|
|
|
|
log "Update complete!"
|
|
log " From: ${CURRENT_HEAD:0:8}"
|
|
log " To: ${NEW_HEAD:0:8}"
|
|
log " Commits: $COMMIT_COUNT"
|
|
log ""
|
|
log "Changelog written to: ${WORKSPACE_DIR}/update-changelog.md"
|
|
|
|
# Final summary for the agent to parse
|
|
echo ""
|
|
echo "=== MOLT SUMMARY ==="
|
|
echo "STATUS: SUCCESS"
|
|
echo "FROM: ${CURRENT_HEAD:0:8}"
|
|
echo "TO: ${NEW_HEAD:0:8}"
|
|
echo "COMMITS: $COMMIT_COUNT"
|
|
echo "CHANGELOG: ${WORKSPACE_DIR}/update-changelog.md"
|