#!/usr/bin/env bash # # molt.sh - Self-healing update script for Clawdbot # # Usage: ./molt.sh [--dry-run] # # This script: # 1. Checks for updates from upstream # 2. Pulls, installs, builds, restarts # 3. Verifies the gateway comes back healthy # 4. Rolls back if something breaks # # Designed to be run via Clawdbot cron or system cron. set -euo pipefail # === Configuration === MOLT_DIR="${HOME}/.clawdbot/molt" CLAWDBOT_DIR="${HOME}/clawd" WORKSPACE_DIR="${HOME}/maja-workspace" REMOTE="upstream" BRANCH="main" # Health check timing STARTUP_TIMEOUT=60 STABILITY_WINDOW=30 # === Parse args === DRY_RUN=false if [[ "${1:-}" == "--dry-run" ]]; then DRY_RUN=true echo "[molt] DRY RUN MODE - no changes will be made" fi # === Helper functions === timestamp() { date -u +"%Y-%m-%dT%H:%M:%SZ" } log() { echo "[molt $(timestamp)] $*" } die() { log "ERROR: $*" exit 1 } # Rollback function - returns repo to pre-update state rollback() { local reason="$1" log "Rolling back: $reason" # Capture logs journalctl --user -u clawdbot-gateway.service -n 200 --no-pager \ > "${MOLT_DIR}/crash-log.txt" 2>&1 || true # Return to branch and reset to pre-update HEAD cd "$CLAWDBOT_DIR" git checkout -q "$BRANCH" 2>/dev/null || true git reset --hard -q "$CURRENT_HEAD" # Reinstall and rebuild pnpm install --frozen-lockfile --prefer-offline > "${MOLT_DIR}/rollback-install.log" 2>&1 \ || pnpm install > "${MOLT_DIR}/rollback-install.log" 2>&1 \ || true pnpm build > "${MOLT_DIR}/rollback-build.log" 2>&1 || true # Restart service systemctl --user restart clawdbot-gateway.service || true } # Trigger the agent for autonomous recovery trigger_recovery_agent() { local prompt="$1" # Check if we already tried recovery for this HEAD local recovery_marker="${MOLT_DIR}/recovery-attempted-${NEW_HEAD}" if [[ -f "$recovery_marker" ]]; then log "Recovery already attempted for ${NEW_HEAD:0:8}; skipping agent trigger" log "Manual intervention required." return fi # Mark that we're attempting recovery touch "$recovery_marker" log "Triggering autonomous recovery agent..." cd "$CLAWDBOT_DIR" node dist/entry.js wake --mode now --text "$prompt" 2>&1 \ || log "Warning: Could not trigger recovery agent" } # === Phase 0: Preflight === log "=== Phase 0: Preflight ===" # Ensure directories exist mkdir -p "$MOLT_DIR" || die "Cannot create $MOLT_DIR" mkdir -p "$WORKSPACE_DIR" || die "Cannot create $WORKSPACE_DIR" # Acquire lock LOCK_DIR="${MOLT_DIR}/lock" if ! mkdir "$LOCK_DIR" 2>/dev/null; then log "Another molt run in progress (lock exists), exiting" exit 0 fi trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT # Change to repo directory cd "$CLAWDBOT_DIR" || die "Cannot cd to $CLAWDBOT_DIR" # Ensure we're on the expected branch CURRENT_BRANCH=$(git branch --show-current) if [[ "$CURRENT_BRANCH" != "$BRANCH" ]]; then log "WARNING: On branch '$CURRENT_BRANCH', expected '$BRANCH'" git checkout -q "$BRANCH" || die "Cannot switch to $BRANCH" fi # Fetch from remote log "Fetching from ${REMOTE}/${BRANCH}..." git fetch "$REMOTE" "$BRANCH" --quiet # Compare HEAD with remote CURRENT_HEAD=$(git rev-parse HEAD) REMOTE_HEAD=$(git rev-parse "${REMOTE}/${BRANCH}") log "Current HEAD: ${CURRENT_HEAD:0:8}" log "Remote HEAD: ${REMOTE_HEAD:0:8}" if [[ "$CURRENT_HEAD" == "$REMOTE_HEAD" ]]; then log "Already up to date" # Write minimal changelog cat > "${WORKSPACE_DIR}/update-changelog.md" << EOF # Clawdbot Update Changelog Generated: $(timestamp) ## Status No changes - already up to date. Current version: ${CURRENT_HEAD:0:8} EOF exit 0 fi # Check for clean workdir (warn but continue if dirty) REQUIRE_CLEAN=${REQUIRE_CLEAN_WORKDIR:-false} if [[ -n "$(git status --porcelain)" ]]; then log "Working directory not clean:" git status --short if [[ "$REQUIRE_CLEAN" == "true" ]]; then die "Aborting - commit or stash your changes first" else log "WARNING: Continuing with dirty workdir (REQUIRE_CLEAN_WORKDIR=false)" log "WARNING: Local changes may be lost during rollback!" fi fi # Save state for potential rollback log "Saving pre-update state..." echo "$CURRENT_HEAD" > "${MOLT_DIR}/pre-update-head" cp pnpm-lock.yaml "${MOLT_DIR}/pre-update-lock.yaml" 2>/dev/null || true git log --oneline -1 > "${MOLT_DIR}/pre-update-info" # Count incoming commits COMMIT_COUNT=$(git rev-list --count "${CURRENT_HEAD}..${REMOTE_HEAD}") log "Incoming commits: $COMMIT_COUNT" if $DRY_RUN; then log "[DRY RUN] Would update from ${CURRENT_HEAD:0:8} to ${REMOTE_HEAD:0:8}" log "[DRY RUN] Incoming changes:" git log --oneline "${CURRENT_HEAD}..${REMOTE_HEAD}" exit 0 fi # === Phase 1: Update === log "=== Phase 1: Update ===" # Merge (fast-forward only) log "Merging ${REMOTE}/${BRANCH}..." if ! git merge "${REMOTE}/${BRANCH}" --ff-only; then die "Merge failed - history has diverged. Manual intervention required." fi NEW_HEAD=$(git rev-parse HEAD) log "Updated to: ${NEW_HEAD:0:8}" # Install dependencies (with logging) log "Installing dependencies..." if ! pnpm install --frozen-lockfile --prefer-offline 2>&1 | tee "${MOLT_DIR}/pnpm-install.log"; then log "pnpm install failed, attempting recovery..." # Recovery: try without --frozen-lockfile in case lockfile is out of sync if ! pnpm install --prefer-offline 2>&1 | tee "${MOLT_DIR}/pnpm-install.log"; then rollback "pnpm install failed" die "pnpm install failed - see ${MOLT_DIR}/pnpm-install.log" fi fi # Build (with logging) log "Building..." if ! pnpm build 2>&1 | tee "${MOLT_DIR}/pnpm-build.log"; then rollback "pnpm build failed" die "Build failed - see ${MOLT_DIR}/pnpm-build.log" fi # Generate changelog before restart log "Generating changelog..." cat > "${WORKSPACE_DIR}/update-changelog.md" << EOF # Clawdbot Update Changelog Generated: $(timestamp) ## Summary Updated from \`${CURRENT_HEAD:0:8}\` to \`${NEW_HEAD:0:8}\` ($COMMIT_COUNT commits) ## Commits $(git log --oneline "${CURRENT_HEAD}..${NEW_HEAD}") ## Changed Files $(git diff --stat "${CURRENT_HEAD}..${NEW_HEAD}" | head -50) EOF # Restart gateway log "Restarting gateway..." systemctl --user restart clawdbot-gateway.service # === Phase 2: Verify === log "=== Phase 2: Verify ===" # Wait for gateway to respond log "Waiting for gateway to come up (timeout: ${STARTUP_TIMEOUT}s)..." waited=0 gateway_up=false while [[ $waited -lt $STARTUP_TIMEOUT ]]; do if systemctl --user is-active --quiet clawdbot-gateway.service; then # Service is running, check RPC probe via daemon status if cd "$CLAWDBOT_DIR" && node dist/entry.js daemon status 2>&1 | grep -q "RPC probe: ok"; then gateway_up=true break fi fi sleep 5 waited=$((waited + 5)) log " ...waiting ($waited/${STARTUP_TIMEOUT}s)" done if ! $gateway_up; then log "Gateway didn't come up within ${STARTUP_TIMEOUT}s" # Attempt rollback rollback "health check failed" sleep 10 if systemctl --user is-active --quiet clawdbot-gateway.service; then log "Rollback successful - gateway is running on ${CURRENT_HEAD:0:8}" # Save the failed HEAD for the agent to analyze echo "$NEW_HEAD" > "${MOLT_DIR}/attempted-head" # Update changelog to reflect rollback cat >> "${WORKSPACE_DIR}/update-changelog.md" << EOF ## ROLLBACK Update failed - rolled back to \`${CURRENT_HEAD:0:8}\` Gateway didn't come up after update. See crash log: ~/.clawdbot/molt/crash-log.txt EOF # Log rollback to history echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"rollback\",\"reason\":\"health_check_failed\"}" >> "${MOLT_DIR}/history.jsonl" # === AUTONOMOUS RECOVERY === trigger_recovery_agent "$(cat < Try: cd ~/clawd && pnpm install --force - "ENOSPC" (disk full) -> Try: pnpm store prune && pnpm cache clean - Network timeout during install -> Just retry: ~/.clawdbot/molt/molt.sh - Lockfile conflict -> Try: cd ~/clawd && rm pnpm-lock.yaml && git checkout pnpm-lock.yaml && pnpm install ## Important - You have 1 retry attempt. If molt.sh fails again, notify the user. - Don't get stuck in a loop - if unsure, ask for help. AGENT_PROMPT )" exit 1 # Exit with error so cron knows it failed else log "CRITICAL: Rollback also failed!" # Write recovery doc cat > "${MOLT_DIR}/RECOVERY.md" << EOF # Molt Recovery Required The nightly update failed and automatic rollback also failed. ## What happened - Update started at: $(cat "${MOLT_DIR}/pre-update-info" 2>/dev/null || echo "unknown") - Old HEAD: ${CURRENT_HEAD} - New HEAD: ${NEW_HEAD} (attempted) - Error: Gateway didn't start ## Logs - Crash log: ~/.clawdbot/molt/crash-log.txt - Install log: ~/.clawdbot/molt/pnpm-install.log - Build log: ~/.clawdbot/molt/pnpm-build.log ## Manual recovery steps 1. Check the logs for the root cause 2. Try: \`cd ~/clawd && git checkout ${BRANCH} && git reset --hard ${CURRENT_HEAD} && pnpm install && pnpm build && systemctl --user restart clawdbot-gateway\` 3. If that fails, see ~/clawd/CLAUDE.md for nuclear options ## Context for AI recovery The gateway failed to start after update. Common causes: - Missing dependency (check for "Cannot find module" in logs) - Syntax error in new code (check for "SyntaxError" in logs) - Config incompatibility (check for "Invalid config" in logs) EOF die "CRITICAL: Both update and rollback failed. Manual intervention required. See ~/.clawdbot/molt/RECOVERY.md" fi fi # Stability window log "Gateway up! Waiting ${STABILITY_WINDOW}s for stability..." sleep "$STABILITY_WINDOW" # Check still running after stability window if ! systemctl --user is-active --quiet clawdbot-gateway.service; then log "Gateway crashed during stability window" # Rollback rollback "stability window crash" # Save the failed HEAD echo "$NEW_HEAD" > "${MOLT_DIR}/attempted-head" cat >> "${WORKSPACE_DIR}/update-changelog.md" << EOF ## ROLLBACK Update failed - crashed during stability window. Rolled back to \`${CURRENT_HEAD:0:8}\` See crash log: ~/.clawdbot/molt/crash-log.txt EOF # Log rollback echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"rollback\",\"reason\":\"stability_window_crash\"}" >> "${MOLT_DIR}/history.jsonl" # Trigger autonomous recovery sleep 5 # Give gateway a moment to stabilize trigger_recovery_agent "$(cat <&1 | grep -q "RPC probe: ok"); then log "Gateway running but not responding to RPC probe" # Don't rollback for this - it's running, just maybe slow log "Warning: RPC probe failed, but service is active" fi # === Phase 3: Report === log "=== Phase 3: Success ===" # Update last-good echo "$NEW_HEAD" > "${MOLT_DIR}/last-good" # Clear any recovery markers for this HEAD (it succeeded!) rm -f "${MOLT_DIR}/recovery-attempted-${NEW_HEAD}" 2>/dev/null || true # Log to history echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"success\"}" >> "${MOLT_DIR}/history.jsonl" log "Update complete!" log " From: ${CURRENT_HEAD:0:8}" log " To: ${NEW_HEAD:0:8}" log " Commits: $COMMIT_COUNT" log "" log "Changelog written to: ${WORKSPACE_DIR}/update-changelog.md" # Final summary for the agent to parse echo "" echo "=== MOLT SUMMARY ===" echo "STATUS: SUCCESS" echo "FROM: ${CURRENT_HEAD:0:8}" echo "TO: ${NEW_HEAD:0:8}" echo "COMMITS: $COMMIT_COUNT" echo "CHANGELOG: ${WORKSPACE_DIR}/update-changelog.md"