diff --git a/scripts/molt.sh b/scripts/molt.sh index 59b7506e4..29b2185b4 100755 --- a/scripts/molt.sh +++ b/scripts/molt.sh @@ -24,7 +24,6 @@ BRANCH="main" # Health check timing STARTUP_TIMEOUT=60 STABILITY_WINDOW=30 -PING_TIMEOUT=5 # === Parse args === DRY_RUN=false @@ -47,9 +46,58 @@ die() { exit 1 } +# Rollback function - returns repo to pre-update state +rollback() { + local reason="$1" + log "Rolling back: $reason" + + # Capture logs + journalctl --user -u clawdbot-gateway.service -n 200 --no-pager \ + > "${MOLT_DIR}/crash-log.txt" 2>&1 || true + + # Return to branch and reset to pre-update HEAD + cd "$CLAWDBOT_DIR" + git checkout -q "$BRANCH" 2>/dev/null || true + git reset --hard -q "$CURRENT_HEAD" + + # Reinstall and rebuild + pnpm install --frozen-lockfile --prefer-offline > "${MOLT_DIR}/rollback-install.log" 2>&1 \ + || pnpm install > "${MOLT_DIR}/rollback-install.log" 2>&1 \ + || true + pnpm build > "${MOLT_DIR}/rollback-build.log" 2>&1 || true + + # Restart service + systemctl --user restart clawdbot-gateway.service || true +} + +# Trigger the agent for autonomous recovery +trigger_recovery_agent() { + local prompt="$1" + + # Check if we already tried recovery for this HEAD + local recovery_marker="${MOLT_DIR}/recovery-attempted-${NEW_HEAD}" + if [[ -f "$recovery_marker" ]]; then + log "Recovery already attempted for ${NEW_HEAD:0:8}; skipping agent trigger" + log "Manual intervention required." + return + fi + + # Mark that we're attempting recovery + touch "$recovery_marker" + + log "Triggering autonomous recovery agent..." + cd "$CLAWDBOT_DIR" + node dist/entry.js wake --mode now --text "$prompt" 2>&1 \ + || log "Warning: Could not trigger recovery agent" +} + # === Phase 0: Preflight === log "=== Phase 0: Preflight ===" +# Ensure directories exist +mkdir -p "$MOLT_DIR" || die "Cannot create $MOLT_DIR" +mkdir -p "$WORKSPACE_DIR" || die "Cannot create $WORKSPACE_DIR" + # Acquire lock LOCK_DIR="${MOLT_DIR}/lock" if ! mkdir "$LOCK_DIR" 2>/dev/null; then @@ -61,6 +109,13 @@ trap 'rmdir "$LOCK_DIR" 2>/dev/null || true' EXIT # Change to repo directory cd "$CLAWDBOT_DIR" || die "Cannot cd to $CLAWDBOT_DIR" +# Ensure we're on the expected branch +CURRENT_BRANCH=$(git branch --show-current) +if [[ "$CURRENT_BRANCH" != "$BRANCH" ]]; then + log "WARNING: On branch '$CURRENT_BRANCH', expected '$BRANCH'" + git checkout -q "$BRANCH" || die "Cannot switch to $BRANCH" +fi + # Fetch from remote log "Fetching from ${REMOTE}/${BRANCH}..." git fetch "$REMOTE" "$BRANCH" --quiet @@ -131,25 +186,22 @@ fi NEW_HEAD=$(git rev-parse HEAD) log "Updated to: ${NEW_HEAD:0:8}" -# Install dependencies +# Install dependencies (with logging) log "Installing dependencies..." -if ! pnpm install --frozen-lockfile --prefer-offline 2>&1; then +if ! pnpm install --frozen-lockfile --prefer-offline 2>&1 | tee "${MOLT_DIR}/pnpm-install.log"; then log "pnpm install failed, attempting recovery..." # Recovery: try without --frozen-lockfile in case lockfile is out of sync - if ! pnpm install --prefer-offline 2>&1; then - log "pnpm install still failed, rolling back..." - source "${MOLT_DIR}/molt.sh" --rollback-internal - die "pnpm install failed" + if ! pnpm install --prefer-offline 2>&1 | tee "${MOLT_DIR}/pnpm-install.log"; then + rollback "pnpm install failed" + die "pnpm install failed - see ${MOLT_DIR}/pnpm-install.log" fi fi -# Build +# Build (with logging) log "Building..." -if ! pnpm build 2>&1; then - log "Build failed, rolling back..." - git checkout "$CURRENT_HEAD" --quiet - pnpm install --frozen-lockfile --prefer-offline 2>&1 || true - die "Build failed" +if ! pnpm build 2>&1 | tee "${MOLT_DIR}/pnpm-build.log"; then + rollback "pnpm build failed" + die "Build failed - see ${MOLT_DIR}/pnpm-build.log" fi # Generate changelog before restart @@ -165,7 +217,7 @@ Updated from \`${CURRENT_HEAD:0:8}\` to \`${NEW_HEAD:0:8}\` ($COMMIT_COUNT commi $(git log --oneline "${CURRENT_HEAD}..${NEW_HEAD}") ## Changed Files -$(git diff --stat "${CURRENT_HEAD}..${NEW_HEAD}" | tail -20) +$(git diff --stat "${CURRENT_HEAD}..${NEW_HEAD}" | head -50) EOF # Restart gateway @@ -196,16 +248,8 @@ done if ! $gateway_up; then log "Gateway didn't come up within ${STARTUP_TIMEOUT}s" - # Capture logs - log "Capturing crash logs..." - journalctl --user -u clawdbot-gateway.service -n 100 --no-pager > "${MOLT_DIR}/crash-log.txt" 2>&1 || true - # Attempt rollback - log "Attempting rollback..." - git checkout "$CURRENT_HEAD" --quiet - pnpm install --frozen-lockfile --prefer-offline 2>&1 || pnpm install 2>&1 || true - pnpm build 2>&1 || true - systemctl --user restart clawdbot-gateway.service + rollback "health check failed" sleep 10 if systemctl --user is-active --quiet clawdbot-gateway.service; then @@ -227,12 +271,8 @@ EOF echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"rollback\",\"reason\":\"health_check_failed\"}" >> "${MOLT_DIR}/history.jsonl" # === AUTONOMOUS RECOVERY === - # Gateway is back up on old version - trigger agent to diagnose and fix - log "Triggering autonomous recovery agent..." - - cd "$CLAWDBOT_DIR" - node dist/entry.js wake --mode now --text "$(cat < Try: cd ~/clawd && pnpm install --force +- "ENOSPC" (disk full) -> Try: pnpm store prune && pnpm cache clean +- Network timeout during install -> Just retry: ~/.clawdbot/molt/molt.sh +- Lockfile conflict -> Try: cd ~/clawd && rm pnpm-lock.yaml && git checkout pnpm-lock.yaml && pnpm install ## Important - You have 1 retry attempt. If molt.sh fails again, notify the user. - Don't get stuck in a loop - if unsure, ask for help. AGENT_PROMPT -)" 2>&1 || log "Warning: Could not trigger recovery agent" +)" exit 1 # Exit with error so cron knows it failed else @@ -286,12 +334,14 @@ The nightly update failed and automatic rollback also failed. - New HEAD: ${NEW_HEAD} (attempted) - Error: Gateway didn't start -## Crash log -See: ~/.clawdbot/molt/crash-log.txt +## Logs +- Crash log: ~/.clawdbot/molt/crash-log.txt +- Install log: ~/.clawdbot/molt/pnpm-install.log +- Build log: ~/.clawdbot/molt/pnpm-build.log ## Manual recovery steps -1. Check the crash log for the root cause -2. Try: \`cd ~/clawd && git checkout ${CURRENT_HEAD} && pnpm install && pnpm build && systemctl --user restart clawdbot-gateway\` +1. Check the logs for the root cause +2. Try: \`cd ~/clawd && git checkout ${BRANCH} && git reset --hard ${CURRENT_HEAD} && pnpm install && pnpm build && systemctl --user restart clawdbot-gateway\` 3. If that fails, see ~/clawd/CLAUDE.md for nuclear options ## Context for AI recovery @@ -312,13 +362,9 @@ sleep "$STABILITY_WINDOW" # Check still running after stability window if ! systemctl --user is-active --quiet clawdbot-gateway.service; then log "Gateway crashed during stability window" - journalctl --user -u clawdbot-gateway.service -n 100 --no-pager > "${MOLT_DIR}/crash-log.txt" 2>&1 || true # Rollback - git checkout "$CURRENT_HEAD" --quiet - pnpm install --frozen-lockfile --prefer-offline 2>&1 || pnpm install 2>&1 || true - pnpm build 2>&1 || true - systemctl --user restart clawdbot-gateway.service + rollback "stability window crash" # Save the failed HEAD echo "$NEW_HEAD" > "${MOLT_DIR}/attempted-head" @@ -334,12 +380,10 @@ EOF echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"rollback\",\"reason\":\"stability_window_crash\"}" >> "${MOLT_DIR}/history.jsonl" # Trigger autonomous recovery - log "Triggering autonomous recovery agent..." sleep 5 # Give gateway a moment to stabilize - cd "$CLAWDBOT_DIR" - node dist/entry.js wake --mode now --text "$(cat <&1 || log "Warning: Could not trigger recovery agent" +)" exit 1 fi @@ -372,6 +421,9 @@ log "=== Phase 3: Success ===" # Update last-good echo "$NEW_HEAD" > "${MOLT_DIR}/last-good" +# Clear any recovery markers for this HEAD (it succeeded!) +rm -f "${MOLT_DIR}/recovery-attempted-${NEW_HEAD}" 2>/dev/null || true + # Log to history echo "{\"timestamp\":\"$(timestamp)\",\"from\":\"${CURRENT_HEAD}\",\"to\":\"${NEW_HEAD}\",\"commits\":${COMMIT_COUNT},\"status\":\"success\"}" >> "${MOLT_DIR}/history.jsonl"