From 36c14c5d612d203b1e373c03e24b17578e79a8c9 Mon Sep 17 00:00:00 2001 From: elliotsecops Date: Tue, 27 Jan 2026 14:43:42 -0400 Subject: [PATCH] fix(infra): prevent gateway crashes on transient network errors --- CHANGELOG.md | 21 +-- ...handled-rejections.fatal-detection.test.ts | 159 ++++++++++++++++++ src/infra/unhandled-rejections.ts | 48 +++++- 3 files changed, 215 insertions(+), 13 deletions(-) create mode 100644 src/infra/unhandled-rejections.fatal-detection.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fb9388a9..6b209c054 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,16 +37,17 @@ Status: unreleased. - Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999. - macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal. -### Fixes -- Telegram: wrap reasoning italics per line to avoid raw underscores. (#2181) Thanks @YuriNachos. -- Voice Call: enforce Twilio webhook signature verification for ngrok URLs; disable ngrok free tier bypass by default. -- Security: harden Tailscale Serve auth by validating identity via local tailscaled before trusting headers. -- Build: align memory-core peer dependency with lockfile. -- Security: add mDNS discovery mode with minimal default to reduce information disclosure. (#1882) Thanks @orlyjamie. -- Security: harden URL fetches with DNS pinning to reduce rebinding risk. Thanks Chris Zheng. -- Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93. -- Security: wrap external hook content by default with a per-hook opt-out. (#1827) Thanks @mertcicekci0. -- Gateway: default auth now fail-closed (token/password required; Tailscale Serve identity remains allowed). + ### Fixes + - Gateway: prevent crashes on transient network errors (fetch failures, timeouts, DNS). Added fatal error detection to only exit on truly critical errors. Fixes #2895, #2879, #2873. + - Telegram: wrap reasoning italics per line to avoid raw underscores. (#2181) Thanks @YuriNachos. + - Voice Call: enforce Twilio webhook signature verification for ngrok URLs; disable ngrok free tier bypass by default. + - Security: harden Tailscale Serve auth by validating identity via local tailscaled before trusting headers. + - Build: align memory-core peer dependency with lockfile. + - Security: add mDNS discovery mode with minimal default to reduce information disclosure. (#1882) Thanks @orlyjamie. + - Security: harden URL fetches with DNS pinning to reduce rebinding risk. Thanks Chris Zheng. + - Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93. + - Security: wrap external hook content by default with a per-hook opt-out. (#1827) Thanks @mertcicekci0. + - Gateway: default auth now fail-closed (token/password required; Tailscale Serve identity remains allowed). ## 2026.1.24-3 diff --git a/src/infra/unhandled-rejections.fatal-detection.test.ts b/src/infra/unhandled-rejections.fatal-detection.test.ts new file mode 100644 index 000000000..7c8d97675 --- /dev/null +++ b/src/infra/unhandled-rejections.fatal-detection.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from "vitest"; +import process from "node:process"; + +import { installUnhandledRejectionHandler } from "./unhandled-rejections.js"; + +describe("installUnhandledRejectionHandler - fatal detection", () => { + let exitCalls: Array = []; + let consoleErrorSpy: ReturnType; + let consoleWarnSpy: ReturnType; + let originalExit: typeof process.exit; + + beforeAll(() => { + originalExit = process.exit; + installUnhandledRejectionHandler(); + }); + + beforeEach(() => { + exitCalls = []; + + vi.spyOn(process, "exit").mockImplementation((code: string | number | null | undefined) => { + if (code !== undefined && code !== null) { + exitCalls.push(code); + } + }); + + consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + consoleWarnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.clearAllMocks(); + consoleErrorSpy.mockRestore(); + consoleWarnSpy.mockRestore(); + }); + + afterAll(() => { + process.exit = originalExit; + }); + + describe("fatal errors", () => { + it("exits on ERR_OUT_OF_MEMORY", () => { + const oomErr = Object.assign(new Error("Out of memory"), { + code: "ERR_OUT_OF_MEMORY", + }); + + process.emit("unhandledRejection", oomErr, Promise.resolve()); + + expect(exitCalls).toEqual([1]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + "[clawdbot] FATAL unhandled rejection:", + expect.stringContaining("Out of memory"), + ); + }); + + it("exits on ERR_SCRIPT_EXECUTION_TIMEOUT", () => { + const timeoutErr = Object.assign(new Error("Script execution timeout"), { + code: "ERR_SCRIPT_EXECUTION_TIMEOUT", + }); + + process.emit("unhandledRejection", timeoutErr, Promise.resolve()); + + expect(exitCalls).toEqual([1]); + }); + + it("exits on ERR_WORKER_OUT_OF_MEMORY", () => { + const workerOomErr = Object.assign(new Error("Worker out of memory"), { + code: "ERR_WORKER_OUT_OF_MEMORY", + }); + + process.emit("unhandledRejection", workerOomErr, Promise.resolve()); + + expect(exitCalls).toEqual([1]); + }); + }); + + describe("configuration errors", () => { + it("exits on INVALID_CONFIG", () => { + const configErr = Object.assign(new Error("Invalid config"), { + code: "INVALID_CONFIG", + }); + + process.emit("unhandledRejection", configErr, Promise.resolve()); + + expect(exitCalls).toEqual([1]); + expect(consoleErrorSpy).toHaveBeenCalledWith( + "[clawdbot] CONFIGURATION ERROR - requires fix:", + expect.stringContaining("Invalid config"), + ); + }); + + it("exits on MISSING_API_KEY", () => { + const missingKeyErr = Object.assign(new Error("Missing API key"), { + code: "MISSING_API_KEY", + }); + + process.emit("unhandledRejection", missingKeyErr, Promise.resolve()); + + expect(exitCalls).toEqual([1]); + }); + }); + + describe("non-fatal errors", () => { + it("does NOT exit on undici fetch failures", () => { + const fetchErr = Object.assign(new TypeError("fetch failed"), { + cause: { code: "UND_ERR_CONNECT_TIMEOUT", syscall: "connect" }, + }); + + process.emit("unhandledRejection", fetchErr, Promise.resolve()); + + expect(exitCalls).toEqual([]); + expect(consoleWarnSpy).toHaveBeenCalledWith( + "[clawdbot] Non-fatal unhandled rejection (continuing):", + expect.stringContaining("fetch failed"), + ); + }); + + it("does NOT exit on DNS resolution failures", () => { + const dnsErr = Object.assign(new Error("DNS resolve failed"), { + code: "UND_ERR_DNS_RESOLVE_FAILED", + }); + + process.emit("unhandledRejection", dnsErr, Promise.resolve()); + + expect(exitCalls).toEqual([]); + expect(consoleWarnSpy).toHaveBeenCalled(); + }); + + it("does NOT exit on generic errors without code", () => { + const genericErr = new Error("Something went wrong"); + + process.emit("unhandledRejection", genericErr, Promise.resolve()); + + expect(exitCalls).toEqual([]); + expect(consoleWarnSpy).toHaveBeenCalled(); + }); + + it("does NOT exit on connection reset errors", () => { + const connResetErr = Object.assign(new Error("Connection reset"), { + code: "ECONNRESET", + }); + + process.emit("unhandledRejection", connResetErr, Promise.resolve()); + + expect(exitCalls).toEqual([]); + expect(consoleWarnSpy).toHaveBeenCalled(); + }); + + it("does NOT exit on timeout errors", () => { + const timeoutErr = Object.assign(new Error("Timeout"), { + code: "ETIMEDOUT", + }); + + process.emit("unhandledRejection", timeoutErr, Promise.resolve()); + + expect(exitCalls).toEqual([]); + expect(consoleWarnSpy).toHaveBeenCalled(); + }); + }); +}); diff --git a/src/infra/unhandled-rejections.ts b/src/infra/unhandled-rejections.ts index c444baaa2..fbc084830 100644 --- a/src/infra/unhandled-rejections.ts +++ b/src/infra/unhandled-rejections.ts @@ -1,11 +1,35 @@ import process from "node:process"; -import { formatUncaughtError } from "./errors.js"; +import { extractErrorCode, formatUncaughtError } from "./errors.js"; type UnhandledRejectionHandler = (reason: unknown) => boolean; const handlers = new Set(); +const FATAL_ERROR_CODES = new Set([ + "ERR_OUT_OF_MEMORY", + "ERR_SCRIPT_EXECUTION_TIMEOUT", + "ERR_WORKER_OUT_OF_MEMORY", + "ERR_WORKER_UNCAUGHT_EXCEPTION", + "ERR_WORKER_INITIALIZATION_FAILED", +]); + +const CONFIG_ERROR_CODES = new Set([ + "INVALID_CONFIG", + "MISSING_API_KEY", + "MISSING_CREDENTIALS", +]); + +function isFatalError(err: unknown): boolean { + const code = extractErrorCode(err); + return code !== undefined && FATAL_ERROR_CODES.has(code); +} + +function isConfigError(err: unknown): boolean { + const code = extractErrorCode(err); + return code !== undefined && CONFIG_ERROR_CODES.has(code); +} + export function registerUnhandledRejectionHandler(handler: UnhandledRejectionHandler): () => void { handlers.add(handler); return () => { @@ -30,7 +54,25 @@ export function isUnhandledRejectionHandled(reason: unknown): boolean { export function installUnhandledRejectionHandler(): void { process.on("unhandledRejection", (reason, _promise) => { if (isUnhandledRejectionHandled(reason)) return; - console.error("[clawdbot] Unhandled promise rejection:", formatUncaughtError(reason)); - process.exit(1); + + if (isFatalError(reason)) { + console.error("[clawdbot] FATAL unhandled rejection:", formatUncaughtError(reason)); + process.exit(1); + return; + } + + if (isConfigError(reason)) { + console.error( + "[clawdbot] CONFIGURATION ERROR - requires fix:", + formatUncaughtError(reason), + ); + process.exit(1); + return; + } + + console.warn( + "[clawdbot] Non-fatal unhandled rejection (continuing):", + formatUncaughtError(reason), + ); }); }