fix(infra): prevent gateway crashes on transient network errors

CHANGELOG.md

@@ -37,16 +37,17 @@ Status: unreleased.
 - Slack: clear ack reaction after streamed replies. (#2044) Thanks @fancyboi999.
 - macOS: keep custom SSH usernames in remote target. (#2046) Thanks @algal.
 
-### Fixes
-- Telegram: wrap reasoning italics per line to avoid raw underscores. (#2181) Thanks @YuriNachos.
-- Voice Call: enforce Twilio webhook signature verification for ngrok URLs; disable ngrok free tier bypass by default.
-- Security: harden Tailscale Serve auth by validating identity via local tailscaled before trusting headers.
-- Build: align memory-core peer dependency with lockfile.
-- Security: add mDNS discovery mode with minimal default to reduce information disclosure. (#1882) Thanks @orlyjamie.
-- Security: harden URL fetches with DNS pinning to reduce rebinding risk. Thanks Chris Zheng.
-- Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93.
-- Security: wrap external hook content by default with a per-hook opt-out. (#1827) Thanks @mertcicekci0.
-- Gateway: default auth now fail-closed (token/password required; Tailscale Serve identity remains allowed).
+ ### Fixes
+ - Gateway: prevent crashes on transient network errors (fetch failures, timeouts, DNS). Added fatal error detection to only exit on truly critical errors. Fixes #2895, #2879, #2873.
+ - Telegram: wrap reasoning italics per line to avoid raw underscores. (#2181) Thanks @YuriNachos.
+ - Voice Call: enforce Twilio webhook signature verification for ngrok URLs; disable ngrok free tier bypass by default.
+ - Security: harden Tailscale Serve auth by validating identity via local tailscaled before trusting headers.
+ - Build: align memory-core peer dependency with lockfile.
+ - Security: add mDNS discovery mode with minimal default to reduce information disclosure. (#1882) Thanks @orlyjamie.
+ - Security: harden URL fetches with DNS pinning to reduce rebinding risk. Thanks Chris Zheng.
+ - Web UI: improve WebChat image paste previews and allow image-only sends. (#1925) Thanks @smartprogrammer93.
+ - Security: wrap external hook content by default with a per-hook opt-out. (#1827) Thanks @mertcicekci0.
+ - Gateway: default auth now fail-closed (token/password required; Tailscale Serve identity remains allowed).
 
 ## 2026.1.24-3
 

src/infra/unhandled-rejections.fatal-detection.test.ts

@@ -0,0 +1,159 @@
+import { describe, it, expect, vi, beforeAll, afterAll, beforeEach, afterEach } from "vitest";
+import process from "node:process";
+
+import { installUnhandledRejectionHandler } from "./unhandled-rejections.js";
+
+describe("installUnhandledRejectionHandler - fatal detection", () => {
+  let exitCalls: Array<string | number | null> = [];
+  let consoleErrorSpy: ReturnType<typeof vi.spyOn>;
+  let consoleWarnSpy: ReturnType<typeof vi.spyOn>;
+  let originalExit: typeof process.exit;
+
+  beforeAll(() => {
+    originalExit = process.exit;
+    installUnhandledRejectionHandler();
+  });
+
+  beforeEach(() => {
+    exitCalls = [];
+
+    vi.spyOn(process, "exit").mockImplementation((code: string | number | null | undefined) => {
+      if (code !== undefined && code !== null) {
+        exitCalls.push(code);
+      }
+    });
+
+    consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {});
+    consoleWarnSpy = vi.spyOn(console, "warn").mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+    consoleErrorSpy.mockRestore();
+    consoleWarnSpy.mockRestore();
+  });
+
+  afterAll(() => {
+    process.exit = originalExit;
+  });
+
+  describe("fatal errors", () => {
+    it("exits on ERR_OUT_OF_MEMORY", () => {
+      const oomErr = Object.assign(new Error("Out of memory"), {
+        code: "ERR_OUT_OF_MEMORY",
+      });
+
+      process.emit("unhandledRejection", oomErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([1]);
+      expect(consoleErrorSpy).toHaveBeenCalledWith(
+        "[clawdbot] FATAL unhandled rejection:",
+        expect.stringContaining("Out of memory"),
+      );
+    });
+
+    it("exits on ERR_SCRIPT_EXECUTION_TIMEOUT", () => {
+      const timeoutErr = Object.assign(new Error("Script execution timeout"), {
+        code: "ERR_SCRIPT_EXECUTION_TIMEOUT",
+      });
+
+      process.emit("unhandledRejection", timeoutErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([1]);
+    });
+
+    it("exits on ERR_WORKER_OUT_OF_MEMORY", () => {
+      const workerOomErr = Object.assign(new Error("Worker out of memory"), {
+        code: "ERR_WORKER_OUT_OF_MEMORY",
+      });
+
+      process.emit("unhandledRejection", workerOomErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([1]);
+    });
+  });
+
+  describe("configuration errors", () => {
+    it("exits on INVALID_CONFIG", () => {
+      const configErr = Object.assign(new Error("Invalid config"), {
+        code: "INVALID_CONFIG",
+      });
+
+      process.emit("unhandledRejection", configErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([1]);
+      expect(consoleErrorSpy).toHaveBeenCalledWith(
+        "[clawdbot] CONFIGURATION ERROR - requires fix:",
+        expect.stringContaining("Invalid config"),
+      );
+    });
+
+    it("exits on MISSING_API_KEY", () => {
+      const missingKeyErr = Object.assign(new Error("Missing API key"), {
+        code: "MISSING_API_KEY",
+      });
+
+      process.emit("unhandledRejection", missingKeyErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([1]);
+    });
+  });
+
+  describe("non-fatal errors", () => {
+    it("does NOT exit on undici fetch failures", () => {
+      const fetchErr = Object.assign(new TypeError("fetch failed"), {
+        cause: { code: "UND_ERR_CONNECT_TIMEOUT", syscall: "connect" },
+      });
+
+      process.emit("unhandledRejection", fetchErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([]);
+      expect(consoleWarnSpy).toHaveBeenCalledWith(
+        "[clawdbot] Non-fatal unhandled rejection (continuing):",
+        expect.stringContaining("fetch failed"),
+      );
+    });
+
+    it("does NOT exit on DNS resolution failures", () => {
+      const dnsErr = Object.assign(new Error("DNS resolve failed"), {
+        code: "UND_ERR_DNS_RESOLVE_FAILED",
+      });
+
+      process.emit("unhandledRejection", dnsErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([]);
+      expect(consoleWarnSpy).toHaveBeenCalled();
+    });
+
+    it("does NOT exit on generic errors without code", () => {
+      const genericErr = new Error("Something went wrong");
+
+      process.emit("unhandledRejection", genericErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([]);
+      expect(consoleWarnSpy).toHaveBeenCalled();
+    });
+
+    it("does NOT exit on connection reset errors", () => {
+      const connResetErr = Object.assign(new Error("Connection reset"), {
+        code: "ECONNRESET",
+      });
+
+      process.emit("unhandledRejection", connResetErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([]);
+      expect(consoleWarnSpy).toHaveBeenCalled();
+    });
+
+    it("does NOT exit on timeout errors", () => {
+      const timeoutErr = Object.assign(new Error("Timeout"), {
+        code: "ETIMEDOUT",
+      });
+
+      process.emit("unhandledRejection", timeoutErr, Promise.resolve());
+
+      expect(exitCalls).toEqual([]);
+      expect(consoleWarnSpy).toHaveBeenCalled();
+    });
+  });
+});

src/infra/unhandled-rejections.ts

@@ -1,11 +1,35 @@
 import process from "node:process";
 
-import { formatUncaughtError } from "./errors.js";
+import { extractErrorCode, formatUncaughtError } from "./errors.js";
 
 type UnhandledRejectionHandler = (reason: unknown) => boolean;
 
 const handlers = new Set<UnhandledRejectionHandler>();
 
+const FATAL_ERROR_CODES = new Set([
+  "ERR_OUT_OF_MEMORY",
+  "ERR_SCRIPT_EXECUTION_TIMEOUT",
+  "ERR_WORKER_OUT_OF_MEMORY",
+  "ERR_WORKER_UNCAUGHT_EXCEPTION",
+  "ERR_WORKER_INITIALIZATION_FAILED",
+]);
+
+const CONFIG_ERROR_CODES = new Set([
+  "INVALID_CONFIG",
+  "MISSING_API_KEY",
+  "MISSING_CREDENTIALS",
+]);
+
+function isFatalError(err: unknown): boolean {
+  const code = extractErrorCode(err);
+  return code !== undefined && FATAL_ERROR_CODES.has(code);
+}
+
+function isConfigError(err: unknown): boolean {
+  const code = extractErrorCode(err);
+  return code !== undefined && CONFIG_ERROR_CODES.has(code);
+}
+
 export function registerUnhandledRejectionHandler(handler: UnhandledRejectionHandler): () => void {
   handlers.add(handler);
   return () => {
@@ -30,7 +54,25 @@ export function isUnhandledRejectionHandled(reason: unknown): boolean {
 export function installUnhandledRejectionHandler(): void {
   process.on("unhandledRejection", (reason, _promise) => {
     if (isUnhandledRejectionHandled(reason)) return;
-    console.error("[clawdbot] Unhandled promise rejection:", formatUncaughtError(reason));
-    process.exit(1);
+
+    if (isFatalError(reason)) {
+      console.error("[clawdbot] FATAL unhandled rejection:", formatUncaughtError(reason));
+      process.exit(1);
+      return;
+    }
+
+    if (isConfigError(reason)) {
+      console.error(
+        "[clawdbot] CONFIGURATION ERROR - requires fix:",
+        formatUncaughtError(reason),
+      );
+      process.exit(1);
+      return;
+    }
+
+    console.warn(
+      "[clawdbot] Non-fatal unhandled rejection (continuing):",
+      formatUncaughtError(reason),
+    );
   });
 }