From 9fc122a7f0f1dd9caaeb20290ffffd3ecb0dcdae Mon Sep 17 00:00:00 2001 From: Anthony Date: Mon, 26 Jan 2026 15:01:32 +0100 Subject: [PATCH] fix: add retry logic for transient unknown errors from LLM providers When providers return generic 'unknown error' messages (often from temporary service issues), clawdbot now automatically retries up to 2 times with exponential backoff (1s, 2s). - Add isRetryableUnknownError() to detect generic error patterns - Add retry loop in runEmbeddedPiAgent for these transient errors - Add unit tests for the new error detection logic --- ...ed-helpers.isretryableunknownerror.test.ts | 95 +++++++++++++++++++ src/agents/pi-embedded-helpers.ts | 2 + src/agents/pi-embedded-helpers/errors.ts | 27 ++++++ src/agents/pi-embedded-runner/run.ts | 23 ++++- 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/agents/pi-embedded-helpers.isretryableunknownerror.test.ts diff --git a/src/agents/pi-embedded-helpers.isretryableunknownerror.test.ts b/src/agents/pi-embedded-helpers.isretryableunknownerror.test.ts new file mode 100644 index 000000000..165000e0a --- /dev/null +++ b/src/agents/pi-embedded-helpers.isretryableunknownerror.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; +import { + isRetryableUnknownError, + isRetryableUnknownAssistantError, +} from "./pi-embedded-helpers.js"; +import type { AssistantMessage } from "@mariozechner/pi-ai"; + +describe("isRetryableUnknownError", () => { + it("matches generic unknown error patterns from pi-mono providers", () => { + const samples = [ + "Unknown error", + "unknown error", + "An unknown error occurred", + "an unknown error occurred", + "An unkown error ocurred", // typo variant from pi-mono + "Response failed", + "response failed", + "Unknown error: something went wrong", + ]; + for (const sample of samples) { + expect(isRetryableUnknownError(sample)).toBe(true); + } + }); + + it("does not match classified error types (these have specific handling)", () => { + const samples = [ + "rate limit exceeded", + "429 Too Many Requests", + "unauthorized", + "authentication failed", + "context length exceeded", + "prompt is too long", + "overloaded_error", + "timeout", + "timed out", + "billing", + "payment required", + ]; + for (const sample of samples) { + expect(isRetryableUnknownError(sample)).toBe(false); + } + }); + + it("does not match empty or undefined", () => { + expect(isRetryableUnknownError(undefined)).toBe(false); + expect(isRetryableUnknownError("")).toBe(false); + expect(isRetryableUnknownError(" ")).toBe(false); + }); + + it("does not match specific error messages that happen to contain 'unknown'", () => { + // These are not the generic "unknown error" pattern we want to retry + expect(isRetryableUnknownError("Unknown model: gpt-5")).toBe(false); + expect(isRetryableUnknownError("Unknown tool: some_tool")).toBe(false); + }); +}); + +describe("isRetryableUnknownAssistantError", () => { + const makeAssistant = (stopReason: string, errorMessage?: string): AssistantMessage => ({ + role: "assistant", + content: [], + stopReason: stopReason as AssistantMessage["stopReason"], + errorMessage, + api: "anthropic", + provider: "anthropic", + model: "claude-3-sonnet", + timestamp: Date.now(), + usage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + }); + + it("returns true for error stopReason with unknown error message", () => { + const msg = makeAssistant("error", "An unknown error occurred"); + expect(isRetryableUnknownAssistantError(msg)).toBe(true); + }); + + it("returns false for non-error stopReason", () => { + const msg = makeAssistant("stop", "An unknown error occurred"); + expect(isRetryableUnknownAssistantError(msg)).toBe(false); + }); + + it("returns false for error stopReason with classified error message", () => { + const msg = makeAssistant("error", "rate limit exceeded"); + expect(isRetryableUnknownAssistantError(msg)).toBe(false); + }); + + it("returns false for undefined message", () => { + expect(isRetryableUnknownAssistantError(undefined)).toBe(false); + }); +}); diff --git a/src/agents/pi-embedded-helpers.ts b/src/agents/pi-embedded-helpers.ts index 6f6bb474f..233b69f86 100644 --- a/src/agents/pi-embedded-helpers.ts +++ b/src/agents/pi-embedded-helpers.ts @@ -27,6 +27,8 @@ export { isRawApiErrorPayload, isRateLimitAssistantError, isRateLimitErrorMessage, + isRetryableUnknownAssistantError, + isRetryableUnknownError, isTimeoutErrorMessage, parseImageDimensionError, } from "./pi-embedded-helpers/errors.js"; diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index b47938c23..1cd9e7fcd 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -495,3 +495,30 @@ export function isFailoverAssistantError(msg: AssistantMessage | undefined): boo if (!msg || msg.stopReason !== "error") return false; return isFailoverErrorMessage(msg.errorMessage ?? ""); } + +/** + * Detects generic "unknown error" messages that may be transient and worth retrying. + * These typically come from providers that don't surface the real error message. + */ +export function isRetryableUnknownError(errorMessage?: string): boolean { + if (!errorMessage) return false; + const lower = errorMessage.toLowerCase().trim(); + // Don't retry if it's a known/classified error type + if (classifyFailoverReason(errorMessage)) return false; + if (isContextOverflowError(errorMessage)) return false; + if (isImageDimensionErrorMessage(errorMessage)) return false; + // Match generic unknown error patterns from pi-mono providers + return ( + lower === "unknown error" || + lower === "an unknown error occurred" || + lower === "an unkown error ocurred" || // typo variant + lower === "response failed" || + lower.startsWith("unknown error") || + /^an?\s+unkn?own\s+error/i.test(lower) + ); +} + +export function isRetryableUnknownAssistantError(msg: AssistantMessage | undefined): boolean { + if (!msg || msg.stopReason !== "error") return false; + return isRetryableUnknownError(msg.errorMessage); +} diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index adbe6ad49..627b18b6f 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -34,6 +34,7 @@ import { isContextOverflowError, isFailoverAssistantError, isFailoverErrorMessage, + isRetryableUnknownAssistantError, parseImageDimensionError, isRateLimitAssistantError, isTimeoutErrorMessage, @@ -135,7 +136,9 @@ export async function runEmbeddedPiAgent( ); } - const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false }); + const authStore = ensureAuthProfileStore(agentDir, { + allowKeychainPrompt: false, + }); const preferredProfileId = params.authProfileId?.trim(); let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined; if (lockedProfileId) { @@ -292,6 +295,8 @@ export async function runEmbeddedPiAgent( } let overflowCompactionAttempted = false; + let unknownErrorRetries = 0; + const MAX_UNKNOWN_ERROR_RETRIES = 2; try { while (true) { attemptedThinking.add(thinkLevel); @@ -581,6 +586,22 @@ export async function runEmbeddedPiAgent( } } + // Retry transient "unknown error" responses from providers (e.g. OpenAI response.failed) + // These are often caused by temporary service issues and may succeed on retry. + if ( + !aborted && + isRetryableUnknownAssistantError(lastAssistant) && + unknownErrorRetries < MAX_UNKNOWN_ERROR_RETRIES + ) { + unknownErrorRetries++; + const delayMs = 1000 * unknownErrorRetries; // 1s, 2s backoff + log.warn( + `Unknown error from ${provider}/${modelId}, retry ${unknownErrorRetries}/${MAX_UNKNOWN_ERROR_RETRIES} in ${delayMs}ms`, + ); + await new Promise((resolve) => setTimeout(resolve, delayMs)); + continue; + } + const usage = normalizeUsage(lastAssistant?.usage as UsageLike); const agentMeta: EmbeddedPiAgentMeta = { sessionId: sessionIdUsed,