fix: add retry logic for transient unknown errors from LLM providers

When providers return generic 'unknown error' messages (often from
temporary service issues), clawdbot now automatically retries up to
2 times with exponential backoff (1s, 2s).

- Add isRetryableUnknownError() to detect generic error patterns
- Add retry loop in runEmbeddedPiAgent for these transient errors
- Add unit tests for the new error detection logic
This commit is contained in:
Anthony 2026-01-26 15:01:32 +01:00
parent 6859e1e6a6
commit 9fc122a7f0
4 changed files with 146 additions and 1 deletions

View File

@ -0,0 +1,95 @@
import { describe, expect, it } from "vitest";
import {
isRetryableUnknownError,
isRetryableUnknownAssistantError,
} from "./pi-embedded-helpers.js";
import type { AssistantMessage } from "@mariozechner/pi-ai";
describe("isRetryableUnknownError", () => {
it("matches generic unknown error patterns from pi-mono providers", () => {
const samples = [
"Unknown error",
"unknown error",
"An unknown error occurred",
"an unknown error occurred",
"An unkown error ocurred", // typo variant from pi-mono
"Response failed",
"response failed",
"Unknown error: something went wrong",
];
for (const sample of samples) {
expect(isRetryableUnknownError(sample)).toBe(true);
}
});
it("does not match classified error types (these have specific handling)", () => {
const samples = [
"rate limit exceeded",
"429 Too Many Requests",
"unauthorized",
"authentication failed",
"context length exceeded",
"prompt is too long",
"overloaded_error",
"timeout",
"timed out",
"billing",
"payment required",
];
for (const sample of samples) {
expect(isRetryableUnknownError(sample)).toBe(false);
}
});
it("does not match empty or undefined", () => {
expect(isRetryableUnknownError(undefined)).toBe(false);
expect(isRetryableUnknownError("")).toBe(false);
expect(isRetryableUnknownError(" ")).toBe(false);
});
it("does not match specific error messages that happen to contain 'unknown'", () => {
// These are not the generic "unknown error" pattern we want to retry
expect(isRetryableUnknownError("Unknown model: gpt-5")).toBe(false);
expect(isRetryableUnknownError("Unknown tool: some_tool")).toBe(false);
});
});
describe("isRetryableUnknownAssistantError", () => {
const makeAssistant = (stopReason: string, errorMessage?: string): AssistantMessage => ({
role: "assistant",
content: [],
stopReason: stopReason as AssistantMessage["stopReason"],
errorMessage,
api: "anthropic",
provider: "anthropic",
model: "claude-3-sonnet",
timestamp: Date.now(),
usage: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
},
});
it("returns true for error stopReason with unknown error message", () => {
const msg = makeAssistant("error", "An unknown error occurred");
expect(isRetryableUnknownAssistantError(msg)).toBe(true);
});
it("returns false for non-error stopReason", () => {
const msg = makeAssistant("stop", "An unknown error occurred");
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
});
it("returns false for error stopReason with classified error message", () => {
const msg = makeAssistant("error", "rate limit exceeded");
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
});
it("returns false for undefined message", () => {
expect(isRetryableUnknownAssistantError(undefined)).toBe(false);
});
});

View File

@ -27,6 +27,8 @@ export {
isRawApiErrorPayload,
isRateLimitAssistantError,
isRateLimitErrorMessage,
isRetryableUnknownAssistantError,
isRetryableUnknownError,
isTimeoutErrorMessage,
parseImageDimensionError,
} from "./pi-embedded-helpers/errors.js";

View File

@ -495,3 +495,30 @@ export function isFailoverAssistantError(msg: AssistantMessage | undefined): boo
if (!msg || msg.stopReason !== "error") return false;
return isFailoverErrorMessage(msg.errorMessage ?? "");
}
/**
* Detects generic "unknown error" messages that may be transient and worth retrying.
* These typically come from providers that don't surface the real error message.
*/
export function isRetryableUnknownError(errorMessage?: string): boolean {
if (!errorMessage) return false;
const lower = errorMessage.toLowerCase().trim();
// Don't retry if it's a known/classified error type
if (classifyFailoverReason(errorMessage)) return false;
if (isContextOverflowError(errorMessage)) return false;
if (isImageDimensionErrorMessage(errorMessage)) return false;
// Match generic unknown error patterns from pi-mono providers
return (
lower === "unknown error" ||
lower === "an unknown error occurred" ||
lower === "an unkown error ocurred" || // typo variant
lower === "response failed" ||
lower.startsWith("unknown error") ||
/^an?\s+unkn?own\s+error/i.test(lower)
);
}
export function isRetryableUnknownAssistantError(msg: AssistantMessage | undefined): boolean {
if (!msg || msg.stopReason !== "error") return false;
return isRetryableUnknownError(msg.errorMessage);
}

View File

@ -34,6 +34,7 @@ import {
isContextOverflowError,
isFailoverAssistantError,
isFailoverErrorMessage,
isRetryableUnknownAssistantError,
parseImageDimensionError,
isRateLimitAssistantError,
isTimeoutErrorMessage,
@ -135,7 +136,9 @@ export async function runEmbeddedPiAgent(
);
}
const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
const authStore = ensureAuthProfileStore(agentDir, {
allowKeychainPrompt: false,
});
const preferredProfileId = params.authProfileId?.trim();
let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined;
if (lockedProfileId) {
@ -292,6 +295,8 @@ export async function runEmbeddedPiAgent(
}
let overflowCompactionAttempted = false;
let unknownErrorRetries = 0;
const MAX_UNKNOWN_ERROR_RETRIES = 2;
try {
while (true) {
attemptedThinking.add(thinkLevel);
@ -581,6 +586,22 @@ export async function runEmbeddedPiAgent(
}
}
// Retry transient "unknown error" responses from providers (e.g. OpenAI response.failed)
// These are often caused by temporary service issues and may succeed on retry.
if (
!aborted &&
isRetryableUnknownAssistantError(lastAssistant) &&
unknownErrorRetries < MAX_UNKNOWN_ERROR_RETRIES
) {
unknownErrorRetries++;
const delayMs = 1000 * unknownErrorRetries; // 1s, 2s backoff
log.warn(
`Unknown error from ${provider}/${modelId}, retry ${unknownErrorRetries}/${MAX_UNKNOWN_ERROR_RETRIES} in ${delayMs}ms`,
);
await new Promise((resolve) => setTimeout(resolve, delayMs));
continue;
}
const usage = normalizeUsage(lastAssistant?.usage as UsageLike);
const agentMeta: EmbeddedPiAgentMeta = {
sessionId: sessionIdUsed,