fix: add retry logic for transient unknown errors from LLM providers
When providers return generic 'unknown error' messages (often from temporary service issues), clawdbot now automatically retries up to 2 times with exponential backoff (1s, 2s). - Add isRetryableUnknownError() to detect generic error patterns - Add retry loop in runEmbeddedPiAgent for these transient errors - Add unit tests for the new error detection logic
This commit is contained in:
parent
6859e1e6a6
commit
9fc122a7f0
@ -0,0 +1,95 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
isRetryableUnknownError,
|
||||
isRetryableUnknownAssistantError,
|
||||
} from "./pi-embedded-helpers.js";
|
||||
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||
|
||||
describe("isRetryableUnknownError", () => {
|
||||
it("matches generic unknown error patterns from pi-mono providers", () => {
|
||||
const samples = [
|
||||
"Unknown error",
|
||||
"unknown error",
|
||||
"An unknown error occurred",
|
||||
"an unknown error occurred",
|
||||
"An unkown error ocurred", // typo variant from pi-mono
|
||||
"Response failed",
|
||||
"response failed",
|
||||
"Unknown error: something went wrong",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isRetryableUnknownError(sample)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not match classified error types (these have specific handling)", () => {
|
||||
const samples = [
|
||||
"rate limit exceeded",
|
||||
"429 Too Many Requests",
|
||||
"unauthorized",
|
||||
"authentication failed",
|
||||
"context length exceeded",
|
||||
"prompt is too long",
|
||||
"overloaded_error",
|
||||
"timeout",
|
||||
"timed out",
|
||||
"billing",
|
||||
"payment required",
|
||||
];
|
||||
for (const sample of samples) {
|
||||
expect(isRetryableUnknownError(sample)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not match empty or undefined", () => {
|
||||
expect(isRetryableUnknownError(undefined)).toBe(false);
|
||||
expect(isRetryableUnknownError("")).toBe(false);
|
||||
expect(isRetryableUnknownError(" ")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not match specific error messages that happen to contain 'unknown'", () => {
|
||||
// These are not the generic "unknown error" pattern we want to retry
|
||||
expect(isRetryableUnknownError("Unknown model: gpt-5")).toBe(false);
|
||||
expect(isRetryableUnknownError("Unknown tool: some_tool")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isRetryableUnknownAssistantError", () => {
|
||||
const makeAssistant = (stopReason: string, errorMessage?: string): AssistantMessage => ({
|
||||
role: "assistant",
|
||||
content: [],
|
||||
stopReason: stopReason as AssistantMessage["stopReason"],
|
||||
errorMessage,
|
||||
api: "anthropic",
|
||||
provider: "anthropic",
|
||||
model: "claude-3-sonnet",
|
||||
timestamp: Date.now(),
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
});
|
||||
|
||||
it("returns true for error stopReason with unknown error message", () => {
|
||||
const msg = makeAssistant("error", "An unknown error occurred");
|
||||
expect(isRetryableUnknownAssistantError(msg)).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false for non-error stopReason", () => {
|
||||
const msg = makeAssistant("stop", "An unknown error occurred");
|
||||
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for error stopReason with classified error message", () => {
|
||||
const msg = makeAssistant("error", "rate limit exceeded");
|
||||
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false for undefined message", () => {
|
||||
expect(isRetryableUnknownAssistantError(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
@ -27,6 +27,8 @@ export {
|
||||
isRawApiErrorPayload,
|
||||
isRateLimitAssistantError,
|
||||
isRateLimitErrorMessage,
|
||||
isRetryableUnknownAssistantError,
|
||||
isRetryableUnknownError,
|
||||
isTimeoutErrorMessage,
|
||||
parseImageDimensionError,
|
||||
} from "./pi-embedded-helpers/errors.js";
|
||||
|
||||
@ -495,3 +495,30 @@ export function isFailoverAssistantError(msg: AssistantMessage | undefined): boo
|
||||
if (!msg || msg.stopReason !== "error") return false;
|
||||
return isFailoverErrorMessage(msg.errorMessage ?? "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Detects generic "unknown error" messages that may be transient and worth retrying.
|
||||
* These typically come from providers that don't surface the real error message.
|
||||
*/
|
||||
export function isRetryableUnknownError(errorMessage?: string): boolean {
|
||||
if (!errorMessage) return false;
|
||||
const lower = errorMessage.toLowerCase().trim();
|
||||
// Don't retry if it's a known/classified error type
|
||||
if (classifyFailoverReason(errorMessage)) return false;
|
||||
if (isContextOverflowError(errorMessage)) return false;
|
||||
if (isImageDimensionErrorMessage(errorMessage)) return false;
|
||||
// Match generic unknown error patterns from pi-mono providers
|
||||
return (
|
||||
lower === "unknown error" ||
|
||||
lower === "an unknown error occurred" ||
|
||||
lower === "an unkown error ocurred" || // typo variant
|
||||
lower === "response failed" ||
|
||||
lower.startsWith("unknown error") ||
|
||||
/^an?\s+unkn?own\s+error/i.test(lower)
|
||||
);
|
||||
}
|
||||
|
||||
export function isRetryableUnknownAssistantError(msg: AssistantMessage | undefined): boolean {
|
||||
if (!msg || msg.stopReason !== "error") return false;
|
||||
return isRetryableUnknownError(msg.errorMessage);
|
||||
}
|
||||
|
||||
@ -34,6 +34,7 @@ import {
|
||||
isContextOverflowError,
|
||||
isFailoverAssistantError,
|
||||
isFailoverErrorMessage,
|
||||
isRetryableUnknownAssistantError,
|
||||
parseImageDimensionError,
|
||||
isRateLimitAssistantError,
|
||||
isTimeoutErrorMessage,
|
||||
@ -135,7 +136,9 @@ export async function runEmbeddedPiAgent(
|
||||
);
|
||||
}
|
||||
|
||||
const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
|
||||
const authStore = ensureAuthProfileStore(agentDir, {
|
||||
allowKeychainPrompt: false,
|
||||
});
|
||||
const preferredProfileId = params.authProfileId?.trim();
|
||||
let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined;
|
||||
if (lockedProfileId) {
|
||||
@ -292,6 +295,8 @@ export async function runEmbeddedPiAgent(
|
||||
}
|
||||
|
||||
let overflowCompactionAttempted = false;
|
||||
let unknownErrorRetries = 0;
|
||||
const MAX_UNKNOWN_ERROR_RETRIES = 2;
|
||||
try {
|
||||
while (true) {
|
||||
attemptedThinking.add(thinkLevel);
|
||||
@ -581,6 +586,22 @@ export async function runEmbeddedPiAgent(
|
||||
}
|
||||
}
|
||||
|
||||
// Retry transient "unknown error" responses from providers (e.g. OpenAI response.failed)
|
||||
// These are often caused by temporary service issues and may succeed on retry.
|
||||
if (
|
||||
!aborted &&
|
||||
isRetryableUnknownAssistantError(lastAssistant) &&
|
||||
unknownErrorRetries < MAX_UNKNOWN_ERROR_RETRIES
|
||||
) {
|
||||
unknownErrorRetries++;
|
||||
const delayMs = 1000 * unknownErrorRetries; // 1s, 2s backoff
|
||||
log.warn(
|
||||
`Unknown error from ${provider}/${modelId}, retry ${unknownErrorRetries}/${MAX_UNKNOWN_ERROR_RETRIES} in ${delayMs}ms`,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||
continue;
|
||||
}
|
||||
|
||||
const usage = normalizeUsage(lastAssistant?.usage as UsageLike);
|
||||
const agentMeta: EmbeddedPiAgentMeta = {
|
||||
sessionId: sessionIdUsed,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user