fix: add retry logic for transient unknown errors from LLM providers
When providers return generic 'unknown error' messages (often from temporary service issues), clawdbot now automatically retries up to 2 times with exponential backoff (1s, 2s). - Add isRetryableUnknownError() to detect generic error patterns - Add retry loop in runEmbeddedPiAgent for these transient errors - Add unit tests for the new error detection logic
This commit is contained in:
parent
6859e1e6a6
commit
9fc122a7f0
@ -0,0 +1,95 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import {
|
||||||
|
isRetryableUnknownError,
|
||||||
|
isRetryableUnknownAssistantError,
|
||||||
|
} from "./pi-embedded-helpers.js";
|
||||||
|
import type { AssistantMessage } from "@mariozechner/pi-ai";
|
||||||
|
|
||||||
|
describe("isRetryableUnknownError", () => {
|
||||||
|
it("matches generic unknown error patterns from pi-mono providers", () => {
|
||||||
|
const samples = [
|
||||||
|
"Unknown error",
|
||||||
|
"unknown error",
|
||||||
|
"An unknown error occurred",
|
||||||
|
"an unknown error occurred",
|
||||||
|
"An unkown error ocurred", // typo variant from pi-mono
|
||||||
|
"Response failed",
|
||||||
|
"response failed",
|
||||||
|
"Unknown error: something went wrong",
|
||||||
|
];
|
||||||
|
for (const sample of samples) {
|
||||||
|
expect(isRetryableUnknownError(sample)).toBe(true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not match classified error types (these have specific handling)", () => {
|
||||||
|
const samples = [
|
||||||
|
"rate limit exceeded",
|
||||||
|
"429 Too Many Requests",
|
||||||
|
"unauthorized",
|
||||||
|
"authentication failed",
|
||||||
|
"context length exceeded",
|
||||||
|
"prompt is too long",
|
||||||
|
"overloaded_error",
|
||||||
|
"timeout",
|
||||||
|
"timed out",
|
||||||
|
"billing",
|
||||||
|
"payment required",
|
||||||
|
];
|
||||||
|
for (const sample of samples) {
|
||||||
|
expect(isRetryableUnknownError(sample)).toBe(false);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not match empty or undefined", () => {
|
||||||
|
expect(isRetryableUnknownError(undefined)).toBe(false);
|
||||||
|
expect(isRetryableUnknownError("")).toBe(false);
|
||||||
|
expect(isRetryableUnknownError(" ")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not match specific error messages that happen to contain 'unknown'", () => {
|
||||||
|
// These are not the generic "unknown error" pattern we want to retry
|
||||||
|
expect(isRetryableUnknownError("Unknown model: gpt-5")).toBe(false);
|
||||||
|
expect(isRetryableUnknownError("Unknown tool: some_tool")).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("isRetryableUnknownAssistantError", () => {
|
||||||
|
const makeAssistant = (stopReason: string, errorMessage?: string): AssistantMessage => ({
|
||||||
|
role: "assistant",
|
||||||
|
content: [],
|
||||||
|
stopReason: stopReason as AssistantMessage["stopReason"],
|
||||||
|
errorMessage,
|
||||||
|
api: "anthropic",
|
||||||
|
provider: "anthropic",
|
||||||
|
model: "claude-3-sonnet",
|
||||||
|
timestamp: Date.now(),
|
||||||
|
usage: {
|
||||||
|
input: 0,
|
||||||
|
output: 0,
|
||||||
|
cacheRead: 0,
|
||||||
|
cacheWrite: 0,
|
||||||
|
totalTokens: 0,
|
||||||
|
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns true for error stopReason with unknown error message", () => {
|
||||||
|
const msg = makeAssistant("error", "An unknown error occurred");
|
||||||
|
expect(isRetryableUnknownAssistantError(msg)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns false for non-error stopReason", () => {
|
||||||
|
const msg = makeAssistant("stop", "An unknown error occurred");
|
||||||
|
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns false for error stopReason with classified error message", () => {
|
||||||
|
const msg = makeAssistant("error", "rate limit exceeded");
|
||||||
|
expect(isRetryableUnknownAssistantError(msg)).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns false for undefined message", () => {
|
||||||
|
expect(isRetryableUnknownAssistantError(undefined)).toBe(false);
|
||||||
|
});
|
||||||
|
});
|
||||||
@ -27,6 +27,8 @@ export {
|
|||||||
isRawApiErrorPayload,
|
isRawApiErrorPayload,
|
||||||
isRateLimitAssistantError,
|
isRateLimitAssistantError,
|
||||||
isRateLimitErrorMessage,
|
isRateLimitErrorMessage,
|
||||||
|
isRetryableUnknownAssistantError,
|
||||||
|
isRetryableUnknownError,
|
||||||
isTimeoutErrorMessage,
|
isTimeoutErrorMessage,
|
||||||
parseImageDimensionError,
|
parseImageDimensionError,
|
||||||
} from "./pi-embedded-helpers/errors.js";
|
} from "./pi-embedded-helpers/errors.js";
|
||||||
|
|||||||
@ -495,3 +495,30 @@ export function isFailoverAssistantError(msg: AssistantMessage | undefined): boo
|
|||||||
if (!msg || msg.stopReason !== "error") return false;
|
if (!msg || msg.stopReason !== "error") return false;
|
||||||
return isFailoverErrorMessage(msg.errorMessage ?? "");
|
return isFailoverErrorMessage(msg.errorMessage ?? "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Detects generic "unknown error" messages that may be transient and worth retrying.
|
||||||
|
* These typically come from providers that don't surface the real error message.
|
||||||
|
*/
|
||||||
|
export function isRetryableUnknownError(errorMessage?: string): boolean {
|
||||||
|
if (!errorMessage) return false;
|
||||||
|
const lower = errorMessage.toLowerCase().trim();
|
||||||
|
// Don't retry if it's a known/classified error type
|
||||||
|
if (classifyFailoverReason(errorMessage)) return false;
|
||||||
|
if (isContextOverflowError(errorMessage)) return false;
|
||||||
|
if (isImageDimensionErrorMessage(errorMessage)) return false;
|
||||||
|
// Match generic unknown error patterns from pi-mono providers
|
||||||
|
return (
|
||||||
|
lower === "unknown error" ||
|
||||||
|
lower === "an unknown error occurred" ||
|
||||||
|
lower === "an unkown error ocurred" || // typo variant
|
||||||
|
lower === "response failed" ||
|
||||||
|
lower.startsWith("unknown error") ||
|
||||||
|
/^an?\s+unkn?own\s+error/i.test(lower)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isRetryableUnknownAssistantError(msg: AssistantMessage | undefined): boolean {
|
||||||
|
if (!msg || msg.stopReason !== "error") return false;
|
||||||
|
return isRetryableUnknownError(msg.errorMessage);
|
||||||
|
}
|
||||||
|
|||||||
@ -34,6 +34,7 @@ import {
|
|||||||
isContextOverflowError,
|
isContextOverflowError,
|
||||||
isFailoverAssistantError,
|
isFailoverAssistantError,
|
||||||
isFailoverErrorMessage,
|
isFailoverErrorMessage,
|
||||||
|
isRetryableUnknownAssistantError,
|
||||||
parseImageDimensionError,
|
parseImageDimensionError,
|
||||||
isRateLimitAssistantError,
|
isRateLimitAssistantError,
|
||||||
isTimeoutErrorMessage,
|
isTimeoutErrorMessage,
|
||||||
@ -135,7 +136,9 @@ export async function runEmbeddedPiAgent(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false });
|
const authStore = ensureAuthProfileStore(agentDir, {
|
||||||
|
allowKeychainPrompt: false,
|
||||||
|
});
|
||||||
const preferredProfileId = params.authProfileId?.trim();
|
const preferredProfileId = params.authProfileId?.trim();
|
||||||
let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined;
|
let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined;
|
||||||
if (lockedProfileId) {
|
if (lockedProfileId) {
|
||||||
@ -292,6 +295,8 @@ export async function runEmbeddedPiAgent(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let overflowCompactionAttempted = false;
|
let overflowCompactionAttempted = false;
|
||||||
|
let unknownErrorRetries = 0;
|
||||||
|
const MAX_UNKNOWN_ERROR_RETRIES = 2;
|
||||||
try {
|
try {
|
||||||
while (true) {
|
while (true) {
|
||||||
attemptedThinking.add(thinkLevel);
|
attemptedThinking.add(thinkLevel);
|
||||||
@ -581,6 +586,22 @@ export async function runEmbeddedPiAgent(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Retry transient "unknown error" responses from providers (e.g. OpenAI response.failed)
|
||||||
|
// These are often caused by temporary service issues and may succeed on retry.
|
||||||
|
if (
|
||||||
|
!aborted &&
|
||||||
|
isRetryableUnknownAssistantError(lastAssistant) &&
|
||||||
|
unknownErrorRetries < MAX_UNKNOWN_ERROR_RETRIES
|
||||||
|
) {
|
||||||
|
unknownErrorRetries++;
|
||||||
|
const delayMs = 1000 * unknownErrorRetries; // 1s, 2s backoff
|
||||||
|
log.warn(
|
||||||
|
`Unknown error from ${provider}/${modelId}, retry ${unknownErrorRetries}/${MAX_UNKNOWN_ERROR_RETRIES} in ${delayMs}ms`,
|
||||||
|
);
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
const usage = normalizeUsage(lastAssistant?.usage as UsageLike);
|
const usage = normalizeUsage(lastAssistant?.usage as UsageLike);
|
||||||
const agentMeta: EmbeddedPiAgentMeta = {
|
const agentMeta: EmbeddedPiAgentMeta = {
|
||||||
sessionId: sessionIdUsed,
|
sessionId: sessionIdUsed,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user