diff --git a/src/agents/pi-embedded-runner/compact.ts b/src/agents/pi-embedded-runner/compact.ts index dc68561c2..6164c28a1 100644 --- a/src/agents/pi-embedded-runner/compact.ts +++ b/src/agents/pi-embedded-runner/compact.ts @@ -100,6 +100,7 @@ export type CompactEmbeddedPiSessionParams = { enqueue?: typeof enqueueCommand; extraSystemPrompt?: string; ownerNumbers?: string[]; + skipSkillEnvOverrides?: boolean; }; /** @@ -110,7 +111,6 @@ export async function compactEmbeddedPiSessionDirect( params: CompactEmbeddedPiSessionParams, ): Promise { const resolvedWorkspace = resolveUserPath(params.workspaceDir); - const prevCwd = process.cwd(); const provider = (params.provider ?? DEFAULT_PROVIDER).trim() || DEFAULT_PROVIDER; const modelId = (params.model ?? DEFAULT_MODEL).trim() || DEFAULT_MODEL; @@ -179,22 +179,24 @@ export async function compactEmbeddedPiSessionDirect( cwd: effectiveWorkspace, }); + const shouldApplySkillEnvOverrides = !params.skipSkillEnvOverrides; let restoreSkillEnv: (() => void) | undefined; - process.chdir(effectiveWorkspace); try { const shouldLoadSkillEntries = !params.skillsSnapshot || !params.skillsSnapshot.resolvedSkills; const skillEntries = shouldLoadSkillEntries ? loadWorkspaceSkillEntries(effectiveWorkspace) : []; - restoreSkillEnv = params.skillsSnapshot - ? applySkillEnvOverridesFromSnapshot({ - snapshot: params.skillsSnapshot, - config: params.config, - }) - : applySkillEnvOverrides({ - skills: skillEntries ?? [], - config: params.config, - }); + if (shouldApplySkillEnvOverrides) { + restoreSkillEnv = params.skillsSnapshot + ? applySkillEnvOverridesFromSnapshot({ + snapshot: params.skillsSnapshot, + config: params.config, + }) + : applySkillEnvOverrides({ + skills: skillEntries ?? [], + config: params.config, + }); + } const skillsPrompt = resolveSkillsPromptForRun({ skillsSnapshot: params.skillsSnapshot, entries: shouldLoadSkillEntries ? skillEntries : undefined, @@ -318,7 +320,7 @@ export async function compactEmbeddedPiSessionDirect( const docsPath = await resolveMoltbotDocsPath({ workspaceDir: effectiveWorkspace, argv1: process.argv[1], - cwd: process.cwd(), + cwd: effectiveWorkspace, moduleUrl: import.meta.url, }); const ttsHint = params.config ? buildTtsSystemPromptHint(params.config) : undefined; @@ -466,7 +468,6 @@ export async function compactEmbeddedPiSessionDirect( }; } finally { restoreSkillEnv?.(); - process.chdir(prevCwd); } } diff --git a/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts b/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts index 2d766ef35..66f15ffda 100644 --- a/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts +++ b/src/agents/pi-embedded-runner/run.overflow-compaction.test.ts @@ -168,6 +168,7 @@ function makeAttemptResult( didSendViaMessagingTool: false, messagingToolSentTexts: [], messagingToolSentTargets: [], + autoCompactionAttempts: 0, cloudCodeAssistFormatError: false, ...overrides, }; @@ -220,6 +221,20 @@ describe("overflow compaction in run loop", () => { expect(result.meta.error).toBeUndefined(); }); + it("skips manual compaction when auto-compaction already ran", async () => { + const overflowError = new Error("request_too_large: Request size exceeds model context window"); + + mockedRunEmbeddedAttempt.mockResolvedValueOnce( + makeAttemptResult({ promptError: overflowError, autoCompactionAttempts: 1 }), + ); + + const result = await runEmbeddedPiAgent(baseParams); + + expect(mockedCompactDirect).not.toHaveBeenCalled(); + expect(mockedRunEmbeddedAttempt).toHaveBeenCalledTimes(1); + expect(result.meta.error?.kind).toBe("context_overflow"); + }); + it("returns error if compaction fails", async () => { const overflowError = new Error("request_too_large: Request size exceeds model context window"); diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 870453f38..47b17c888 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -86,222 +86,216 @@ export async function runEmbeddedPiAgent( : "markdown"); const isProbeSession = params.sessionId?.startsWith("probe-") ?? false; - return enqueueSession(() => - enqueueGlobal(async () => { - const started = Date.now(); - const resolvedWorkspace = resolveUserPath(params.workspaceDir); - const prevCwd = process.cwd(); + const started = Date.now(); + const resolvedWorkspace = resolveUserPath(params.workspaceDir); + const prevCwd = process.cwd(); - const provider = (params.provider ?? DEFAULT_PROVIDER).trim() || DEFAULT_PROVIDER; - const modelId = (params.model ?? DEFAULT_MODEL).trim() || DEFAULT_MODEL; - const agentDir = params.agentDir ?? resolveMoltbotAgentDir(); - const fallbackConfigured = - (params.config?.agents?.defaults?.model?.fallbacks?.length ?? 0) > 0; - await ensureMoltbotModelsJson(params.config, agentDir); + const provider = (params.provider ?? DEFAULT_PROVIDER).trim() || DEFAULT_PROVIDER; + const modelId = (params.model ?? DEFAULT_MODEL).trim() || DEFAULT_MODEL; + const agentDir = params.agentDir ?? resolveMoltbotAgentDir(); + const fallbackConfigured = (params.config?.agents?.defaults?.model?.fallbacks?.length ?? 0) > 0; + await ensureMoltbotModelsJson(params.config, agentDir); - const { model, error, authStorage, modelRegistry } = resolveModel( + const { model, error, authStorage, modelRegistry } = resolveModel( + provider, + modelId, + agentDir, + params.config, + ); + if (!model) { + throw new Error(error ?? `Unknown model: ${provider}/${modelId}`); + } + + const ctxInfo = resolveContextWindowInfo({ + cfg: params.config, + provider, + modelId, + modelContextWindow: model.contextWindow, + defaultTokens: DEFAULT_CONTEXT_TOKENS, + }); + const ctxGuard = evaluateContextWindowGuard({ + info: ctxInfo, + warnBelowTokens: CONTEXT_WINDOW_WARN_BELOW_TOKENS, + hardMinTokens: CONTEXT_WINDOW_HARD_MIN_TOKENS, + }); + if (ctxGuard.shouldWarn) { + log.warn( + `low context window: ${provider}/${modelId} ctx=${ctxGuard.tokens} (warn<${CONTEXT_WINDOW_WARN_BELOW_TOKENS}) source=${ctxGuard.source}`, + ); + } + if (ctxGuard.shouldBlock) { + log.error( + `blocked model (context window too small): ${provider}/${modelId} ctx=${ctxGuard.tokens} (min=${CONTEXT_WINDOW_HARD_MIN_TOKENS}) source=${ctxGuard.source}`, + ); + throw new FailoverError( + `Model context window too small (${ctxGuard.tokens} tokens). Minimum is ${CONTEXT_WINDOW_HARD_MIN_TOKENS}.`, + { reason: "unknown", provider, model: modelId }, + ); + } + + const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false }); + const preferredProfileId = params.authProfileId?.trim(); + let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined; + if (lockedProfileId) { + const lockedProfile = authStore.profiles[lockedProfileId]; + if ( + !lockedProfile || + normalizeProviderId(lockedProfile.provider) !== normalizeProviderId(provider) + ) { + lockedProfileId = undefined; + } + } + const profileOrder = resolveAuthProfileOrder({ + cfg: params.config, + store: authStore, + provider, + preferredProfile: preferredProfileId, + }); + if (lockedProfileId && !profileOrder.includes(lockedProfileId)) { + throw new Error(`Auth profile "${lockedProfileId}" is not configured for ${provider}.`); + } + const profileCandidates = lockedProfileId + ? [lockedProfileId] + : profileOrder.length > 0 + ? profileOrder + : [undefined]; + let profileIndex = 0; + + const initialThinkLevel = params.thinkLevel ?? "off"; + let thinkLevel = initialThinkLevel; + const attemptedThinking = new Set(); + let apiKeyInfo: ApiKeyInfo | null = null; + let lastProfileId: string | undefined; + + const resolveAuthProfileFailoverReason = (params: { + allInCooldown: boolean; + message: string; + }): FailoverReason => { + if (params.allInCooldown) return "rate_limit"; + const classified = classifyFailoverReason(params.message); + return classified ?? "auth"; + }; + + const throwAuthProfileFailover = (params: { + allInCooldown: boolean; + message?: string; + error?: unknown; + }): never => { + const fallbackMessage = `No available auth profile for ${provider} (all in cooldown or unavailable).`; + const message = + params.message?.trim() || + (params.error ? describeUnknownError(params.error).trim() : "") || + fallbackMessage; + const reason = resolveAuthProfileFailoverReason({ + allInCooldown: params.allInCooldown, + message, + }); + if (fallbackConfigured) { + throw new FailoverError(message, { + reason, provider, - modelId, - agentDir, - params.config, - ); - if (!model) { - throw new Error(error ?? `Unknown model: ${provider}/${modelId}`); - } + model: modelId, + status: resolveFailoverStatus(reason), + cause: params.error, + }); + } + if (params.error instanceof Error) throw params.error; + throw new Error(message); + }; - const ctxInfo = resolveContextWindowInfo({ - cfg: params.config, - provider, - modelId, - modelContextWindow: model.contextWindow, - defaultTokens: DEFAULT_CONTEXT_TOKENS, - }); - const ctxGuard = evaluateContextWindowGuard({ - info: ctxInfo, - warnBelowTokens: CONTEXT_WINDOW_WARN_BELOW_TOKENS, - hardMinTokens: CONTEXT_WINDOW_HARD_MIN_TOKENS, - }); - if (ctxGuard.shouldWarn) { - log.warn( - `low context window: ${provider}/${modelId} ctx=${ctxGuard.tokens} (warn<${CONTEXT_WINDOW_WARN_BELOW_TOKENS}) source=${ctxGuard.source}`, + const resolveApiKeyForCandidate = async (candidate?: string) => { + return getApiKeyForModel({ + model, + cfg: params.config, + profileId: candidate, + store: authStore, + agentDir, + }); + }; + + const applyApiKeyInfo = async (candidate?: string): Promise => { + apiKeyInfo = await resolveApiKeyForCandidate(candidate); + const resolvedProfileId = apiKeyInfo.profileId ?? candidate; + if (!apiKeyInfo.apiKey) { + if (apiKeyInfo.mode !== "aws-sdk") { + throw new Error( + `No API key resolved for provider "${model.provider}" (auth mode: ${apiKeyInfo.mode}).`, ); } - if (ctxGuard.shouldBlock) { - log.error( - `blocked model (context window too small): ${provider}/${modelId} ctx=${ctxGuard.tokens} (min=${CONTEXT_WINDOW_HARD_MIN_TOKENS}) source=${ctxGuard.source}`, - ); - throw new FailoverError( - `Model context window too small (${ctxGuard.tokens} tokens). Minimum is ${CONTEXT_WINDOW_HARD_MIN_TOKENS}.`, - { reason: "unknown", provider, model: modelId }, - ); - } - - const authStore = ensureAuthProfileStore(agentDir, { allowKeychainPrompt: false }); - const preferredProfileId = params.authProfileId?.trim(); - let lockedProfileId = params.authProfileIdSource === "user" ? preferredProfileId : undefined; - if (lockedProfileId) { - const lockedProfile = authStore.profiles[lockedProfileId]; - if ( - !lockedProfile || - normalizeProviderId(lockedProfile.provider) !== normalizeProviderId(provider) - ) { - lockedProfileId = undefined; - } - } - const profileOrder = resolveAuthProfileOrder({ - cfg: params.config, - store: authStore, - provider, - preferredProfile: preferredProfileId, + lastProfileId = resolvedProfileId; + return; + } + if (model.provider === "github-copilot") { + const { resolveCopilotApiToken } = await import("../../providers/github-copilot-token.js"); + const copilotToken = await resolveCopilotApiToken({ + githubToken: apiKeyInfo.apiKey, }); - if (lockedProfileId && !profileOrder.includes(lockedProfileId)) { - throw new Error(`Auth profile "${lockedProfileId}" is not configured for ${provider}.`); + authStorage.setRuntimeApiKey(model.provider, copilotToken.token); + } else { + authStorage.setRuntimeApiKey(model.provider, apiKeyInfo.apiKey); + } + lastProfileId = apiKeyInfo.profileId; + }; + + const advanceAuthProfile = async (): Promise => { + if (lockedProfileId) return false; + let nextIndex = profileIndex + 1; + while (nextIndex < profileCandidates.length) { + const candidate = profileCandidates[nextIndex]; + if (candidate && isProfileInCooldown(authStore, candidate)) { + nextIndex += 1; + continue; } - const profileCandidates = lockedProfileId - ? [lockedProfileId] - : profileOrder.length > 0 - ? profileOrder - : [undefined]; - let profileIndex = 0; - - const initialThinkLevel = params.thinkLevel ?? "off"; - let thinkLevel = initialThinkLevel; - const attemptedThinking = new Set(); - let apiKeyInfo: ApiKeyInfo | null = null; - let lastProfileId: string | undefined; - - const resolveAuthProfileFailoverReason = (params: { - allInCooldown: boolean; - message: string; - }): FailoverReason => { - if (params.allInCooldown) return "rate_limit"; - const classified = classifyFailoverReason(params.message); - return classified ?? "auth"; - }; - - const throwAuthProfileFailover = (params: { - allInCooldown: boolean; - message?: string; - error?: unknown; - }): never => { - const fallbackMessage = `No available auth profile for ${provider} (all in cooldown or unavailable).`; - const message = - params.message?.trim() || - (params.error ? describeUnknownError(params.error).trim() : "") || - fallbackMessage; - const reason = resolveAuthProfileFailoverReason({ - allInCooldown: params.allInCooldown, - message, - }); - if (fallbackConfigured) { - throw new FailoverError(message, { - reason, - provider, - model: modelId, - status: resolveFailoverStatus(reason), - cause: params.error, - }); - } - if (params.error instanceof Error) throw params.error; - throw new Error(message); - }; - - const resolveApiKeyForCandidate = async (candidate?: string) => { - return getApiKeyForModel({ - model, - cfg: params.config, - profileId: candidate, - store: authStore, - agentDir, - }); - }; - - const applyApiKeyInfo = async (candidate?: string): Promise => { - apiKeyInfo = await resolveApiKeyForCandidate(candidate); - const resolvedProfileId = apiKeyInfo.profileId ?? candidate; - if (!apiKeyInfo.apiKey) { - if (apiKeyInfo.mode !== "aws-sdk") { - throw new Error( - `No API key resolved for provider "${model.provider}" (auth mode: ${apiKeyInfo.mode}).`, - ); - } - lastProfileId = resolvedProfileId; - return; - } - if (model.provider === "github-copilot") { - const { resolveCopilotApiToken } = - await import("../../providers/github-copilot-token.js"); - const copilotToken = await resolveCopilotApiToken({ - githubToken: apiKeyInfo.apiKey, - }); - authStorage.setRuntimeApiKey(model.provider, copilotToken.token); - } else { - authStorage.setRuntimeApiKey(model.provider, apiKeyInfo.apiKey); - } - lastProfileId = apiKeyInfo.profileId; - }; - - const advanceAuthProfile = async (): Promise => { - if (lockedProfileId) return false; - let nextIndex = profileIndex + 1; - while (nextIndex < profileCandidates.length) { - const candidate = profileCandidates[nextIndex]; - if (candidate && isProfileInCooldown(authStore, candidate)) { - nextIndex += 1; - continue; - } - try { - await applyApiKeyInfo(candidate); - profileIndex = nextIndex; - thinkLevel = initialThinkLevel; - attemptedThinking.clear(); - return true; - } catch (err) { - if (candidate && candidate === lockedProfileId) throw err; - nextIndex += 1; - } - } - return false; - }; - try { - while (profileIndex < profileCandidates.length) { - const candidate = profileCandidates[profileIndex]; - if ( - candidate && - candidate !== lockedProfileId && - isProfileInCooldown(authStore, candidate) - ) { - profileIndex += 1; - continue; - } - await applyApiKeyInfo(profileCandidates[profileIndex]); - break; - } - if (profileIndex >= profileCandidates.length) { - throwAuthProfileFailover({ allInCooldown: true }); - } + await applyApiKeyInfo(candidate); + profileIndex = nextIndex; + thinkLevel = initialThinkLevel; + attemptedThinking.clear(); + return true; } catch (err) { - if (err instanceof FailoverError) throw err; - if (profileCandidates[profileIndex] === lockedProfileId) { - throwAuthProfileFailover({ allInCooldown: false, error: err }); - } - const advanced = await advanceAuthProfile(); - if (!advanced) { - throwAuthProfileFailover({ allInCooldown: false, error: err }); - } + if (candidate && candidate === lockedProfileId) throw err; + nextIndex += 1; } + } + return false; + }; - let overflowCompactionAttempted = false; - try { - while (true) { - attemptedThinking.add(thinkLevel); - await fs.mkdir(resolvedWorkspace, { recursive: true }); + try { + while (profileIndex < profileCandidates.length) { + const candidate = profileCandidates[profileIndex]; + if (candidate && candidate !== lockedProfileId && isProfileInCooldown(authStore, candidate)) { + profileIndex += 1; + continue; + } + await applyApiKeyInfo(profileCandidates[profileIndex]); + break; + } + if (profileIndex >= profileCandidates.length) { + throwAuthProfileFailover({ allInCooldown: true }); + } + } catch (err) { + if (err instanceof FailoverError) throw err; + if (profileCandidates[profileIndex] === lockedProfileId) { + throwAuthProfileFailover({ allInCooldown: false, error: err }); + } + const advanced = await advanceAuthProfile(); + if (!advanced) { + throwAuthProfileFailover({ allInCooldown: false, error: err }); + } + } - const prompt = - provider === "anthropic" ? scrubAnthropicRefusalMagic(params.prompt) : params.prompt; + let overflowCompactionAttempted = false; + try { + while (true) { + attemptedThinking.add(thinkLevel); + await fs.mkdir(resolvedWorkspace, { recursive: true }); - const attempt = await runEmbeddedAttempt({ + const prompt = + provider === "anthropic" ? scrubAnthropicRefusalMagic(params.prompt) : params.prompt; + + const attempt = await enqueueSession(() => + enqueueGlobal(async () => + runEmbeddedAttempt({ sessionId: params.sessionId, sessionKey: params.sessionKey, messageChannel: params.messageChannel, @@ -354,326 +348,340 @@ export async function runEmbeddedPiAgent( streamParams: params.streamParams, ownerNumbers: params.ownerNumbers, enforceFinalTag: params.enforceFinalTag, - }); + }), + ), + ); - const { aborted, promptError, timedOut, sessionIdUsed, lastAssistant } = attempt; + const { + aborted, + promptError, + timedOut, + sessionIdUsed, + lastAssistant, + autoCompactionAttempts, + } = attempt; - if (promptError && !aborted) { - const errorText = describeUnknownError(promptError); - if (isContextOverflowError(errorText)) { - const isCompactionFailure = isCompactionFailureError(errorText); - // Attempt auto-compaction on context overflow (not compaction_failure) - if (!isCompactionFailure && !overflowCompactionAttempted) { - log.warn( - `context overflow detected; attempting auto-compaction for ${provider}/${modelId}`, - ); - overflowCompactionAttempted = true; - const compactResult = await compactEmbeddedPiSessionDirect({ - sessionId: params.sessionId, - sessionKey: params.sessionKey, - messageChannel: params.messageChannel, - messageProvider: params.messageProvider, - agentAccountId: params.agentAccountId, - authProfileId: lastProfileId, - sessionFile: params.sessionFile, - workspaceDir: params.workspaceDir, - agentDir, - config: params.config, - skillsSnapshot: params.skillsSnapshot, - provider, - model: modelId, - thinkLevel, - reasoningLevel: params.reasoningLevel, - bashElevated: params.bashElevated, - extraSystemPrompt: params.extraSystemPrompt, - ownerNumbers: params.ownerNumbers, - }); - if (compactResult.compacted) { - log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`); - continue; - } - log.warn( - `auto-compaction failed for ${provider}/${modelId}: ${compactResult.reason ?? "nothing to compact"}`, - ); - } - const kind = isCompactionFailure ? "compaction_failure" : "context_overflow"; - return { - payloads: [ - { - text: - "Context overflow: prompt too large for the model. " + - "Try again with less input or a larger-context model.", - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: { - sessionId: sessionIdUsed, - provider, - model: model.id, - }, - systemPromptReport: attempt.systemPromptReport, - error: { kind, message: errorText }, - }, - }; - } - // Handle role ordering errors with a user-friendly message - if (/incorrect role information|roles must alternate/i.test(errorText)) { - return { - payloads: [ - { - text: - "Message ordering conflict - please try again. " + - "If this persists, use /new to start a fresh session.", - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: { - sessionId: sessionIdUsed, - provider, - model: model.id, - }, - systemPromptReport: attempt.systemPromptReport, - error: { kind: "role_ordering", message: errorText }, - }, - }; - } - // Handle image size errors with a user-friendly message (no retry needed) - const imageSizeError = parseImageSizeError(errorText); - if (imageSizeError) { - const maxMb = imageSizeError.maxMb; - const maxMbLabel = - typeof maxMb === "number" && Number.isFinite(maxMb) ? `${maxMb}` : null; - const maxBytesHint = maxMbLabel ? ` (max ${maxMbLabel}MB)` : ""; - return { - payloads: [ - { - text: - `Image too large for the model${maxBytesHint}. ` + - "Please compress or resize the image and try again.", - isError: true, - }, - ], - meta: { - durationMs: Date.now() - started, - agentMeta: { - sessionId: sessionIdUsed, - provider, - model: model.id, - }, - systemPromptReport: attempt.systemPromptReport, - error: { kind: "image_size", message: errorText }, - }, - }; - } - const promptFailoverReason = classifyFailoverReason(errorText); - if (promptFailoverReason && promptFailoverReason !== "timeout" && lastProfileId) { - await markAuthProfileFailure({ - store: authStore, - profileId: lastProfileId, - reason: promptFailoverReason, - cfg: params.config, - agentDir: params.agentDir, - }); - } - if ( - isFailoverErrorMessage(errorText) && - promptFailoverReason !== "timeout" && - (await advanceAuthProfile()) - ) { - continue; - } - const fallbackThinking = pickFallbackThinkingLevel({ - message: errorText, - attempted: attemptedThinking, - }); - if (fallbackThinking) { - log.warn( - `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, - ); - thinkLevel = fallbackThinking; - continue; - } - // FIX: Throw FailoverError for prompt errors when fallbacks configured - // This enables model fallback for quota/rate limit errors during prompt submission - if (fallbackConfigured && isFailoverErrorMessage(errorText)) { - throw new FailoverError(errorText, { - reason: promptFailoverReason ?? "unknown", + if (promptError && !aborted) { + const errorText = describeUnknownError(promptError); + if (isContextOverflowError(errorText)) { + const isCompactionFailure = isCompactionFailureError(errorText); + // Attempt auto-compaction on context overflow (not compaction_failure) + if ( + !isCompactionFailure && + !overflowCompactionAttempted && + autoCompactionAttempts === 0 + ) { + log.warn( + `context overflow detected; attempting auto-compaction for ${provider}/${modelId}`, + ); + overflowCompactionAttempted = true; + const compactResult = await enqueueSession(() => + compactEmbeddedPiSessionDirect({ + sessionId: params.sessionId, + sessionKey: params.sessionKey, + messageChannel: params.messageChannel, + messageProvider: params.messageProvider, + agentAccountId: params.agentAccountId, + authProfileId: lastProfileId, + sessionFile: params.sessionFile, + workspaceDir: params.workspaceDir, + agentDir, + config: params.config, + skillsSnapshot: params.skillsSnapshot, provider, model: modelId, - profileId: lastProfileId, - status: resolveFailoverStatus(promptFailoverReason ?? "unknown"), - }); - } - throw promptError; - } - - const fallbackThinking = pickFallbackThinkingLevel({ - message: lastAssistant?.errorMessage, - attempted: attemptedThinking, - }); - if (fallbackThinking && !aborted) { - log.warn( - `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, + thinkLevel, + reasoningLevel: params.reasoningLevel, + bashElevated: params.bashElevated, + extraSystemPrompt: params.extraSystemPrompt, + ownerNumbers: params.ownerNumbers, + skipSkillEnvOverrides: true, + }), ); - thinkLevel = fallbackThinking; - continue; - } - - const authFailure = isAuthAssistantError(lastAssistant); - const rateLimitFailure = isRateLimitAssistantError(lastAssistant); - const failoverFailure = isFailoverAssistantError(lastAssistant); - const assistantFailoverReason = classifyFailoverReason(lastAssistant?.errorMessage ?? ""); - const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError; - const imageDimensionError = parseImageDimensionError(lastAssistant?.errorMessage ?? ""); - - if (imageDimensionError && lastProfileId) { - const details = [ - imageDimensionError.messageIndex !== undefined - ? `message=${imageDimensionError.messageIndex}` - : null, - imageDimensionError.contentIndex !== undefined - ? `content=${imageDimensionError.contentIndex}` - : null, - imageDimensionError.maxDimensionPx !== undefined - ? `limit=${imageDimensionError.maxDimensionPx}px` - : null, - ] - .filter(Boolean) - .join(" "); + if (compactResult.compacted) { + log.info(`auto-compaction succeeded for ${provider}/${modelId}; retrying prompt`); + continue; + } log.warn( - `Profile ${lastProfileId} rejected image payload${details ? ` (${details})` : ""}.`, + `auto-compaction failed for ${provider}/${modelId}: ${compactResult.reason ?? "nothing to compact"}`, ); } - - // Treat timeout as potential rate limit (Antigravity hangs on rate limit) - const shouldRotate = (!aborted && failoverFailure) || timedOut; - - if (shouldRotate) { - if (lastProfileId) { - const reason = - timedOut || assistantFailoverReason === "timeout" - ? "timeout" - : (assistantFailoverReason ?? "unknown"); - await markAuthProfileFailure({ - store: authStore, - profileId: lastProfileId, - reason, - cfg: params.config, - agentDir: params.agentDir, - }); - if (timedOut && !isProbeSession) { - log.warn( - `Profile ${lastProfileId} timed out (possible rate limit). Trying next account...`, - ); - } - if (cloudCodeAssistFormatError) { - log.warn( - `Profile ${lastProfileId} hit Cloud Code Assist format error. Tool calls will be sanitized on retry.`, - ); - } - } - - const rotated = await advanceAuthProfile(); - if (rotated) continue; - - if (fallbackConfigured) { - // Prefer formatted error message (user-friendly) over raw errorMessage - const message = - (lastAssistant - ? formatAssistantErrorText(lastAssistant, { - cfg: params.config, - sessionKey: params.sessionKey ?? params.sessionId, - }) - : undefined) || - lastAssistant?.errorMessage?.trim() || - (timedOut - ? "LLM request timed out." - : rateLimitFailure - ? "LLM request rate limited." - : authFailure - ? "LLM request unauthorized." - : "LLM request failed."); - const status = - resolveFailoverStatus(assistantFailoverReason ?? "unknown") ?? - (isTimeoutErrorMessage(message) ? 408 : undefined); - throw new FailoverError(message, { - reason: assistantFailoverReason ?? "unknown", - provider, - model: modelId, - profileId: lastProfileId, - status, - }); - } - } - - const usage = normalizeUsage(lastAssistant?.usage as UsageLike); - const agentMeta: EmbeddedPiAgentMeta = { - sessionId: sessionIdUsed, - provider: lastAssistant?.provider ?? provider, - model: lastAssistant?.model ?? model.id, - usage, - }; - - const payloads = buildEmbeddedRunPayloads({ - assistantTexts: attempt.assistantTexts, - toolMetas: attempt.toolMetas, - lastAssistant: attempt.lastAssistant, - lastToolError: attempt.lastToolError, - config: params.config, - sessionKey: params.sessionKey ?? params.sessionId, - verboseLevel: params.verboseLevel, - reasoningLevel: params.reasoningLevel, - toolResultFormat: resolvedToolResultFormat, - inlineToolResultsAllowed: false, - }); - - log.debug( - `embedded run done: runId=${params.runId} sessionId=${params.sessionId} durationMs=${Date.now() - started} aborted=${aborted}`, - ); - if (lastProfileId) { - await markAuthProfileGood({ - store: authStore, - provider, - profileId: lastProfileId, - agentDir: params.agentDir, - }); - await markAuthProfileUsed({ - store: authStore, - profileId: lastProfileId, - agentDir: params.agentDir, - }); - } + const kind = isCompactionFailure ? "compaction_failure" : "context_overflow"; return { - payloads: payloads.length ? payloads : undefined, + payloads: [ + { + text: + "Context overflow: prompt too large for the model. " + + "Try again with less input or a larger-context model.", + isError: true, + }, + ], meta: { durationMs: Date.now() - started, - agentMeta, - aborted, + agentMeta: { + sessionId: sessionIdUsed, + provider, + model: model.id, + }, systemPromptReport: attempt.systemPromptReport, - // Handle client tool calls (OpenResponses hosted tools) - stopReason: attempt.clientToolCall ? "tool_calls" : undefined, - pendingToolCalls: attempt.clientToolCall - ? [ - { - id: `call_${Date.now()}`, - name: attempt.clientToolCall.name, - arguments: JSON.stringify(attempt.clientToolCall.params), - }, - ] - : undefined, + error: { kind, message: errorText }, }, - didSendViaMessagingTool: attempt.didSendViaMessagingTool, - messagingToolSentTexts: attempt.messagingToolSentTexts, - messagingToolSentTargets: attempt.messagingToolSentTargets, }; } - } finally { - process.chdir(prevCwd); + // Handle role ordering errors with a user-friendly message + if (/incorrect role information|roles must alternate/i.test(errorText)) { + return { + payloads: [ + { + text: + "Message ordering conflict - please try again. " + + "If this persists, use /new to start a fresh session.", + isError: true, + }, + ], + meta: { + durationMs: Date.now() - started, + agentMeta: { + sessionId: sessionIdUsed, + provider, + model: model.id, + }, + systemPromptReport: attempt.systemPromptReport, + error: { kind: "role_ordering", message: errorText }, + }, + }; + } + // Handle image size errors with a user-friendly message (no retry needed) + const imageSizeError = parseImageSizeError(errorText); + if (imageSizeError) { + const maxMb = imageSizeError.maxMb; + const maxMbLabel = + typeof maxMb === "number" && Number.isFinite(maxMb) ? `${maxMb}` : null; + const maxBytesHint = maxMbLabel ? ` (max ${maxMbLabel}MB)` : ""; + return { + payloads: [ + { + text: + `Image too large for the model${maxBytesHint}. ` + + "Please compress or resize the image and try again.", + isError: true, + }, + ], + meta: { + durationMs: Date.now() - started, + agentMeta: { + sessionId: sessionIdUsed, + provider, + model: model.id, + }, + systemPromptReport: attempt.systemPromptReport, + error: { kind: "image_size", message: errorText }, + }, + }; + } + const promptFailoverReason = classifyFailoverReason(errorText); + if (promptFailoverReason && promptFailoverReason !== "timeout" && lastProfileId) { + await markAuthProfileFailure({ + store: authStore, + profileId: lastProfileId, + reason: promptFailoverReason, + cfg: params.config, + agentDir: params.agentDir, + }); + } + if ( + isFailoverErrorMessage(errorText) && + promptFailoverReason !== "timeout" && + (await advanceAuthProfile()) + ) { + continue; + } + const fallbackThinking = pickFallbackThinkingLevel({ + message: errorText, + attempted: attemptedThinking, + }); + if (fallbackThinking) { + log.warn( + `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, + ); + thinkLevel = fallbackThinking; + continue; + } + // FIX: Throw FailoverError for prompt errors when fallbacks configured + // This enables model fallback for quota/rate limit errors during prompt submission + if (fallbackConfigured && isFailoverErrorMessage(errorText)) { + throw new FailoverError(errorText, { + reason: promptFailoverReason ?? "unknown", + provider, + model: modelId, + profileId: lastProfileId, + status: resolveFailoverStatus(promptFailoverReason ?? "unknown"), + }); + } + throw promptError; } - }), - ); + + const fallbackThinking = pickFallbackThinkingLevel({ + message: lastAssistant?.errorMessage, + attempted: attemptedThinking, + }); + if (fallbackThinking && !aborted) { + log.warn( + `unsupported thinking level for ${provider}/${modelId}; retrying with ${fallbackThinking}`, + ); + thinkLevel = fallbackThinking; + continue; + } + + const authFailure = isAuthAssistantError(lastAssistant); + const rateLimitFailure = isRateLimitAssistantError(lastAssistant); + const failoverFailure = isFailoverAssistantError(lastAssistant); + const assistantFailoverReason = classifyFailoverReason(lastAssistant?.errorMessage ?? ""); + const cloudCodeAssistFormatError = attempt.cloudCodeAssistFormatError; + const imageDimensionError = parseImageDimensionError(lastAssistant?.errorMessage ?? ""); + + if (imageDimensionError && lastProfileId) { + const details = [ + imageDimensionError.messageIndex !== undefined + ? `message=${imageDimensionError.messageIndex}` + : null, + imageDimensionError.contentIndex !== undefined + ? `content=${imageDimensionError.contentIndex}` + : null, + imageDimensionError.maxDimensionPx !== undefined + ? `limit=${imageDimensionError.maxDimensionPx}px` + : null, + ] + .filter(Boolean) + .join(" "); + log.warn( + `Profile ${lastProfileId} rejected image payload${details ? ` (${details})` : ""}.`, + ); + } + + // Treat timeout as potential rate limit (Antigravity hangs on rate limit) + const shouldRotate = (!aborted && failoverFailure) || timedOut; + + if (shouldRotate) { + if (lastProfileId) { + const reason = + timedOut || assistantFailoverReason === "timeout" + ? "timeout" + : (assistantFailoverReason ?? "unknown"); + await markAuthProfileFailure({ + store: authStore, + profileId: lastProfileId, + reason, + cfg: params.config, + agentDir: params.agentDir, + }); + if (timedOut && !isProbeSession) { + log.warn( + `Profile ${lastProfileId} timed out (possible rate limit). Trying next account...`, + ); + } + if (cloudCodeAssistFormatError) { + log.warn( + `Profile ${lastProfileId} hit Cloud Code Assist format error. Tool calls will be sanitized on retry.`, + ); + } + } + + const rotated = await advanceAuthProfile(); + if (rotated) continue; + + if (fallbackConfigured) { + // Prefer formatted error message (user-friendly) over raw errorMessage + const message = + (lastAssistant + ? formatAssistantErrorText(lastAssistant, { + cfg: params.config, + sessionKey: params.sessionKey ?? params.sessionId, + }) + : undefined) || + lastAssistant?.errorMessage?.trim() || + (timedOut + ? "LLM request timed out." + : rateLimitFailure + ? "LLM request rate limited." + : authFailure + ? "LLM request unauthorized." + : "LLM request failed."); + const status = + resolveFailoverStatus(assistantFailoverReason ?? "unknown") ?? + (isTimeoutErrorMessage(message) ? 408 : undefined); + throw new FailoverError(message, { + reason: assistantFailoverReason ?? "unknown", + provider, + model: modelId, + profileId: lastProfileId, + status, + }); + } + } + + const usage = normalizeUsage(lastAssistant?.usage as UsageLike); + const agentMeta: EmbeddedPiAgentMeta = { + sessionId: sessionIdUsed, + provider: lastAssistant?.provider ?? provider, + model: lastAssistant?.model ?? model.id, + usage, + }; + + const payloads = buildEmbeddedRunPayloads({ + assistantTexts: attempt.assistantTexts, + toolMetas: attempt.toolMetas, + lastAssistant: attempt.lastAssistant, + lastToolError: attempt.lastToolError, + config: params.config, + sessionKey: params.sessionKey ?? params.sessionId, + verboseLevel: params.verboseLevel, + reasoningLevel: params.reasoningLevel, + toolResultFormat: resolvedToolResultFormat, + inlineToolResultsAllowed: false, + }); + + log.debug( + `embedded run done: runId=${params.runId} sessionId=${params.sessionId} durationMs=${Date.now() - started} aborted=${aborted}`, + ); + if (lastProfileId) { + await markAuthProfileGood({ + store: authStore, + provider, + profileId: lastProfileId, + agentDir: params.agentDir, + }); + await markAuthProfileUsed({ + store: authStore, + profileId: lastProfileId, + agentDir: params.agentDir, + }); + } + return { + payloads: payloads.length ? payloads : undefined, + meta: { + durationMs: Date.now() - started, + agentMeta, + aborted, + systemPromptReport: attempt.systemPromptReport, + // Handle client tool calls (OpenResponses hosted tools) + stopReason: attempt.clientToolCall ? "tool_calls" : undefined, + pendingToolCalls: attempt.clientToolCall + ? [ + { + id: `call_${Date.now()}`, + name: attempt.clientToolCall.name, + arguments: JSON.stringify(attempt.clientToolCall.params), + }, + ] + : undefined, + }, + didSendViaMessagingTool: attempt.didSendViaMessagingTool, + messagingToolSentTexts: attempt.messagingToolSentTexts, + messagingToolSentTargets: attempt.messagingToolSentTargets, + }; + } + } finally { + process.chdir(prevCwd); + } } diff --git a/src/agents/pi-embedded-runner/run/attempt.ts b/src/agents/pi-embedded-runner/run/attempt.ts index 46a53bd8f..d49a6081c 100644 --- a/src/agents/pi-embedded-runner/run/attempt.ts +++ b/src/agents/pi-embedded-runner/run/attempt.ts @@ -619,6 +619,7 @@ export async function runEmbeddedAttempt( toolMetas, unsubscribe, waitForCompactionRetry, + getCompactionAttempts, getMessagingToolSentTexts, getMessagingToolSentTargets, didSendViaMessagingTool, @@ -865,6 +866,7 @@ export async function runEmbeddedAttempt( didSendViaMessagingTool: didSendViaMessagingTool(), messagingToolSentTexts: getMessagingToolSentTexts(), messagingToolSentTargets: getMessagingToolSentTargets(), + autoCompactionAttempts: getCompactionAttempts(), cloudCodeAssistFormatError: Boolean( lastAssistant?.errorMessage && isCloudCodeAssistFormatError(lastAssistant.errorMessage), ), diff --git a/src/agents/pi-embedded-runner/run/types.ts b/src/agents/pi-embedded-runner/run/types.ts index 92bb3ff46..d435aa129 100644 --- a/src/agents/pi-embedded-runner/run/types.ts +++ b/src/agents/pi-embedded-runner/run/types.ts @@ -102,6 +102,7 @@ export type EmbeddedRunAttemptResult = { didSendViaMessagingTool: boolean; messagingToolSentTexts: string[]; messagingToolSentTargets: MessagingToolSend[]; + autoCompactionAttempts: number; cloudCodeAssistFormatError: boolean; /** Client tool call detected (OpenResponses hosted tools). */ clientToolCall?: { name: string; params: Record }; diff --git a/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts b/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts index 1c8402465..a302d2144 100644 --- a/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts +++ b/src/agents/pi-embedded-subscribe.handlers.lifecycle.ts @@ -22,6 +22,7 @@ export function handleAgentStart(ctx: EmbeddedPiSubscribeContext) { export function handleAutoCompactionStart(ctx: EmbeddedPiSubscribeContext) { ctx.state.compactionInFlight = true; + ctx.state.compactionAttempts += 1; ctx.ensureCompactionPromise(); ctx.log.debug(`embedded run compaction start: runId=${ctx.params.runId}`); emitAgentEvent({ diff --git a/src/agents/pi-embedded-subscribe.handlers.types.ts b/src/agents/pi-embedded-subscribe.handlers.types.ts index 4a464c5e2..cdae3b84e 100644 --- a/src/agents/pi-embedded-subscribe.handlers.types.ts +++ b/src/agents/pi-embedded-subscribe.handlers.types.ts @@ -49,6 +49,7 @@ export type EmbeddedPiSubscribeState = { lastReasoningSent?: string; compactionInFlight: boolean; + compactionAttempts: number; pendingCompactionRetry: number; compactionRetryResolve?: () => void; compactionRetryPromise: Promise | null; diff --git a/src/agents/pi-embedded-subscribe.ts b/src/agents/pi-embedded-subscribe.ts index a4a4b906a..43fa59a6a 100644 --- a/src/agents/pi-embedded-subscribe.ts +++ b/src/agents/pi-embedded-subscribe.ts @@ -57,6 +57,7 @@ export function subscribeEmbeddedPiSession(params: SubscribeEmbeddedPiSessionPar suppressBlockChunks: false, // Avoid late chunk inserts after final text merge. lastReasoningSent: undefined, compactionInFlight: false, + compactionAttempts: 0, pendingCompactionRetry: 0, compactionRetryResolve: undefined, compactionRetryPromise: null, @@ -472,6 +473,7 @@ export function subscribeEmbeddedPiSession(params: SubscribeEmbeddedPiSessionPar toolMetas, unsubscribe, isCompacting: () => state.compactionInFlight || state.pendingCompactionRetry > 0, + getCompactionAttempts: () => state.compactionAttempts, getMessagingToolSentTexts: () => messagingToolSentTexts.slice(), getMessagingToolSentTargets: () => messagingToolSentTargets.slice(), // Returns true if any messaging tool successfully sent a message.