diff --git a/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts b/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts index ddc866a02d9..cb7f9a94c6d 100644 --- a/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts +++ b/server/src/services/__tests__/run-failure-report-native-usage-limit.test.ts @@ -117,6 +117,33 @@ describeEmbeddedPostgres("reportRunFailure native provider usage limits", () => expect(run).toEqual(before); }); + it("keeps a native rate limit that refused the session start on the task", async () => { + // The persisted shape of staging PAPERCLIP-BACKEND-8Z once it ends as a usage-limit hold. + await reportRunFailure(db, nativeUsageLimitRun({ + error: "PRP command session.open failed: failed to start ACPX provider: ACPX sidecar command session.open was rejected (retryable=false, classification=provider_rate_limit)", + resultJson: { providerFailure: { ...limitFailure, phase: "startup", category: "rate_limit", retryability: "unknown", + httpStatus: 429, message: "API error: 429 This request would exceed your account's rate limit. Please try again later" } }, + })); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it("keeps a startup rate limit on the task when an earlier attempt of the run saved another failure", async () => { + // A same-run startup retry keeps the first attempt's providerFailure (here an + // overload); the limit that ended the run is saved as providerLimitStartup. + const startupLimit = { ...limitFailure, phase: "startup", category: "rate_limit", retryability: "unknown", httpStatus: 429, + message: "API error: 429 This request would exceed your account's rate limit. Please try again later" }; + await reportRunFailure(db, nativeUsageLimitRun({ resultJson: { + providerFailure: { ...limitFailure, phase: "startup", category: "overloaded", retryability: "transient", httpStatus: 529 }, + providerLimitStartup: { failure: startupLimit, observedAt: new Date().toISOString() }, + } })); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + // Without that saved limit the overload alone is still reported. + await reportRunFailure(db, nativeUsageLimitRun({ resultJson: { + providerFailure: { ...limitFailure, phase: "startup", category: "overloaded", retryability: "transient", httpStatus: 529 }, + } })); + expect(mockCaptureRunFailure).toHaveBeenCalledTimes(1); + }); + it.each([ { status: "timed_out" }, { signal: "SIGTERM" }, { runtimeMode: "legacy" }, { errorCode: "native_provider_terminal_failed" }, { errorCode: "native_session_retry_exhausted" }, diff --git a/server/src/services/native-runtime/native-provider-capacity.integration.test.ts b/server/src/services/native-runtime/native-provider-capacity.integration.test.ts index ee14d01a6a6..b445650310b 100644 --- a/server/src/services/native-runtime/native-provider-capacity.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-capacity.integration.test.ts @@ -36,7 +36,13 @@ import { executePaperclipNativeSession } from "./native-session-executor.js"; import { buildNativeCompletionContract } from "./completion-contracts.js"; import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; -import { PROVIDER_LIMIT_RESUME_KEY_PREFIX, registerProviderLimitResumeWork, requestProviderLimitResumeRetry } from "./provider-limit-resume.js"; +import { + planProviderLimitResume, + PROVIDER_LIMIT_RESUME_KEY_PREFIX, + readNativeUsageLimitReset, + registerProviderLimitResumeWork, + requestProviderLimitResumeRetry, +} from "./provider-limit-resume.js"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; @@ -424,6 +430,52 @@ describe("native provider capacity failure persistence", () => { expect(receipt).toMatchObject({ reason: "task_retry", requestedByActorId: userId }); }); + it("retries a task whose session start the provider refused with a rate limit, before any turn", async () => { + // Staging PAPERCLIP-BACKEND-8Z: Claude's account 429 refused session.open, so no turn event exists. + const startupLimit = { schema: "paperclip.provider-failure.v1" as const, provider: "anthropic", phase: "startup" as const, + category: "rate_limit" as const, httpStatus: 429, retryability: "unknown" as const, + message: "API error: 429 This request would exceed your account's rate limit. Please try again later · model not changed" }; + const userId = `user-${randomUUID()}`; + const issueId = randomUUID(); + const runId = randomUUID(); + const finishedAt = new Date(); + await db.insert(issues).values({ id: issueId, companyId, title: "Startup rate limit", status: "blocked", + workMode: "standard", assigneeAgentId: agentId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "failed", runtimeMode: "native", + nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId }, + responsibleUserId: userId, errorCode: "native_provider_usage_limit", finishedAt, + resultJson: { providerFailure: startupLimit } }); + const [hold] = await db.insert(issueRecoveryActions).values({ companyId, sourceIssueId: issueId, + kind: "active_run_watchdog", ownerType: "board", cause: "native_provider_usage_limit", + fingerprint: `startup-limit:${runId}`, evidence: { runId }, nextAction: "Wait for the provider limit." }).returning(); + + const resetAt = new Date(finishedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS); + // The executor plans from the failure it holds, before the run row records it. + await expect(planProviderLimitResume(db, runId, { failure: startupLimit, observedAt: finishedAt })) + .resolves.toEqual({ automatic: true, resetAt, actorId: userId, estimated: true }); + // The hold owner later reads the same reset from the run's saved failure. + await expect(requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id })) + .resolves.toEqual({ requested: false, waitUntil: resetAt }); + const outcome = await requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id }, new Date(resetAt.getTime() + 1000)); + expect(outcome).toMatchObject({ requested: true }); + const [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.idempotencyKey, `task-retry:${PROVIDER_LIMIT_RESUME_KEY_PREFIX}${runId}`))); + expect(receipt).toMatchObject({ reason: "task_retry", requestedByActorId: userId }); + + // Only a refusal at session start counts without a turn: a saved turn-phase failure does not. + const turnRun = randomUUID(); + await db.insert(heartbeatRuns).values({ id: turnRun, companyId, agentId, status: "failed", runtimeMode: "native", + nativeIssueId: issueId, invocationSource: "assignment", triggerDetail: "system", contextSnapshot: { issueId }, + errorCode: "native_provider_usage_limit", finishedAt, resultJson: { providerFailure: { ...startupLimit, phase: "turn" } } }); + await expect(readNativeUsageLimitReset(db, { id: turnRun, companyId, resultJson: { providerFailure: { ...startupLimit, phase: "turn" } }, + finishedAt })).resolves.toBeNull(); + // A turn that started but never settled is still unknown, whatever the run's saved failure says. + await db.insert(heartbeatRunEvents).values({ companyId, runId: turnRun, agentId, seq: 1, eventType: "turn.started", + stream: "system", level: "info", message: "turn started" }); + await expect(readNativeUsageLimitReset(db, { id: turnRun, companyId, resultJson: { providerFailure: startupLimit }, finishedAt }, + { failure: startupLimit, observedAt: finishedAt })).resolves.toBeNull(); + }); + it("starts the Retry from the hold's own deadline after a restart rebuilds it", async () => { // A typed reset a few seconds out; the owner learns it only from durable rows. const resetAt = new Date(Date.now() + 3000); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 9df1d167895..ba38b91623b 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -16057,6 +16057,35 @@ describe("native session bounded recovery", () => { const deterministic = confirmed({ category: "authentication", retryability: "requires_action" }); expect(nativeSessionFailureSourceCode(deterministic)).toBe(NATIVE_PROVIDER_STARTUP_REJECTED); }); + + it("waits out a remote startup rate limit or quota as a usage-limit hold, not a board Retry", () => { + // Staging PAPERCLIP-BACKEND-8Z: an account-wide Claude 429 refused session.open, the same-run + // retry met it again after the cooldown, and the repeat ended as a startup rejection. + const limit = (fields: Record) => startupFailure({ provider: "anthropic", httpStatus: 429, + message: "API error: 429 This request would exceed your account's rate limit. Please try again later · model not changed", + ...fields }); + const confirmed = (fields: Record) => + Object.defineProperty(sessionOpenFailure(limit(fields)), "runnerExitConfirmed", { value: true, enumerable: false }); + for (const fields of [{ category: "rate_limit" }, { category: "quota_exhausted", retryability: "requires_action" }]) { + const first = confirmed(fields); + expect(nativeSessionFailureSourceCode(first)).toBe("provider_session_open_rejected"); + // The first refusal still retries the same run after the cooldown. + expect(nativeRemoteStartupRejectionCode("provider_session_open_rejected", first, true)) + .toBe("provider_session_open_rejected"); + // The repeat, or a refusal whose runner exit is unproven, waits for the reset. + const previous = nativeStartupFailureSignature(limit(fields)); + const repeated = nativeRemoteStartupRejectionCode("provider_session_open_rejected", confirmed(fields), true, previous); + const unproven = nativeRemoteStartupRejectionCode("provider_session_open_rejected", sessionOpenFailure(limit(fields)), true); + for (const code of [repeated, unproven]) { + expect(code).toBe("native_provider_usage_limit"); + const disposition = nativeSessionFailureDisposition(2, new Date(), code); + expect(disposition).toEqual({ phase: "terminal_failure", failureCode: "native_provider_usage_limit", nextAttemptAt: null }); + } + } + // A local target keeps its bounded same-run retries. + expect(nativeRemoteStartupRejectionCode("provider_session_open_rejected", + sessionOpenFailure(limit({ category: "rate_limit" })), false)).toBe("provider_session_open_rejected"); + }); }); it("stops same-run retries for an operator while a saved side-effecting call has no result", () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index ecba5b780de..7c9a0275ace 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -9207,18 +9207,23 @@ export function nativeStartupFailureSignature(failure: unknown): string | null { * can own the sandbox) and the previous attempt did not fail with the same * startup error. Otherwise it ends at once with Retry. A deterministic * rejection never reaches here: it is already NATIVE_PROVIDER_STARTUP_REJECTED. + * + * An exhausted provider allowance (a rate limit or quota) ends as a usage-limit + * hold instead: it passes by itself, so the hold retries the task at the reset + * like a limit reported on a turn, rather than waiting for a person's Retry. */ export function nativeRemoteStartupRejectionCode( sourceFailureCode: Code, error: unknown, remoteTarget: boolean, previousStartupSignature?: string | null, -): Code | typeof NATIVE_PROVIDER_STARTUP_REJECTED { +): Code | typeof NATIVE_PROVIDER_STARTUP_REJECTED | "native_provider_usage_limit" { if (!remoteTarget || sourceFailureCode !== "provider_session_open_rejected") return sourceFailureCode; const failure = providerFailureFromError(error); if (failure?.phase !== "startup") return sourceFailureCode; - return nativeRunnerExitConfirmed(error) && !nativeRepeatedStartupFailure(error, previousStartupSignature) - ? sourceFailureCode : NATIVE_PROVIDER_STARTUP_REJECTED; + if (nativeRunnerExitConfirmed(error) && !nativeRepeatedStartupFailure(error, previousStartupSignature)) return sourceFailureCode; + return failure.category === "rate_limit" || failure.category === "quota_exhausted" + ? "native_provider_usage_limit" : NATIVE_PROVIDER_STARTUP_REJECTED; } export function nativeRepeatedStartupFailure(error: unknown, previousStartupSignature: string | null | undefined): boolean { @@ -13468,9 +13473,13 @@ async function executePaperclipNativeSessionAttempt( // An exhausted provider allowance is not ambiguous session state: the // provider ended its turn. Say when it resets and whether Paperclip will // retry then. A failed read only loses that detail. + // A limit at session start has no turn event; its reset comes from this + // failure. The hold keeps it: the run row keeps an earlier attempt's failure. + const startupProviderLimit = sourceFailureCode === "native_provider_usage_limit" && providerFailure?.phase === "startup" + ? { failure: providerFailure, observedAt: now } : undefined; const providerLimit = sourceFailureCode === "native_provider_usage_limit" && phase === "terminal_failure" ? await import("./provider-limit-resume.js").then(async ({ planProviderLimitResume, providerLimitNextAction }) => { - const plan = await planProviderLimitResume(input.db, input.execution.binding.runId); + const plan = await planProviderLimitResume(input.db, input.execution.binding.runId, startupProviderLimit); return { plan, nextAction: providerLimitNextAction(plan) }; }).catch((failure: unknown) => { logger.warn({ err: failure, runId: input.execution.binding.runId }, "provider limit reset unavailable"); @@ -13762,7 +13771,14 @@ async function executePaperclipNativeSessionAttempt( error: message, resultJson: sql`(coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) - 'remoteRunnerRecovery') || ${JSON.stringify(remoteRunnerRecovery ? { remoteRunnerRecovery } : {})}::jsonb - || ${providerFailure ? sql`jsonb_build_object('providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', ${JSON.stringify(providerFailure)}::jsonb))` : sql`'{}'::jsonb`}`, + || ${providerFailure ? sql`jsonb_build_object('providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', ${JSON.stringify(providerFailure)}::jsonb))` : sql`'{}'::jsonb`} + || ${startupProviderLimit && phase === "terminal_failure" + // providerFailure keeps an earlier attempt's failure; this names the limit that ended the run. + ? sql`jsonb_build_object('providerLimitStartup', ${JSON.stringify({ + failure: outputRedactor.redact(startupProviderLimit.failure), + observedAt: startupProviderLimit.observedAt.toISOString(), + })}::jsonb)` + : sql`'{}'::jsonb`}`, errorCode: ownershipUnverified ? NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE : sourceFailureCode, @@ -13839,6 +13855,10 @@ async function executePaperclipNativeSessionAttempt( recoveryEvidence.providerSessionEstablished, ...(providerCooldownAt ? { providerCooldownAt: providerCooldownAt.toISOString() } : {}), ...(providerLimit?.plan.resetAt ? { providerLimitResetAt: providerLimit.plan.resetAt.toISOString() } : {}), + ...(providerLimit && startupProviderLimit ? { providerLimitStartup: { + failure: outputRedactor.redact(startupProviderLimit.failure), + observedAt: startupProviderLimit.observedAt.toISOString(), + } } : {}), }, nextAction: providerLimit diff --git a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts index 44605b8f4ba..c0e7b648f58 100644 --- a/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-rejected-retry.integration.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, companies, companyMemberships, completionContracts, createDb, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { agents, companies, companyMemberships, completionContracts, createDb, heartbeatRuns, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import type { NativeExecutionInputV1 } from "@paperclipai/paperclip-runner"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -14,6 +14,7 @@ vi.mock("../../vendor/paperclip-runner/index.js", async (importOriginal) => ({ import { executePaperclipNativeSession, NATIVE_PROVIDER_STARTUP_REJECTED } from "./native-session-executor.js"; import { buildNativeCompletionContract } from "./completion-contracts.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; +import { requestProviderLimitResumeRetry } from "./provider-limit-resume.js"; import { heartbeatService } from "../heartbeat.js"; import { requestTaskRetry } from "../task-retry.js"; import { taskInterruptionPresentation } from "../task-interruption-presentation.js"; @@ -47,7 +48,9 @@ describe("native session.open the provider rejected before any provider work", ( ...fields }); async function rejectSessionOpen(failure: Error, attempt = 0, - options: { remote?: boolean; recoveryHistory?: Record[]; resumesProviderSession?: boolean } = {}) { + options: { remote?: boolean; recoveryHistory?: Record[]; resumesProviderSession?: boolean; + /** What an earlier attempt of this same run saved; the retry claim keeps it. */ + priorResultJson?: Record } = {}) { const issueId = randomUUID(); await db.insert(issues).values({ id: issueId, companyId, title: "Plan the change", status: "in_progress", workMode: "standard", assigneeAgentId: agentId, createdByUserId: "board-user", responsibleUserId: "board-user" }); @@ -55,7 +58,7 @@ describe("native session.open the provider rejected before any provider work", ( const runId = randomUUID(); const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", runtimeMode: "native", nativeIssueId: issueId, responsibleUserId: "board-user", invocationSource: "assignment", triggerDetail: "system", - contextSnapshot: { issueId } }).returning(); + contextSnapshot: { issueId }, ...(options.priorResultJson ? { resultJson: options.priorResultJson } : {}) }).returning(); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); const native = await prepareNativeHeartbeatRun({ db, run, issue, environmentLeaseId: randomUUID() }); const [contract] = await db.select().from(completionContracts).where(eq(completionContracts.issueId, issueId)); @@ -154,6 +157,34 @@ describe("native session.open the provider rejected before any provider work", ( await expect(taskInterruptionPresentation(db, issue)).resolves.toMatchObject({ state: "failed", canRetry: true }); }, 120_000); + it("holds a remote startup rate limit until the reset its own attempt reported, then retries the task", async () => { + // Staging PAPERCLIP-BACKEND-8Z: attempt 1 met a 429 and saved its failure; the + // same-run retry after that reset met the limit again with a later reset. + const now = Date.now(); + const limit = (retryAt: Date) => startupFailure({ provider: "anthropic", category: "rate_limit", httpStatus: 429, + retryAt: retryAt.toISOString(), message: "API error: 429 This request would exceed your account's rate limit. Please try again later" }); + const firstReset = new Date(now - 10 * 60_000); + const secondReset = new Date(now + 30 * 60_000); + const { issueId, runId, coordinator } = await rejectSessionOpen(rejection(limit(secondReset)), 1, { remote: true, + priorResultJson: { providerFailure: limit(firstReset) } }); + expect(coordinator).toMatchObject({ phase: "terminal_failure", failureCode: "native_provider_usage_limit", nextAttemptAt: null }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); + expect(run.errorCode).toBe("native_provider_usage_limit"); + // The run row still names the first attempt's reset; the hold names this attempt's. + expect((run.resultJson as Record>).providerFailure.retryAt).toBe(firstReset.toISOString()); + // The run row also names the limit that ended it, which keeps the expected wait out of Sentry. + expect((run.resultJson as Record).providerLimitStartup).toMatchObject({ + failure: expect.objectContaining({ phase: "startup", category: "rate_limit", retryAt: secondReset.toISOString() }) }); + const [hold] = await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.sourceIssueId, issueId)); + expect(hold).toMatchObject({ cause: "native_provider_usage_limit", + evidence: expect.objectContaining({ providerLimitResetAt: secondReset.toISOString() }) }); + expect(hold.nextAction).toContain(`resets at ${secondReset.toISOString()}`); + await expect(requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id })) + .resolves.toEqual({ requested: false, waitUntil: secondReset }); + await expect(requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id }, new Date(secondReset.getTime() + 1000))) + .resolves.toMatchObject({ requested: true }); + }, 120_000); + it("stops a repeated remote startup rejection on a run that resumes its predecessor's provider session", async () => { const first = await rejectSessionOpen(exitProven(rejection(startupFailure())), 0, { remote: true, resumesProviderSession: true }); expect(first.coordinator).toMatchObject({ phase: "retryable_failure", diff --git a/server/src/services/native-runtime/provider-limit-resume.ts b/server/src/services/native-runtime/provider-limit-resume.ts index f3ada6e961f..97b3959946d 100644 --- a/server/src/services/native-runtime/provider-limit-resume.ts +++ b/server/src/services/native-runtime/provider-limit-resume.ts @@ -7,7 +7,7 @@ import { issues, type Db, } from "@paperclipai/db"; -import { readProviderFailure } from "@paperclipai/shared"; +import { readProviderFailure, type ProviderFailure } from "@paperclipai/shared"; import { HttpError } from "../../errors.js"; import { logger } from "../../middleware/logger.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; @@ -56,16 +56,34 @@ export type ProviderLimitReset = { estimated?: true; }; +/** A provider limit that refused the session before any turn started. */ +export type StartupProviderLimit = { failure: ProviderFailure; observedAt: Date }; + +/** + * The startup refusal a hold saved when it was created. A same-run retry keeps + * the run's first saved failure, so only the hold names the refusal that ended it. + */ +function holdStartupLimit(evidence: Record | null | undefined): StartupProviderLimit | undefined { + const saved = asRecord(asRecord(evidence).providerLimitStartup); + const failure = readProviderFailure(saved.failure); + const observedAt = typeof saved.observedAt === "string" ? new Date(saved.observedAt) : null; + return failure?.phase === "startup" && observedAt && Number.isFinite(observedAt.getTime()) ? { failure, observedAt } : undefined; +} + /** * When the provider said the usage limit resets, from the run's own durable - * events. Null unless the last turn event is a provider-observed settlement - * reporting an exhausted allowance with a reset after it. A rate limit that - * names no reset (Claude's "try again later") waits the default cooldown, the - * same wait that holds the agent's new work, so the task is not stranded. + * evidence. Null unless the last turn event is a provider-observed settlement + * reporting an exhausted allowance with a reset after it, or no turn started + * and the provider refused the session itself with an exhausted allowance. A + * rate limit that names no reset (Claude's "try again later") waits the + * default cooldown, the same wait that holds the agent's new work, so the task + * is not stranded. */ export async function readNativeUsageLimitReset( db: Pick, - run: { id: string; companyId: string }, + run: { id: string; companyId: string; resultJson?: unknown; finishedAt?: Date | null; updatedAt?: Date }, + /** The current failure, before the run's own result records it. */ + startup?: StartupProviderLimit, ): Promise { const [lastTurn] = await db.select({ eventType: heartbeatRunEvents.eventType, payload: heartbeatRunEvents.payload, createdAt: heartbeatRunEvents.createdAt }) @@ -73,11 +91,21 @@ export async function readNativeUsageLimitReset( eq(heartbeatRunEvents.companyId, run.companyId), eq(heartbeatRunEvents.runId, run.id), inArray(heartbeatRunEvents.eventType, [...TURN_STARTED_EVENTS, ...TURN_SETTLED_EVENTS]), )).orderBy(desc(heartbeatRunEvents.seq)).limit(1); - // An unsettled or unobserved turn may still be working: its outcome is unknown. - if (!lastTurn || lastTurn.eventType !== "turn.failed" || !isProvenTurnSettlement(lastTurn)) return null; - const failure = providerFailureOf(lastTurn.payload); + let failure: ProviderFailure | null; + let observedAt: Date; + if (lastTurn) { + // An unsettled or unobserved turn may still be working: its outcome is unknown. + if (lastTurn.eventType !== "turn.failed" || !isProvenTurnSettlement(lastTurn)) return null; + failure = providerFailureOf(lastTurn.payload); + observedAt = lastTurn.createdAt; + } else { + // No turn started, so no provider work began; only the session start's own refusal counts. + failure = startup?.failure ?? readProviderFailure(asRecord(run.resultJson).providerFailure); + const settledAt = startup?.observedAt ?? run.finishedAt ?? run.updatedAt; + if (failure?.phase !== "startup" || !settledAt) return null; + observedAt = settledAt; + } if (!failure || !LIMIT_CATEGORIES.includes(failure.category)) return null; - const observedAt = lastTurn.createdAt; const typed = failure.retryAt ? new Date(failure.retryAt) : null; const reported = typed && Number.isFinite(typed.getTime()) ? typed : parseProviderQuotaClockReset(failure.message, observedAt); // An exhausted allowance without a reset still needs a person; a rate limit passes by itself. @@ -103,10 +131,11 @@ export type ProviderLimitResumePlan = | { automatic: false; resetAt: Date | null; estimated?: true }; /** Whether this failed run's task continues by itself once the limit resets. */ -export async function planProviderLimitResume(db: Pick, runId: string): Promise { +export async function planProviderLimitResume(db: Pick, runId: string, + startup?: StartupProviderLimit): Promise { const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); if (!run) return { automatic: false, resetAt: null }; - const reset = await readNativeUsageLimitReset(db, run); + const reset = await readNativeUsageLimitReset(db, run, startup); if (!reset) return { automatic: false, resetAt: null }; // A reset beyond the bounded wait is still reported; only the automatic Retry is bounded. const actorId = reset.withinAutomaticWait ? await accountableUser(db, run) : null; @@ -184,7 +213,7 @@ export async function requestProviderLimitResumeRetry(db: Db, input: { if (run && ["queued", "running"].includes(run.status)) return { requested: false, waitUntil: new Date(now.getTime() + 60_000) }; if (!run || run.runtimeMode !== "native" || run.status !== "failed" || run.errorCode !== NATIVE_PROVIDER_USAGE_LIMIT) return done("source_run_mismatch"); - const reset = await readNativeUsageLimitReset(db, run); + const reset = await readNativeUsageLimitReset(db, run, holdStartupLimit(action.evidence)); if (!reset) return done("reset_unknown"); if (!reset.withinAutomaticWait) { // Too far out to wait by itself: the hold names the reset and leaves the diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index a514a340bd8..c4d27b3d675 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -277,8 +277,13 @@ function isProviderQuotaFailure(run: HeartbeatRun, options: RunFailureReportOpti function isNativeProviderUsageLimitFailure(run: HeartbeatRun, options: RunFailureReportOptions): boolean { if (run.status !== "failed" || run.runtimeMode !== "native" || run.errorCode !== "native_provider_usage_limit" || options.phase === "setup" || run.signal != null) return false; - const category = readProviderFailure(asRecord(run.resultJson)?.providerFailure)?.category; - return category === "quota_exhausted" || category === "rate_limit"; + const result = asRecord(run.resultJson); + // A same-run startup retry keeps the first attempt's providerFailure; the + // limit that ended the run is saved beside it. + return [result?.providerFailure, asRecord(asRecord(result?.providerLimitStartup)?.failure)].some(value => { + const category = readProviderFailure(value)?.category; + return category === "quota_exhausted" || category === "rate_limit"; + }); } /**