diff --git a/server/src/services/native-runtime/native-provider-capacity.integration.test.ts b/server/src/services/native-runtime/native-provider-capacity.integration.test.ts index d3a9a3b4167..ee14d01a6a6 100644 --- a/server/src/services/native-runtime/native-provider-capacity.integration.test.ts +++ b/server/src/services/native-runtime/native-provider-capacity.integration.test.ts @@ -38,6 +38,7 @@ import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; import { prepareNativeHeartbeatRun } from "./prepare-native-run.js"; import { PROVIDER_LIMIT_RESUME_KEY_PREFIX, registerProviderLimitResumeWork, requestProviderLimitResumeRetry } from "./provider-limit-resume.js"; import { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; +import { PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; describe("native provider capacity failure persistence", () => { let temporary: Awaited>; @@ -397,6 +398,32 @@ describe("native provider capacity failure persistence", () => { expect(receipts.filter(row => (row.payload as Record).issueId === issueId)).toHaveLength(1); }); + it("backs off and retries a Claude account rate limit that reports no reset", async () => { + // Claude's account-level 429 says only "try again later": no window, no reset. + const payload = { status: "failed", error: { code: "RUNTIME", message: "ACP agent reported a terminal request failure.", + providerFailure: { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "turn", category: "rate_limit", + httpStatus: 429, retryability: "unknown", + message: "API error: 429 This request would exceed your account's rate limit. Please try again later" } } }; + const userId = `user-${randomUUID()}`; + const failedAt = Date.now(); + const { runId, issueId } = await failTurn({ payload, responsibleUserId: userId }); + const hold = await holdFor(issueId); + // The hold waits the default cooldown that also holds the agent's new work. + const resetAt = new Date(String(hold.evidence.providerLimitResetAt)); + expect(resetAt.getTime()).toBeGreaterThanOrEqual(failedAt + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS); + expect(resetAt.getTime()).toBeLessThanOrEqual(Date.now() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS); + expect(hold.nextAction).toBe(`The provider did not report when its rate limit resets. Paperclip will retry this task at ${ + resetAt.toISOString()}; Retry starts it sooner if usage capacity is already restored.`); + await expect(requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id })) + .resolves.toEqual({ requested: false, waitUntil: resetAt }); + // After the wait Paperclip asks for the Retry once; saved work resumes through Retry admission. + const outcome = await requestProviderLimitResumeRetry(db, { companyId, actionId: hold.id }, new Date(resetAt.getTime() + 1000)); + expect(outcome).toMatchObject({ requested: true }); + const [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, companyId), + eq(agentWakeupRequests.idempotencyKey, `task-retry:${PROVIDER_LIMIT_RESUME_KEY_PREFIX}${runId}`))); + expect(receipt).toMatchObject({ reason: "task_retry", requestedByActorId: userId }); + }); + it("starts the Retry from the hold's own deadline after a restart rebuilds it", async () => { // A typed reset a few seconds out; the owner learns it only from durable rows. const resetAt = new Date(Date.now() + 3000); diff --git a/server/src/services/native-runtime/provider-limit-resume.ts b/server/src/services/native-runtime/provider-limit-resume.ts index 82497b620ba..f3ada6e961f 100644 --- a/server/src/services/native-runtime/provider-limit-resume.ts +++ b/server/src/services/native-runtime/provider-limit-resume.ts @@ -15,6 +15,7 @@ import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import { isProvenTurnSettlement, TURN_SETTLED_EVENTS, TURN_STARTED_EVENTS } from "../heartbeat/restart-interrupted-turn.js"; import { parseProviderQuotaClockReset } from "../provider-quota-reset.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; +import { PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; import { requestTaskRetry } from "../task-retry.js"; /** The chat service's Retry callback, which a connected conversation's Retry needs. */ @@ -51,12 +52,16 @@ export type ProviderLimitReset = { resetAt: Date; observedAt: Date; /** False for a reset further out than Paperclip waits by itself (a monthly limit). */ withinAutomaticWait: boolean; + /** True when the provider reported no reset and Paperclip waits its default cooldown. */ + estimated?: true; }; /** * When the provider said the usage limit resets, from the run's own durable * events. Null unless the last turn event is a provider-observed settlement - * reporting an exhausted allowance with a reset after it. + * reporting an exhausted allowance with a reset after it. A rate limit that + * names no reset (Claude's "try again later") waits the default cooldown, the + * same wait that holds the agent's new work, so the task is not stranded. */ export async function readNativeUsageLimitReset( db: Pick, @@ -74,10 +79,14 @@ export async function readNativeUsageLimitReset( if (!failure || !LIMIT_CATEGORIES.includes(failure.category)) return null; const observedAt = lastTurn.createdAt; const typed = failure.retryAt ? new Date(failure.retryAt) : null; - const resetAt = typed && Number.isFinite(typed.getTime()) ? typed : parseProviderQuotaClockReset(failure.message, observedAt); + const reported = typed && Number.isFinite(typed.getTime()) ? typed : parseProviderQuotaClockReset(failure.message, observedAt); + // An exhausted allowance without a reset still needs a person; a rate limit passes by itself. + const estimated = !reported && failure.category === "rate_limit"; + const resetAt = estimated ? new Date(observedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS) : reported; if (!resetAt || resetAt.getTime() <= observedAt.getTime()) return null; return { resetAt, observedAt, - withinAutomaticWait: resetAt.getTime() - observedAt.getTime() <= PROVIDER_LIMIT_RESUME_MAX_WAIT_MS }; + withinAutomaticWait: resetAt.getTime() - observedAt.getTime() <= PROVIDER_LIMIT_RESUME_MAX_WAIT_MS, + ...(estimated ? { estimated: true as const } : {}) }; } /** The user whose authority Retry uses: the requester, else the run's accountable user. */ @@ -90,8 +99,8 @@ async function accountableUser(db: Pick, run: typeof heartbeatRuns } export type ProviderLimitResumePlan = - | { automatic: true; resetAt: Date; actorId: string } - | { automatic: false; resetAt: Date | null }; + | { automatic: true; resetAt: Date; actorId: string; estimated?: true } + | { automatic: false; resetAt: Date | null; estimated?: true }; /** Whether this failed run's task continues by itself once the limit resets. */ export async function planProviderLimitResume(db: Pick, runId: string): Promise { @@ -101,10 +110,18 @@ export async function planProviderLimitResume(db: Pick, runId: str if (!reset) return { automatic: false, resetAt: null }; // A reset beyond the bounded wait is still reported; only the automatic Retry is bounded. const actorId = reset.withinAutomaticWait ? await accountableUser(db, run) : null; - return actorId ? { automatic: true, resetAt: reset.resetAt, actorId } : { automatic: false, resetAt: reset.resetAt }; + const estimated = reset.estimated ? { estimated: true as const } : {}; + return actorId ? { automatic: true, resetAt: reset.resetAt, actorId, ...estimated } + : { automatic: false, resetAt: reset.resetAt, ...estimated }; } export function providerLimitNextAction(plan: ProviderLimitResumePlan) { + if (plan.estimated && plan.resetAt) { + const unreported = "The provider did not report when its rate limit resets."; + return plan.automatic + ? `${unreported} Paperclip will retry this task at ${plan.resetAt.toISOString()}; Retry starts it sooner if usage capacity is already restored.` + : `${unreported} Retry the task after ${plan.resetAt.toISOString()}, or sooner if usage capacity is restored.`; + } if (plan.automatic) { return `The provider usage limit resets at ${plan.resetAt.toISOString()}. Paperclip will retry this task then; Retry starts it sooner if usage capacity is already restored.`; } @@ -187,7 +204,8 @@ export async function requestProviderLimitResumeRetry(db: Db, input: { const actorId = await accountableUser(db, run); await db.update(issueRecoveryActions).set({ nextAction: providerLimitNextAction(actorId - ? { automatic: true, resetAt: reset.resetAt, actorId } : { automatic: false, resetAt: reset.resetAt }), + ? { automatic: true, resetAt: reset.resetAt, actorId, estimated: reset.estimated } + : { automatic: false, resetAt: reset.resetAt, estimated: reset.estimated }), evidence: sql`coalesce(${issueRecoveryActions.evidence}, '{}'::jsonb) || jsonb_build_object('providerLimitResetAt', ${reset.resetAt.toISOString()}::text)`, updatedAt: now, }).where(and(eq(issueRecoveryActions.id, action.id), eq(issueRecoveryActions.status, "active")));