diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 35ced98709a..37092285f2b 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1978,6 +1978,21 @@ not grant retry authority or change the budget. When the budget is exhausted, the task is blocked and its recovery action states that provider startup failed before any provider work began. +The same limit evidence also holds new work for the agent. Before any workspace +or provider work, a queued run for an agent with an active provider limit is +cancelled as `provider_limit_wait`, records the expected reset and the run that +reported the limit, and schedules one retry of the same wake at that reset. +The retry is a resource wait: it does not consume the failure retry budget, and +a wake that is still limited at its retry waits again until the evidence +expires within a day. Manual runs, recovering runs, conversation turns and +pool-routed connections are not held, so an operator can test a repaired +account at once. ACP harnesses report an exhausted allowance as a generic +terminal limit failure; Runner adds the provider's typed session-failure title, +and only a rate-limit or quota title becomes `rate_limit` or `quota_exhausted`. +Context, turn and configured budget limits stay ordinary terminal failures. A +reset clock in the provider's wording, such as "resets 2am (UTC)", is used as +the reset time. + An ingress endpoint failure can use that same bounded retry only when the environment provider returns a structured HTTP 429 or 5xx failure and the complete, encrypted controller checkpoint proves that this exact run never diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts index 8510323ef77..f721ccb5a01 100644 --- a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -2,7 +2,7 @@ import { resolveGrokDisableWebSearch } from "../drivers/acpx/grok-web-search.js"; import { redactObservabilityDiagnostic } from "../drivers/agent-observability.js"; import { resolveClaudeEffort } from "../drivers/acpx/claude-effort.js"; -import { acpxProviderFailure, acpxTerminalResult } from "../drivers/acpx/provider-failure.js"; +import { acpxProviderFailure, acpxTerminalResult, type AcpxTerminalSessionFailure } from "../drivers/acpx/provider-failure.js"; import { providerFailureFromError } from "../drivers/provider-failure.js"; import { resolveGrokEffort } from "../drivers/acpx/grok-effort.js"; import { parseProviderMode } from "../contracts/provider-mode.js"; @@ -426,6 +426,8 @@ async function dispatch( }); activeCopilotEvidence = toolEvidence && "captureSemanticReceipt" in toolEvidence ? toolEvidence as CopilotToolEvidence : undefined; let usageBefore: unknown; + // ACPX reports a generic terminal error; the provider's typed reason arrives here first. + const sessionFailure: { value?: AcpxTerminalSessionFailure } = {}; const timed = request.params.performance === true; const timingStart = receivedAt; let statusMs = 0; @@ -442,12 +444,13 @@ async function dispatch( waitForInput(currentTurnId, providerRequest, context), onPermissionRequest: (providerRequest, context) => waitForPermission(currentTurnId, activeAgent, providerRequest, context, toolEvidence), + onTerminalSessionFailure: (failure) => { sessionFailure.value = failure; }, }); } catch (error) { turnId = null; throw error; } - void pumpTurn(currentTurnId, runtimeTurn, activeHost, usageBefore, extensions.drain, activity, toolEvidence); + void pumpTurn(currentTurnId, runtimeTurn, activeHost, usageBefore, extensions.drain, activity, toolEvidence, sessionFailure); // Warm sessions may defer initialize until their first prompt. Publish only // the capabilities of that live initialized connection, never old disk state. const promptStart = timed ? performance.now() : 0; @@ -664,6 +667,7 @@ async function pumpTurn( drainExtensions: () => Promise, activity: AcpxActivityAdapter, toolEvidence?: AcpxToolEvidence, + sessionFailure: { value?: AcpxTerminalSessionFailure } = {}, ): Promise { let terminal: Record; try { @@ -716,7 +720,7 @@ async function pumpTurn( // not replace the provider's authoritative completed/cancelled result. diagnostic("acpx_terminal_usage_unavailable", safeMessage(error)); } - terminal = openParams ? acpxTerminalResult(result, openParams.agent, process.env) : boundedSidecarValue(result); + terminal = openParams ? acpxTerminalResult(result, openParams.agent, process.env, sessionFailure.value) : boundedSidecarValue(result); } catch (error) { terminal = { status: "failed", @@ -724,7 +728,7 @@ async function pumpTurn( ...(error instanceof AcpxApprovalRequiredError || error instanceof AcpxProviderProcessLostError ? { code: error.code } : {}), message: safeMessage(error), retryable: false, - ...(openParams ? { providerFailure: acpxProviderFailure(error, openParams.agent, "turn", process.env) ?? undefined } : {}), + ...(openParams ? { providerFailure: acpxProviderFailure(error, openParams.agent, "turn", process.env, false, sessionFailure.value) ?? undefined } : {}), }, }; } finally { diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 6c8f9295f73..8300c1a9fb1 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -567,7 +567,10 @@ describe("Codex ACPX runtime adapter", () => { expect(JSON.stringify(vi.mocked(runtime.ensureSession).mock.calls)).not.toContain("clientCapabilities"); const notification = vi.fn(); const request = vi.fn(async (_method, _params, context) => ({ requestId: context.requestId })); - const admittedFirst = port.startTurn({ text: "First", requestId: "turn-1", onExtensionRequest: request, onExtensionNotification: notification }); + const onTerminalSessionFailure = vi.fn(); + const admittedFirst = port.startTurn({ text: "First", requestId: "turn-1", onExtensionRequest: request, onExtensionNotification: notification, onTerminalSessionFailure }); + // ACPX reports only a generic terminal error; the typed reason must reach the host. + expect(vi.mocked(runtime.startTurn).mock.calls.at(-1)?.[0]).toMatchObject({ onTerminalSessionFailure }); const signal = new AbortController().signal; await expect(created.onExtensionRequest!("cursor/ask_question", { sessionId: "backend-1" }, { requestId: 0, signal })).resolves.toEqual({ requestId: 0 }); const sessionless = { toolCallId: "native-cursor-plan", plan: "Review the complete plan" }; diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index 48d3eea115c..cfed1a03412 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -1538,6 +1538,9 @@ function runtimePort( ...(input.onElicitation ? { onElicitation: input.onElicitation } : {}), + ...(input.onTerminalSessionFailure + ? { onTerminalSessionFailure: input.onTerminalSessionFailure } + : {}), }); } catch (error) { releaseExtensionTurn(); diff --git a/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts b/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts index 2962c9c826c..51cd3ae111e 100644 --- a/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts +++ b/packages/paperclip-runner/src/drivers/acpx/provider-failure.ts @@ -30,33 +30,58 @@ const CATEGORIES = new Map([ // ACP reserves this JSON-RPC code for "Authentication required". const ACP_AUTH_REQUIRED = -32000; +/** The typed AIR session failure ACPX reports beside its generic terminal error. */ +export type AcpxTerminalSessionFailure = { category: string; title?: string; details?: string }; + +/** + * AIR's `limit` category also covers context, turn and configured budget + * limits. Only an exhausted account allowance is a provider limit; the + * provider's own title distinguishes it. + */ +function sessionLimitCategory(failure: AcpxTerminalSessionFailure | undefined): ProviderFailure["category"] | undefined { + if (failure?.category !== "limit") return undefined; + const text = [failure.title, failure.details].filter(value => typeof value === "string").join("\n"); + if (/\brate[- ]limit/i.test(text)) return "rate_limit"; + if (/no available quota|usage limit|you(?:'|’)ve hit your (?:\w+ )?limit|out of (?:extra )?usage|quota (?:limit )?exceeded/i.test(text)) { + return "quota_exhausted"; + } + return undefined; +} + /** * `ownMessage` keeps an error's own message when it carries no ACP error. Use * it only where every rejection belongs to the provider: a failed terminal - * result or a provider session ensure. + * result or a provider session ensure. `sessionFailure` adds the provider's + * title from a typed terminal session failure; its details stay private. */ export function acpxProviderFailure(error: unknown, agent: string, phase: ProviderFailure["phase"], - environment: NodeJS.ProcessEnv = {}, ownMessage = false): ProviderFailure | null { + environment: NodeJS.ProcessEnv = {}, ownMessage = false, sessionFailure?: AcpxTerminalSessionFailure): ProviderFailure | null { const acp = acpError(error); const data = record(acp?.data); const detail = record(data.error); - const fallback = ownMessage ? record(error) : {}; - const message = typeof detail.message === "string" ? detail.message + const fallback = ownMessage || sessionFailure ? record(error) : {}; + const ownText = typeof detail.message === "string" ? detail.message : typeof data.details === "string" ? data.details : typeof acp?.message === "string" ? acp.message : fallback.message; - if (typeof message !== "string") return null; + const title = typeof sessionFailure?.title === "string" ? sessionFailure.title.trim() : ""; + const message = typeof ownText === "string" && title && !ownText.includes(title) ? `${ownText}\n${title}` + : typeof ownText === "string" ? ownText : title || undefined; + if (typeof message !== "string" || !message) return null; const status = data.status ?? data.statusCode ?? detail.status ?? (agent === "claude" ? Number(/^API error: (\d{3})\b/.exec(message)?.[1]) : undefined); const code = detail.type ?? detail.code ?? data.errorType ?? data.errorKind; const category = (typeof code === "string" ? CATEGORIES.get(code) : undefined) - ?? (acp?.code === ACP_AUTH_REQUIRED ? "authentication" : undefined); + ?? (acp?.code === ACP_AUTH_REQUIRED ? "authentication" : undefined) + ?? sessionLimitCategory(sessionFailure); return createProviderFailure({ provider: agent === "claude" ? "anthropic" : agent === "codex" ? "openai" : agent, phase, message, httpStatus: status, providerCode: code, requestId: data.request_id, category, environment }); } -export function acpxTerminalResult(result: unknown, agent: string, environment: NodeJS.ProcessEnv = {}): Record { +export function acpxTerminalResult(result: unknown, agent: string, environment: NodeJS.ProcessEnv = {}, + sessionFailure?: AcpxTerminalSessionFailure): Record { const raw = record(result); - const providerFailure = raw.status === "failed" ? acpxProviderFailure(raw.error, agent, "turn", environment, true) : null; + const providerFailure = raw.status === "failed" + ? acpxProviderFailure(raw.error, agent, "turn", environment, true, sessionFailure) : null; const terminal = boundedSidecarValue(result); if (providerFailure) { terminal.status = "failed"; diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index 87548d4b507..3c663b40447 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -1778,6 +1778,7 @@ describe("ACPX runtime host", () => { const onElicitation = vi.fn(); const onExtensionRequest = vi.fn(); const onExtensionNotification = vi.fn(); + const onTerminalSessionFailure = vi.fn(); const runtime = runtimePort({ startTurn }); const host = await AcpxRuntimeHost.open( { @@ -1797,6 +1798,7 @@ describe("ACPX runtime host", () => { onElicitation, onExtensionRequest, onExtensionNotification, + onTerminalSessionFailure, }), ).toMatchObject({ requestId: turn.requestId, result: turn.result }); expect(startTurn).toHaveBeenCalledWith({ @@ -1805,6 +1807,7 @@ describe("ACPX runtime host", () => { onElicitation, onExtensionRequest, onExtensionNotification, + onTerminalSessionFailure, }); expect(() => host.startTurn({ text: "Concurrent", requestId: "turn-2" }), diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts index c01b8125811..8b64d609fc4 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts @@ -6,6 +6,7 @@ import { resolveClaudeEffort } from "./claude-effort.js"; import { resolveGrokEffort } from "./grok-effort.js"; import { piProviderConfiguration } from "./pi-provider-config.js"; import { withAcpxTurnCancellation } from "./turn-cancellation.js"; +import type { AcpxTerminalSessionFailure } from "./provider-failure.js"; import { resolveAcpxProviderMode } from "./provider-mode.js"; import { resolvePiThinkingLevel, type PiThinkingLevel } from "./pi-thinking.js"; import { preparedHarnessHome, preconfiguredHomeEnvironment, usesPreconfiguredHome } from "../preconfigured-home.js"; @@ -131,6 +132,8 @@ export interface AcpxRuntimeTurnInput { onPermissionRequest?: AcpRuntimeOptions["onPermissionRequest"]; onExtensionRequest?: AcpRuntimeOptions["onExtensionRequest"]; onExtensionNotification?: AcpRuntimeOptions["onExtensionNotification"]; + /** Receives the typed AIR failure behind ACPX's generic terminal error. */ + onTerminalSessionFailure?: (failure: AcpxTerminalSessionFailure) => void; } export interface AcpxRuntimeTurn { @@ -914,6 +917,7 @@ export class AcpxRuntimeHost { ...(input.onPermissionRequest ? { onPermissionRequest: input.onPermissionRequest } : {}), ...(input.onExtensionRequest ? { onExtensionRequest: input.onExtensionRequest } : {}), ...(input.onExtensionNotification ? { onExtensionNotification: input.onExtensionNotification } : {}), + ...(input.onTerminalSessionFailure ? { onTerminalSessionFailure: input.onTerminalSessionFailure } : {}), }); this.#activeTurn = turn; const managed = withAcpxTurnCancellation(turn, reason => this.close({ reason }), ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS); diff --git a/packages/paperclip-runner/src/drivers/provider-failure.test.ts b/packages/paperclip-runner/src/drivers/provider-failure.test.ts index 27f28d6f757..fb128eaa21a 100644 --- a/packages/paperclip-runner/src/drivers/provider-failure.test.ts +++ b/packages/paperclip-runner/src/drivers/provider-failure.test.ts @@ -56,6 +56,29 @@ describe("provider failure boundary", () => { expect(acpxProviderFailure(new Error(""), "claude", "startup", {}, true)).toBeNull(); }); + it("keeps a typed ACP session limit's provider title and classifies an exhausted allowance", () => { + const generic = "ACP agent reported a terminal limit failure."; + const failed = { status: "failed", error: { code: "RUNTIME", message: generic } }; + const terminal = (title: string, category = "limit") => + acpxTerminalResult(failed, "claude", { ANTHROPIC_API_KEY: "fixture-secret" }, { category, title, details: "private stack fixture-secret" }); + expect(terminal("You've hit your session limit · resets 2am (UTC)")).toMatchObject({ status: "failed", error: { providerFailure: { + provider: "anthropic", phase: "turn", category: "quota_exhausted", retryability: "requires_action", + message: `${generic}\nYou've hit your session limit · resets 2am (UTC)` } } }); + expect(terminal("The Claude account has no available quota.")).toMatchObject({ error: { providerFailure: { category: "quota_exhausted" } } }); + expect(terminal("Claude is temporarily rate limited.")).toMatchObject({ error: { providerFailure: { category: "rate_limit" } } }); + // Context, turn and configured budget limits are not an account allowance. + for (const title of ["This Claude turn reached its configured limit.", "This Claude session reached its configured budget.", "Context window limit exceeded"]) { + expect(terminal(title)).toMatchObject({ error: { providerFailure: { category: "unknown", message: `${generic}\n${title}` } } }); + } + expect(terminal("You've hit your session limit", "request")).toMatchObject({ error: { providerFailure: { category: "unknown" } } }); + expect(JSON.stringify(terminal("You've hit your session limit"))).not.toMatch(/fixture-secret|private stack/); + // Without the typed callback the generic terminal stays unclassified. + expect(acpxTerminalResult(failed, "claude")).toMatchObject({ error: { providerFailure: { category: "unknown", message: generic } } }); + // The thrown form of the same terminal is classified the same way. + expect(acpxProviderFailure(new Error(generic), "claude", "turn", {}, false, { category: "limit", title: "Claude is temporarily rate limited." })) + .toMatchObject({ category: "rate_limit", message: `${generic}\nClaude is temporarily rate limited.` }); + }); + it("keeps the ACPX terminal cause when the raw result exceeds its payload limit", () => { const terminal = acpxTerminalResult({ status: "failed", error: { ...acp, data: { ...acp.data, responseBody: "private-body".repeat(7000) } } }, "claude"); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index d6f28c91da0..c60caf97dc6 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -32,7 +32,7 @@ import * as executionTarget from "@paperclipai/adapter-utils/execution-target"; import * as runnerProbe from "../vendor/paperclip-runner/index.js"; import * as nativeCodexRunner from "../services/native-runtime/native-codex-runner.js"; import { testNativeRunnerAuthentication } from "../services/native-runtime/setup-readiness.js"; -import { prepareManagedAiRuntime, withManagedAiProbe, assertManagedAiProjectAuth, isAiConnectionBusy, managedAiSessionFingerprintConfig } from "../services/ai-connection-runtime.js"; +import { prepareManagedAiRuntime, withManagedAiProbe, assertManagedAiProjectAuth, isAiConnectionBusy, isAiConnectionIncompatible, managedAiSessionFingerprintConfig } from "../services/ai-connection-runtime.js"; import { execute as executeGemini, testEnvironment as testGeminiEnvironment } from "@paperclipai/adapter-gemini-local/server"; import { toolAccessService } from "../services/tool-access.js"; import { secretService } from "../services/secrets.js"; @@ -1949,6 +1949,26 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello" await expect(assertManagedAiProjectAuth({ args: ["--api-key=override"] }, "xai", target)).rejects.toThrow("overrides"); } finally { execute.mockRestore(); } }); + it("reports only a found conflict as configuration, naming the files to fix", async () => { + const execute = vi.spyOn(executionTarget, "runAdapterExecutionTargetProcess"); + const target = { kind: "remote", transport: "sandbox", remoteCwd: "/workspace/project" } as Parameters[2]; + const outcome = (exitCode: number | null, timedOut = false) => + ({ exitCode, stdout: "", stderr: "", signal: null, timedOut }) as Awaited>; + try { + execute.mockResolvedValue(outcome(42)); + const conflict = await assertManagedAiProjectAuth({}, "anthropic", target).catch((error: unknown) => error); + expect(isAiConnectionIncompatible(conflict)).toBe(true); + expect(String((conflict as Error).message)).toContain(".claude/settings.json"); + // A check that could not complete is not proof of a configuration conflict. + for (const failed of [outcome(43), outcome(null, true), outcome(127)]) { + execute.mockResolvedValue(failed); + const unavailable = await assertManagedAiProjectAuth({}, "anthropic", target).catch((error: unknown) => error); + expect(unavailable).toBeInstanceOf(Error); + expect(isAiConnectionIncompatible(unavailable)).toBe(false); + } + expect(isAiConnectionIncompatible(await assertManagedAiProjectAuth({ args: ["--settings=x"] }, "anthropic", target).catch((error: unknown) => error))).toBe(true); + } finally { execute.mockRestore(); } + }); it("requires a typed project authentication capability for task-only environments", async () => { const execute = vi.spyOn(executionTarget, "runAdapterExecutionTargetProcess"); const description = { kind: "description", taskId: "task", projectAuthCheckVersion: 1, diff --git a/server/src/__tests__/heartbeat-provider-limit-wait.test.ts b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts new file mode 100644 index 00000000000..66067a46441 --- /dev/null +++ b/server/src/__tests__/heartbeat-provider-limit-wait.test.ts @@ -0,0 +1,197 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRunEvents, heartbeatRuns, issues } from "@paperclipai/db"; +import { registerServerAdapter, unregisterServerAdapter } from "../adapters/index.js"; +import { heartbeatService } from "../services/heartbeat.js"; +import { createProviderLimitWait } from "../services/heartbeat/provider-limit-wait.js"; +import { executionProjectionsForRuns } from "../services/execution-projection.js"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; +if (!recoveryPostgresSupport.supported) console.warn(`Skipping provider limit wait tests: ${recoveryPostgresSupport.reason}`); + +/** + * An agent whose provider account is out of allowance must not start new work + * that will fail the same way. The wake waits for the reported reset instead. + */ +describePostgres("heartbeat provider usage limit wait", () => { + let temporary: Awaited>; + let db: ReturnType; + let heartbeat: ReturnType; + const adapterType = "codex_local"; + const dispatched = new Set(); + const oldHome = process.env.PAPERCLIP_HOME; + let home: string; + + beforeAll(async () => { + temporary = await startRecoveryPostgres("heartbeat-provider-limit-"); + db = createDb(temporary.connectionString); + home = await mkdtemp(path.join(tmpdir(), "paperclip-provider-limit-")); + process.env.PAPERCLIP_HOME = home; + heartbeat = heartbeatService(db); + registerServerAdapter({ + type: adapterType, supportsLocalAgentJwt: false, + execute: async input => { + dispatched.add(input.runId); + await db.update(issues).set({ status: "in_review" }).where(eq(issues.id, String(input.context.issueId))); + return { exitCode: 0, signal: null, timedOut: false, resultJson: {} }; + }, + testEnvironment: async () => ({ adapterType, status: "pass", checks: [], testedAt: new Date().toISOString() }), + }); + }, 60_000); + + afterAll(async () => { + await heartbeat?.drainActiveRunExecutions(); + unregisterServerAdapter(adapterType); + await db?.$client.end({ timeout: 0 }); + await temporary?.cleanup(); + if (oldHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = oldHome; + if (home) await rm(home, { recursive: true, force: true }); + }); + + async function seed(prior: Array>) { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(); + const userId = `limit-user-${companyId}`; + await db.insert(authUsers).values({ id: userId, name: "Limit user", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(companies).values({ id: companyId, name: "Provider limit fixture", issuePrefix: `L${companyId.slice(0, 6)}` }); + await db.insert(companyMemberships).values({ companyId, principalId: userId, principalType: "user", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Limited agent", adapterType, status: "idle", adapterConfig: {} }); + await db.insert(issues).values({ id: issueId, companyId, title: "Triage the next alert", status: "todo", assigneeAgentId: agentId, responsibleUserId: userId, issueNumber: 1 }); + for (const values of prior) { + await db.insert(heartbeatRuns).values({ companyId, agentId, invocationSource: "automation", triggerDetail: "system", ...values }); + } + return { companyId, agentId, issueId }; + } + + async function wake(f: Awaited>, manual = false) { + const run = await heartbeat.wakeup(f.agentId, manual + ? { source: "on_demand", triggerDetail: "manual", reason: "manual", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } } + : { source: "automation", triggerDetail: "system", reason: "issue_assigned", payload: { issueId: f.issueId }, contextSnapshot: { issueId: f.issueId } }); + expect(run).not.toBeNull(); + await heartbeat.drainActiveRunExecutions(); + const [finished] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run!.id)); + return finished; + } + + const rateLimited = (minutesAgo: number, overrides: Record = {}) => ({ + status: "failed", + startedAt: new Date(Date.now() - (minutesAgo + 2) * 60_000), + finishedAt: new Date(Date.now() - minutesAgo * 60_000), + errorCode: "provider_session_open_rejected", + resultJson: { providerFailure: { schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "startup", + category: "rate_limit", httpStatus: 429, retryability: "unknown", + message: "API error: 429 This request would exceed your account's rate limit. Please try again later", ...overrides } }, + }); + + it("defers new work until the reported reset instead of dispatching a doomed run", async () => { + const limited = rateLimited(16); + const f = await seed([limited]); + const resetAt = new Date(limited.finishedAt.getTime() + 60 * 60_000); + const finished = await wake(f); + + expect(dispatched.has(finished.id)).toBe(false); + expect(finished).toMatchObject({ + status: "cancelled", + errorCode: "provider_limit_wait", + resultJson: { + executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, + providerLimitWait: { resetAt: resetAt.toISOString(), resetReported: false, category: "rate_limit" }, + }, + }); + expect(finished.error).toContain(resetAt.toISOString()); + const successors = await db.select().from(heartbeatRuns) + .where(eq(heartbeatRuns.agentId, f.agentId)); + const retry = successors.find(run => run.status === "scheduled_retry"); + expect(retry).toMatchObject({ scheduledRetryReason: "ai_connection_pool_wait" }); + expect(retry!.scheduledRetryAt!.getTime()).toBeGreaterThanOrEqual(resetAt.getTime()); + expect(retry!.scheduledRetryAt!.getTime()).toBeLessThan(resetAt.getTime() + 5 * 60_000); + const events = await db.select().from(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, finished.id)); + expect(events.some(event => event.message?.includes("provider usage limit"))).toBe(true); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(issue.status).not.toBe("blocked"); + // The persisted projection names the wait and its reset. + expect((await executionProjectionsForRuns(db, f.companyId, [retry!.id])).get(retry!.id)).toMatchObject({ + label: "Waiting for usage limit reset", phase: "retry_scheduled", + nextAction: expect.stringContaining(retry!.scheduledRetryAt!.toISOString()), + }); + + // Nothing is dropped: the wait starts the work once the reset has passed. + expect((await heartbeat.promoteDueScheduledRetries(new Date())).runIds).not.toContain(retry!.id); + // Model the hour passing by moving the limit evidence back past its reset. + await db.update(heartbeatRuns).set({ startedAt: new Date(Date.now() - 3 * 60 * 60_000), finishedAt: new Date(Date.now() - 2 * 60 * 60_000) }) + .where(eq(heartbeatRuns.errorCode, "provider_session_open_rejected")); + expect((await heartbeat.promoteDueScheduledRetries(new Date(retry!.scheduledRetryAt!.getTime() + 1))).runIds).toContain(retry!.id); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + expect(dispatched.has(retry!.id)).toBe(true); + const [resumed] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, retry!.id)); + expect(resumed.status).toBe("succeeded"); + }, 60_000); + + it("releases the task when its retry cannot be scheduled, even if diagnostics fail", async () => { + const f = await seed([rateLimited(16)]); + const [agent] = await db.select().from(agents).where(eq(agents.id, f.agentId)); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "running", + invocationSource: "automation", triggerDetail: "system", contextSnapshot: { issueId: f.issueId } }).returning(); + const deps = { + setRunStatusIfRunning: vi.fn(async () => ({ updated: true, run: { ...run, status: "cancelled" } })), + setWakeupStatus: vi.fn(async () => { throw new Error("wakeup write failed"); }), + appendRunEvent: vi.fn(async () => { throw new Error("event write failed"); }), + scheduleBoundedRetryForRun: vi.fn(async () => { throw new Error("retry write failed"); }), + releaseIssueExecutionAndPromote: vi.fn(async () => undefined), + finalizeAgentStatus: vi.fn(async () => undefined), + }; + const { deferRunForProviderLimit } = createProviderLimitWait(db, deps as never); + await expect(deferRunForProviderLimit(run, agent, { resumingAdmittedRun: false, conversation: false })).resolves.toBe(true); + expect(deps.scheduleBoundedRetryForRun).toHaveBeenCalledTimes(2); + expect(deps.releaseIssueExecutionAndPromote).toHaveBeenCalledOnce(); + expect(deps.finalizeAgentStatus).toHaveBeenCalledOnce(); + + // A scheduled retry keeps the task lock for its successor. + deps.scheduleBoundedRetryForRun.mockReset().mockResolvedValue({ outcome: "scheduled" } as never); + deps.releaseIssueExecutionAndPromote.mockClear(); + await expect(deferRunForProviderLimit(run, agent, { resumingAdmittedRun: false, conversation: false })).resolves.toBe(true); + expect(deps.releaseIssueExecutionAndPromote).not.toHaveBeenCalled(); + }); + + it("recovers a wait whose retry was lost to a crash after its cancellation committed", async () => { + const f = await seed([]); + const [lost] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.agentId, status: "cancelled", + invocationSource: "automation", triggerDetail: "system", issueId: f.issueId, scopeKind: "issue", + startedAt: new Date(Date.now() - 10 * 60_000), finishedAt: new Date(Date.now() - 9 * 60_000), errorCode: "provider_limit_wait", + resultJson: { executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false } }, + contextSnapshot: { issueId: f.issueId } }).returning(); + await db.update(issues).set({ status: "in_progress", executionRunId: lost.id, executionLockedAt: new Date(Date.now() - 9 * 60_000) }) + .where(eq(issues.id, f.issueId)); + await heartbeat.sweepStaleIssueLocks(); + await heartbeat.reconcileStrandedAssignedIssues(); + await heartbeat.drainActiveRunExecutions(); + const runs = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.agentId, f.agentId)); + // Existing stranded-work recovery gives the wake a bounded successor; the gate re-checks the limit when it runs. + expect(runs.find(run => run.id !== lost.id)).toMatchObject({ status: "scheduled_retry", retryOfRunId: lost.id }); + const [issue] = await db.select().from(issues).where(eq(issues.id, f.issueId)); + expect(issue).toMatchObject({ status: "in_progress", executionRunId: null }); + }, 60_000); + + it("still dispatches a manual run, and work after a later success or an expired reset", async () => { + const manual = await seed([rateLimited(16)]); + expect(dispatched.has((await wake(manual, true)).id)).toBe(true); + + const recovered = await seed([rateLimited(30), { + status: "succeeded", startedAt: new Date(Date.now() - 10 * 60_000), finishedAt: new Date(Date.now() - 9 * 60_000), + }]); + expect(dispatched.has((await wake(recovered)).id)).toBe(true); + + const expired = await seed([rateLimited(90)]); + expect(dispatched.has((await wake(expired)).id)).toBe(true); + + const unrelated = await seed([{ ...rateLimited(5), resultJson: { providerFailure: { + schema: "paperclip.provider-failure.v1", provider: "anthropic", phase: "turn", category: "unknown", + retryability: "unknown", message: "ACP agent reported a terminal limit failure." } } }]); + expect(dispatched.has((await wake(unrelated)).id)).toBe(true); + }, 60_000); +}); diff --git a/server/src/services/ai-connection-runtime.ts b/server/src/services/ai-connection-runtime.ts index 593b6aeeb34..fc94bd8c1d0 100644 --- a/server/src/services/ai-connection-runtime.ts +++ b/server/src/services/ai-connection-runtime.ts @@ -34,6 +34,19 @@ export function isAiConnectionBusy(error: unknown): error is HttpError { (error.details as { code?: unknown } | undefined)?.code === "ai_connection_busy"; } +/** A real conflict between the agent's AI connection and its project or arguments. */ +export function isAiConnectionIncompatible(error: unknown): error is HttpError { + return error instanceof HttpError && error.status === 422 && + (error.details as { code?: unknown } | undefined)?.code === "ai_connection_incompatible"; +} + +function projectAuthConflict(files: string[]) { + return unprocessable( + `This agent's managed AI connection conflicts with project authentication settings in ${files.join(" or ")}. Remove the API key, token or apiKeyHelper settings from that project file, or choose a different AI connection for this agent.`, + { code: "ai_connection_incompatible" }, + ); +} + // Blank values intentionally override inherited credentials in CLI child environments. export const AI_AUTH_ENV_KEYS = [ "COPILOT_GITHUB_TOKEN", @@ -162,10 +175,11 @@ done`, onLog: async () => {}, }, ); + if (result.exitCode === 42) throw projectAuthConflict(files); + // The check fails closed, but an unfinished check is not a found conflict. if (result.exitCode !== 0) - throw unprocessable( - "The environment's project authentication settings must be checked before using this AI connection", - { code: "ai_connection_incompatible" }, + throw new Error( + `Paperclip could not check the environment's project authentication settings (${result.timedOut ? "timed out" : `exit ${result.exitCode ?? result.signal ?? "unknown"}`}). The run did not start; it can be retried.`, ); return; } @@ -180,10 +194,7 @@ done`, content, ) ) { - throw unprocessable( - "Project authentication settings conflict with the selected AI connection", - { code: "ai_connection_incompatible" }, - ); + throw projectAuthConflict([relative]); } } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; diff --git a/server/src/services/execution-projection.test.ts b/server/src/services/execution-projection.test.ts index 05ca7391a76..ca5995027c1 100644 --- a/server/src/services/execution-projection.test.ts +++ b/server/src/services/execution-projection.test.ts @@ -101,6 +101,15 @@ describe("execution truth projection", () => { scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeAiConnectionWait: 0 } }), undefined, [], undefined, now)) .toMatchObject({ label: "Waiting for AI subscription", phase: "retry_scheduled", attempt: 1, recoveryOwner: null }); }); + it("shows a provider usage limit wait with its reset time", () => { + const scheduledRetryAt = new Date("2026-11-05T13:47:50.375Z"); + expect(projectExecution(run({ runtimeMode: "native", status: "scheduled_retry", scheduledRetryReason: "ai_connection_pool_wait", + scheduledRetryAt, scheduledRetryAttempt: 3, contextSnapshot: { wakeReason: "provider_limit_retry", failureRetriesBeforeAiConnectionWait: 0 } }), + undefined, [], undefined, now)).toMatchObject({ + label: "Waiting for usage limit reset", phase: "retry_scheduled", attempt: 1, recoveryOwner: null, + nextAction: expect.stringContaining(scheduledRetryAt.toISOString()), + }); + }); it("shows a workspace wait without presenting its deferral count as failed attempts", () => { expect(projectExecution(run({ runtimeMode: "legacy", status: "scheduled_retry", scheduledRetryReason: "workspace_busy", scheduledRetryAttempt: 12, contextSnapshot: { failureRetriesBeforeWorkspaceWait: 1 } }), undefined, [], undefined, now)) diff --git a/server/src/services/execution-projection.ts b/server/src/services/execution-projection.ts index 1e7b5c65340..12a4520a578 100644 --- a/server/src/services/execution-projection.ts +++ b/server/src/services/execution-projection.ts @@ -37,6 +37,7 @@ const executionRunColumns = { status: heartbeatRuns.status, contextSnapshot: sql>`jsonb_build_object( 'issueId', ${heartbeatRuns.contextSnapshot}->'issueId', + 'wakeReason', ${heartbeatRuns.contextSnapshot}->'wakeReason', 'failureRetriesBeforeAiConnectionWait', ${heartbeatRuns.contextSnapshot}->'failureRetriesBeforeAiConnectionWait', 'failureRetriesBeforeWorkspaceWait', ${heartbeatRuns.contextSnapshot}->'failureRetriesBeforeWorkspaceWait')`, }; @@ -302,6 +303,12 @@ export function projectExecution( projection.nextAction = "Waiting for the AI subscription's current execution to finish; the scheduled check will revalidate access."; return set("retry_scheduled", "Waiting for AI subscription"); } + if (run.status === "scheduled_retry" && run.scheduledRetryReason === "ai_connection_pool_wait" + && run.contextSnapshot?.wakeReason === "provider_limit_retry") { + const resetAt = run.scheduledRetryAt ? new Date(run.scheduledRetryAt).toISOString() : "its reported reset"; + projection.nextAction = `Waiting for the agent's provider usage limit to reset; the work starts at ${resetAt}. To continue sooner, restore the provider allowance or change the agent's AI connection, then start a manual run.`; + return set("retry_scheduled", "Waiting for usage limit reset"); + } if (run.status === "scheduled_retry" && run.scheduledRetryReason === "workspace_busy") { projection.nextAction = "Waiting for the live workspace holder to finish; the scheduled check will revalidate ownership."; return set("retry_scheduled", "Waiting for workspace"); diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index d59865fcc80..97f58880e84 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -5,6 +5,7 @@ import { registerIssueRecoveryWork } from "./recovery/work.js"; import { registerRunDispatchWork } from "./heartbeat/dispatch-work.js"; import { NATIVE_RETAINED_OWNER_WAIT } from "./native-runtime/native-controller-recovery.js"; import { describeFailedAttachment } from "./native-runtime/remote-warm-session-recovery.js"; +import { createProviderLimitWait } from "./heartbeat/provider-limit-wait.js"; import { externalConversationSettlementWork } from "./external-conversation-settlement-work.js"; import { nativeRecoveryNoticeWork } from "./native-runtime/native-recovery-notice-work.js"; import { nativeRunProjectionWork } from "./native-runtime/native-run-projection-work.js"; @@ -288,6 +289,7 @@ import { retryChatControlAdmission } from "./chat-control-admission-retry.js"; import { prepareManagedAiRuntime, assertManagedAiProjectAuth, + isAiConnectionIncompatible, } from "./ai-connection-runtime.js"; import { aiConnectionBindingSchema, aiRuntimeConnectionBindingSchema, type AiConnectionRouterSelection } from "@paperclipai/shared"; import { aiConnectionRouterService, AiConnectionPoolExhausted, applyAiConnectionRouterTaskSettings } from "./ai-connection-router.js"; @@ -1837,6 +1839,15 @@ export function heartbeatService( return withRunExecutionTiming(runId, () => executeRunAttempt(runId, runOptions)); } + const { deferRunForProviderLimit } = createProviderLimitWait(db, { + setRunStatusIfRunning, + setWakeupStatus, + appendRunEvent, + scheduleBoundedRetryForRun, + releaseIssueExecutionAndPromote, + finalizeAgentStatus, + }); + async function executeRunAttempt( runId: string, runOptions: { nativeLeaseOwner?: string; nativeRestartRecovery?: NativeRestartRecoveryClaim; onNativeExecutionLeaseAcquired?: () => Promise }, @@ -2069,6 +2080,10 @@ export function heartbeatService( await finalizeAgentStatus(agent.id, "cancelled"); return; } + if (await deferRunForProviderLimit(run, agent, { + resumingAdmittedRun: !!runOptions.nativeLeaseOwner, + conversation: isConversation(conversationIdentity), + })) return; run = await timePreparation("chat_completion", () => prepareChatCompletionTurn(db, run)); const preparedConversation = await timePreparation("conversation", () => prepareConversationTurn(db, run)); run = { ...run, contextSnapshot: preparedConversation.context }; @@ -2810,7 +2825,11 @@ export function heartbeatService( }); if (managedAiRuntime && aiBinding) { try { await assertManagedAiProjectAuth({ ...resolvedConfig, cwd: executionWorkspace.cwd }, aiBinding.provider, executionTarget); } - catch { throw new ConfigurationIncompleteFailure("Project authentication conflicts with this agent’s managed AI connection", { configurationIncomplete: { reason: "ai_connection_incompatible", actionUrl: `/agents/${agent.id}/runtime` } }); } + catch (error) { + // Only a found conflict is the board's to fix; an unfinished check stays an ordinary failure. + if (!isAiConnectionIncompatible(error)) throw error; + throw new ConfigurationIncompleteFailure(error.message, { configurationIncomplete: { reason: "ai_connection_incompatible", actionUrl: `/agents/${agent.id}/runtime` } }); + } } const remoteExecution = realizationResult.remoteExecution; if (nativeChatWorkspaceScope && materializeNativeChatWorktree) { diff --git a/server/src/services/heartbeat/provider-limit-wait.ts b/server/src/services/heartbeat/provider-limit-wait.ts new file mode 100644 index 00000000000..154bed8250c --- /dev/null +++ b/server/src/services/heartbeat/provider-limit-wait.ts @@ -0,0 +1,117 @@ +import type { Db } from "@paperclipai/db"; +import { agents, heartbeatRuns } from "@paperclipai/db"; +import { logger } from "../../middleware/logger.js"; +import { AI_CONNECTION_POOL_WAIT_RETRY_REASON } from "../../modules/run-dispatch/index.js"; +import { agentProviderLimitCooldown } from "../native-runtime/native-startup-retry.js"; +import { parseObject } from "../../adapters/utils.js"; + +type HeartbeatRun = typeof heartbeatRuns.$inferSelect; +type Agent = typeof agents.$inferSelect; + +export interface ProviderLimitWaitDependencies { + setRunStatusIfRunning: (runId: string, status: "cancelled", patch: { + error: string; errorCode: string; finishedAt: Date; resultJson: Record; + }) => Promise<{ updated: boolean; run?: HeartbeatRun | null }>; + setWakeupStatus: (wakeupRequestId: string | null | undefined, status: "cancelled", patch: { finishedAt: Date; error: string }) => Promise; + appendRunEvent: (run: HeartbeatRun, event: { + eventType: string; stream: "system"; level: "warn"; message: string; payload: Record; + }) => Promise; + scheduleBoundedRetryForRun: (run: HeartbeatRun, agent: Agent, opts: { + now: Date; retryReason: string; wakeReason: string; maxAttempts: number; delayMs: number; + }) => Promise<{ outcome: string }>; + releaseIssueExecutionAndPromote: (run: HeartbeatRun) => Promise; + finalizeAgentStatus: (agentId: string, outcome: "cancelled") => Promise; +} + +/** Holds new work while the agent's provider account is out of allowance. */ +export function createProviderLimitWait(db: Db, deps: ProviderLimitWaitDependencies) { + const { + setRunStatusIfRunning, setWakeupStatus, appendRunEvent, scheduleBoundedRetryForRun, + releaseIssueExecutionAndPromote, finalizeAgentStatus, + } = deps; + + /** + * An agent whose provider account recently reported an exhausted allowance + * would fail new work the same way. Before any workspace or provider work, + * hold the wake until the reported reset and retry it then. A manual run, a + * recovering run, a conversation turn, and pool routing (which selects + * another account) still start, so an operator can always test a repair. + */ + async function deferRunForProviderLimit( + run: HeartbeatRun, + agent: Agent, + options: { resumingAdmittedRun: boolean; conversation: boolean }, + ): Promise { + if (options.resumingAdmittedRun || options.conversation) return false; + if (run.invocationSource === "on_demand" && run.triggerDetail === "manual") return false; + if (parseObject(parseObject(agent.runtimeConfig).aiConnection).mode === "router") return false; + const now = new Date(); + const cooldown = await agentProviderLimitCooldown(db, { + companyId: run.companyId, agentId: run.agentId, runId: run.id, now, + }).catch((err: unknown) => { + // Missing evidence must not stop work; the provider still reports its own limit. + logger.warn({ err, runId: run.id }, "provider limit evidence unavailable before dispatch"); + return null; + }); + if (!cooldown) return false; + const resetAt = cooldown.resetAt.toISOString(); + const message = `The provider usage limit for this agent is expected to reset at ${resetAt}${ + cooldown.resetReported ? "" : " (no reset time was reported, so Paperclip waits one hour)" + }. Paperclip will start this work then.`; + const cancelled = await setRunStatusIfRunning(run.id, "cancelled", { + error: message, errorCode: "provider_limit_wait", finishedAt: now, + resultJson: { + executionRecovery: { kind: "ai_connection_wait", providerWorkStarted: false }, + providerLimitWait: { + resetAt, resetReported: cooldown.resetReported, category: cooldown.category, + sourceRunId: cooldown.sourceRunId, actionUrl: `/agents/${agent.id}/runtime`, + }, + cancellation: { + source: "control_plane", expected: true, initiator: { type: "system" }, + reason: "Waiting for the agent's provider usage limit to reset", recordedAt: now.toISOString(), + }, + retryAt: resetAt, + }, + }); + if (!cancelled.updated) return true; + const deferred = cancelled.run ?? run; + // The cancellation is committed. From here the wake must either gain its + // retry or release its task lock; diagnostics never decide that. + let scheduled = false; + try { + // Scheduling is idempotent per source run, so one more attempt is safe. + for (let attempt = 1; attempt <= 2 && !scheduled; attempt += 1) { + try { + // Each wait is one retry at the reset; evidence expires within a day. + const retry = await scheduleBoundedRetryForRun(deferred, agent, { + now, retryReason: AI_CONNECTION_POOL_WAIT_RETRY_REASON, wakeReason: "provider_limit_retry", + maxAttempts: (run.scheduledRetryAttempt ?? 0) + 1, + delayMs: Math.max(1000, cooldown.resetAt.getTime() - now.getTime() + 1000), + }); + scheduled = retry.outcome === "scheduled"; + if (!scheduled) break; + } catch (err) { + logger.warn({ err, runId: run.id, attempt }, "failed to schedule the provider limit retry"); + } + } + await appendRunEvent(deferred, { + eventType: "lifecycle", stream: "system", level: "warn", + message: scheduled + ? `Waiting for the provider usage limit to reset at ${resetAt} before starting work` + : `The provider usage limit resets at ${resetAt}; this wake could not schedule its retry and released the task`, + payload: { resetAt, resetReported: cooldown.resetReported, category: cooldown.category, sourceRunId: cooldown.sourceRunId, retryScheduled: scheduled }, + }).catch((err: unknown) => logger.warn({ err, runId: run.id }, "failed to record the provider limit wait")); + await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: now, error: message }) + .catch((err: unknown) => logger.warn({ err, runId: run.id }, "failed to settle the provider limit wakeup")); + } finally { + try { + if (!scheduled) await releaseIssueExecutionAndPromote(deferred); + } finally { + await finalizeAgentStatus(run.agentId, "cancelled"); + } + } + return true; + } + + return { deferRunForProviderLimit }; +} diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 5dc1077bffa..294f20d9b4b 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -69,7 +69,7 @@ export function legacyExecutionNeedsReconciliation( && (run.resultJson?.executionCancellation as Record | undefined)?.state === "acknowledged") return false; // Waiting for a subscription or workspace precedes provider execution. It is // a resource wait, not a failed provider attempt or permission to replay work. - if (run.status === "cancelled" && (run.errorCode === "ai_connection_busy" || run.errorCode === "ai_connection_pool_exhausted") && + if (run.status === "cancelled" && (run.errorCode === "ai_connection_busy" || run.errorCode === "ai_connection_pool_exhausted" || run.errorCode === "provider_limit_wait") && evidence?.kind === "ai_connection_wait" && evidence.providerWorkStarted === false) return false; if (run.status === "cancelled" && run.errorCode === "workspace_busy" && evidence?.kind === "workspace_wait" && evidence.providerWorkStarted === false) return false; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 652cfd5780b..0dd73006c1d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12842,6 +12842,30 @@ describe("native session bounded recovery", () => { payload: { status: "failed", error: { message: "usageLimitExceeded" } }, }), ).toBe(false); + // ACP harnesses report the same terminal through a typed provider failure. + const acpLimit = (category: string) => ({ + ...event, + payload: { + status: "failed", + error: { + code: "RUNTIME", + message: "ACP agent reported a terminal limit failure.", + providerFailure: { + schema: "paperclip.provider-failure.v1", + provider: "anthropic", + phase: "turn", + category, + message: "ACP agent reported a terminal limit failure.\nYou've hit your session limit · resets 2am (UTC)", + retryability: "unknown", + }, + }, + }, + }); + expect(nativeProviderUsageLimitFromEvent(acpLimit("quota_exhausted"))).toBe(true); + expect(nativeProviderUsageLimitFromEvent(acpLimit("rate_limit"))).toBe(true); + // A context, turn or budget limit is not an account allowance. + expect(nativeProviderUsageLimitFromEvent(acpLimit("unknown"))).toBe(false); + expect(nativeProviderUsageLimitFromEvent({ ...acpLimit("quota_exhausted"), sourceKind: "provider" as never })).toBe(false); expect( nativeSessionFailureDisposition( 1, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 2432068f8ec..1d1febfb4f6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6,6 +6,7 @@ import { NATIVE_PROVIDER_DURABLE_EVENT_TYPES, nativeHarnessBackupManifestSha256, import { prepareCursorCloudRuntime } from "./cursor-cloud-runtime.js"; import { resolveLocalRuntimeApiUrl } from "../../runtime-api.js"; import { providerFailureFromError } from "../../vendor/paperclip-runner/index.js"; +import { readProviderFailure } from "@paperclipai/shared"; import { nativeCancellationHasUnsettledOwner, notifyNativeCancellation, waitForNativeCancellation } from "./native-cancellation-delivery.js"; import { readClaimedNativeStopRecovery } from "./native-cancellation-request.js"; import { publishNativeHarnessBackup } from "./native-harness-backup-publication.js"; @@ -7123,12 +7124,13 @@ export function nativeProviderUsageLimitFromEvent( event: Pick, ): boolean { const payload = record(event.payload); - return ( - event.sourceKind === "runner" && - event.eventType === "turn.failed" && - payload.status === "failed" && - record(payload.error).codexErrorInfo === "usageLimitExceeded" - ); + if (event.sourceKind !== "runner" || event.eventType !== "turn.failed" || payload.status !== "failed") return false; + const error = record(payload.error); + if (error.codexErrorInfo === "usageLimitExceeded") return true; + // ACP harnesses type the same exhausted allowance as a provider failure. + // Context, turn and configured budget limits stay ordinary terminals. + const category = readProviderFailure(error.providerFailure)?.category; + return category === "quota_exhausted" || category === "rate_limit"; } export function nativeSessionFailureDisposition( diff --git a/server/src/services/native-runtime/native-startup-retry.integration.test.ts b/server/src/services/native-runtime/native-startup-retry.integration.test.ts index df2eece61f7..117280af19d 100644 --- a/server/src/services/native-runtime/native-startup-retry.integration.test.ts +++ b/server/src/services/native-runtime/native-startup-retry.integration.test.ts @@ -39,6 +39,7 @@ import { loadNativeRunnerCheckpoint } from "./native-runner-checkpoint.js"; import { environmentRunnerIngressErrorData, JsonRpcCallError, PluginEnvironmentRunnerIngressError } from "@paperclipai/plugin-sdk"; import { accountRunCost } from "../run-cost-accounting.js"; import { settleUnrecoverableExecutions } from "../execution-recovery-resolution.js"; +import { agentProviderLimitCooldown } from "./native-startup-retry.js"; type SessionOpenStatus = "failed" | "rejected" | "indeterminate"; @@ -659,6 +660,34 @@ describe("native provider startup retry", () => { expect(state.issue.status).toBe("in_progress"); }); + it("reads the reset clock from an ACP terminal limit failure's provider wording", async () => { + const agentId = await createAgent(); + const limitedAt = new Date(Date.now() - 5 * 60_000); + const resetHour = new Date(Date.now() + 3 * 60 * 60_000).getUTCHours(); + await recordPriorRun(agentId, { + status: "failed", + startedAt: new Date(limitedAt.getTime() - 60_000), + finishedAt: limitedAt, + errorCode: "native_provider_usage_limit", + resultJson: { + providerFailure: limitFailure({ + category: "quota_exhausted", + message: `ACP agent reported a terminal limit failure.\nYou've hit your session limit · resets ${resetHour}:00 (UTC)`, + }), + }, + }); + const expected = new Date(limitedAt); + expected.setUTCHours(resetHour, 0, 0, 0); + if (expected <= limitedAt) expected.setUTCDate(expected.getUTCDate() + 1); + + const seeded = await seedRun(agentId); + await failStartup(seeded, sessionOpenFailure()); + const state = await read(seeded); + expect(state.coordinator.failureDetail).toMatchObject({ providerCooldownAt: expected.toISOString() }); + expect(await agentProviderLimitCooldown(db, { companyId, agentId, now: new Date() })) + .toMatchObject({ resetAt: expected, resetReported: true }); + }); + it("holds a startup retry for at most one day before trying again", async () => { const seeded = await seedRun(await createAgent()); const failedAt = await failStartup(seeded, sessionOpenFailure({ diff --git a/server/src/services/native-runtime/native-startup-retry.ts b/server/src/services/native-runtime/native-startup-retry.ts index 619859de44f..f645c54aaa8 100644 --- a/server/src/services/native-runtime/native-startup-retry.ts +++ b/server/src/services/native-runtime/native-startup-retry.ts @@ -4,6 +4,7 @@ import { heartbeatRuns, type Db } from "@paperclipai/db"; import { readProviderFailure, type ProviderFailure } from "@paperclipai/shared"; import { logger } from "../../middleware/logger.js"; import { infrastructureRetryDelayMs, PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; +import { parseProviderQuotaClockReset } from "../provider-quota-reset.js"; /** Categories that report an exhausted account or request allowance. */ const PROVIDER_LIMIT_CATEGORIES = ["rate_limit", "quota_exhausted"]; @@ -13,26 +14,46 @@ const PROVIDER_LIMIT_ERROR_CODES = ["native_provider_usage_limit", "provider_quo const NATIVE_PROVIDER_LIMIT_WINDOW_MS = 24 * 60 * 60_000; const LIMIT_EVIDENCE_ROWS = 20; -function limitCooldownAt(failure: ProviderFailure | null, observedAt: Date): number | null { +type ProviderLimitEvidence = { resetAt: number; resetReported: boolean; category: string; sourceRunId: string | null }; + +function limitEvidence(failure: ProviderFailure | null, observedAt: Date, sourceRunId: string | null): ProviderLimitEvidence | null { if (!failure || !PROVIDER_LIMIT_CATEGORIES.includes(failure.category)) return null; const retryAt = failure.retryAt ? Date.parse(failure.retryAt) : Number.NaN; - return Number.isFinite(retryAt) ? retryAt : observedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS; + if (Number.isFinite(retryAt)) return { resetAt: retryAt, resetReported: true, category: failure.category, sourceRunId }; + // ACP harnesses report the reset only in the provider's own wording. + const clockReset = parseProviderQuotaClockReset(failure.message, observedAt); + return clockReset + ? { resetAt: clockReset.getTime(), resetReported: true, category: failure.category, sourceRunId } + : { resetAt: observedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS, resetReported: false, category: failure.category, sourceRunId }; } +export type AgentProviderLimitCooldown = { + /** When the agent's provider allowance is expected to return. */ + resetAt: Date; + /** False when no reset was reported and the default cooldown applies. */ + resetReported: boolean; + category: string; + /** The run whose failure recorded the latest reset. */ + sourceRunId: string | null; +}; + /** * The latest known provider limit for this agent, from the current failure or * from the agent's recent runs. A later successful run clears older evidence. */ -async function nativeProviderCooldownAt(input: { - db: Db; - binding: { companyId: string; agentId: string; runId: string }; - failure: ProviderFailure | null; +export async function agentProviderLimitCooldown(db: Db, input: { + companyId: string; + agentId: string; + /** Excluded from the evidence; the caller supplies its own failure. */ + runId?: string; + failure?: ProviderFailure | null; now: Date; -}): Promise { +}): Promise { const now = input.now.getTime(); - let cooldownAt = limitCooldownAt(input.failure, input.now) ?? 0; - const rows = await input.db + let latest = limitEvidence(input.failure ?? null, input.now, input.runId ?? null); + const rows = await db .select({ + id: heartbeatRuns.id, status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode, observedAt: sql`coalesce(${heartbeatRuns.finishedAt}, ${heartbeatRuns.updatedAt})`.mapWith(heartbeatRuns.updatedAt), @@ -41,10 +62,10 @@ async function nativeProviderCooldownAt(input: { }) .from(heartbeatRuns) .where(and( - eq(heartbeatRuns.companyId, input.binding.companyId), - eq(heartbeatRuns.agentId, input.binding.agentId), + eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.agentId, input.agentId), gte(heartbeatRuns.startedAt, new Date(now - NATIVE_PROVIDER_LIMIT_WINDOW_MS)), - ne(heartbeatRuns.id, input.binding.runId), + ...(input.runId ? [ne(heartbeatRuns.id, input.runId)] : []), or( eq(heartbeatRuns.status, "succeeded"), inArray(heartbeatRuns.errorCode, PROVIDER_LIMIT_ERROR_CODES), @@ -56,15 +77,22 @@ async function nativeProviderCooldownAt(input: { for (const row of rows) { // The provider accepted work after this point, so its allowance was restored. if (row.status === "succeeded") break; - const failure = readProviderFailure(row.providerFailure); - let recorded = limitCooldownAt(failure, row.observedAt); + let recorded = limitEvidence(readProviderFailure(row.providerFailure), row.observedAt, row.id); if (recorded === null && PROVIDER_LIMIT_ERROR_CODES.includes(row.errorCode ?? "")) { const reset = row.retryNotBefore ? Date.parse(row.retryNotBefore) : Number.NaN; - recorded = Number.isFinite(reset) ? reset : row.observedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS; + recorded = Number.isFinite(reset) + ? { resetAt: reset, resetReported: true, category: "quota_exhausted", sourceRunId: row.id } + : { resetAt: row.observedAt.getTime() + PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS, resetReported: false, category: "quota_exhausted", sourceRunId: row.id }; } - if (recorded !== null) cooldownAt = Math.max(cooldownAt, recorded); + if (recorded && (!latest || recorded.resetAt > latest.resetAt)) latest = recorded; } - return cooldownAt > now ? new Date(Math.min(cooldownAt, now + NATIVE_PROVIDER_LIMIT_WINDOW_MS)) : null; + if (!latest || latest.resetAt <= now) return null; + return { + resetAt: new Date(Math.min(latest.resetAt, now + NATIVE_PROVIDER_LIMIT_WINDOW_MS)), + resetReported: latest.resetReported, + category: latest.category, + sourceRunId: latest.sourceRunId, + }; } /** @@ -82,7 +110,13 @@ export async function planNativeStartupRetry(input: { }): Promise<{ nextAttemptAt: Date | null; providerCooldownAt: Date | null }> { // Limit evidence only refines the schedule. A failed read must not stop the // failure from settling, so the retry falls back to plain backoff. - const providerCooldownAt = await nativeProviderCooldownAt(input).catch((error: unknown) => { + const providerCooldownAt = await agentProviderLimitCooldown(input.db, { + companyId: input.binding.companyId, + agentId: input.binding.agentId, + runId: input.binding.runId, + failure: input.failure, + now: input.now, + }).then((cooldown) => cooldown?.resetAt ?? null, (error: unknown) => { logger.warn({ err: error, runId: input.binding.runId }, "native provider limit evidence unavailable"); return null; }); diff --git a/server/src/services/provider-quota-reset.ts b/server/src/services/provider-quota-reset.ts new file mode 100644 index 00000000000..b2d62613b93 --- /dev/null +++ b/server/src/services/provider-quota-reset.ts @@ -0,0 +1,81 @@ +/** + * Reads a provider's "try again at 4:30 PM (America/Chicago)" or + * "resets 2am (UTC)" wording as the next matching wall-clock instant after + * `now`. A clock without a time zone is read as UTC. + */ +export function parseProviderQuotaClockReset(error: string, now: Date) { + const match = error.match( + /(?:try again at|resets?(?:\s+at)?)\s+(\d{1,2})(?::(\d{2}))?\s*(?:([ap])\.?\s*m\.?)?(?:\s*\(([^)]+)\)|\s+([A-Z]{2,5}))?/i, + ); + if (!match) return null; + + const hourValue = Number.parseInt(match[1] ?? "", 10); + const minute = Number.parseInt(match[2] ?? "0", 10); + const meridiem = (match[3] ?? "").toLowerCase(); + if (!Number.isInteger(hourValue)) return null; + if ( + meridiem ? hourValue < 1 || hourValue > 12 : hourValue < 0 || hourValue > 23 + ) + return null; + if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null; + + let hour = meridiem ? hourValue % 12 : hourValue; + if (meridiem === "p") hour += 12; + const timeZone = (match[4] ?? match[5])?.trim(); + if (!timeZone) { + const retryAt = new Date(now); + retryAt.setUTCHours(hour, minute, 0, 0); + if (retryAt.getTime() <= now.getTime()) + retryAt.setUTCDate(retryAt.getUTCDate() + 1); + return retryAt; + } + + try { + const wallClock = (date: Date) => + Object.fromEntries( + new Intl.DateTimeFormat("en-US", { + timeZone, + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + }) + .formatToParts(date) + .map((part) => [part.type, part.value]), + ); + const nowParts = wallClock(now); + const buildRetryAt = (dayOffset: number) => { + const targetDay = new Date( + Date.UTC( + Number(nowParts.year), + Number(nowParts.month) - 1, + Number(nowParts.day) + dayOffset, + hour, + minute, + ), + ); + let candidate = targetDay; + const targetMs = targetDay.getTime(); + for (let attempt = 0; attempt < 4; attempt += 1) { + const actual = wallClock(candidate); + const actualMs = Date.UTC( + Number(actual.year), + Number(actual.month) - 1, + Number(actual.day), + Number(actual.hour), + Number(actual.minute), + ); + const adjustment = targetMs - actualMs; + if (adjustment === 0) break; + candidate = new Date(candidate.getTime() + adjustment); + } + return candidate; + }; + const sameDay = buildRetryAt(0); + return sameDay.getTime() > now.getTime() ? sameDay : buildRetryAt(1); + } catch { + return null; + } +} diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 31a703d60e7..971c45f4636 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -116,6 +116,7 @@ import { budgetService } from "../budgets.js"; import { unadmittedChatWakeupCondition } from "../durable-chat-wakeup.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { PROVIDER_LIMIT_DEFAULT_COOLDOWN_MS } from "../system-recovery-state.js"; +import { parseProviderQuotaClockReset } from "../provider-quota-reset.js"; import { hasUnverifiedNativeWorkspaceFinalizationOwner } from "../native-runtime/native-workspace-finalization-recovery-state.js"; import { legacyExecutionNeedsReconciliationWithEvidence, @@ -588,83 +589,6 @@ export type AdapterFailureRecoveryClassification = | { kind: "configuration_incomplete" } | null; -function parseProviderQuotaClockReset(error: string, now: Date) { - const match = error.match( - /(?:try again at|resets?(?:\s+at)?)\s+(\d{1,2})(?::(\d{2}))?\s*(?:([ap])\.?\s*m\.?)?(?:\s*\(([^)]+)\)|\s+([A-Z]{2,5}))?/i, - ); - if (!match) return null; - - const hourValue = Number.parseInt(match[1] ?? "", 10); - const minute = Number.parseInt(match[2] ?? "0", 10); - const meridiem = (match[3] ?? "").toLowerCase(); - if (!Number.isInteger(hourValue)) return null; - if ( - meridiem ? hourValue < 1 || hourValue > 12 : hourValue < 0 || hourValue > 23 - ) - return null; - if (!Number.isInteger(minute) || minute < 0 || minute > 59) return null; - - let hour = meridiem ? hourValue % 12 : hourValue; - if (meridiem === "p") hour += 12; - const timeZone = (match[4] ?? match[5])?.trim(); - if (!timeZone) { - const retryAt = new Date(now); - retryAt.setUTCHours(hour, minute, 0, 0); - if (retryAt.getTime() <= now.getTime()) - retryAt.setUTCDate(retryAt.getUTCDate() + 1); - return retryAt; - } - - try { - const wallClock = (date: Date) => - Object.fromEntries( - new Intl.DateTimeFormat("en-US", { - timeZone, - hourCycle: "h23", - year: "numeric", - month: "2-digit", - day: "2-digit", - hour: "2-digit", - minute: "2-digit", - }) - .formatToParts(date) - .map((part) => [part.type, part.value]), - ); - const nowParts = wallClock(now); - const buildRetryAt = (dayOffset: number) => { - const targetDay = new Date( - Date.UTC( - Number(nowParts.year), - Number(nowParts.month) - 1, - Number(nowParts.day) + dayOffset, - hour, - minute, - ), - ); - let candidate = targetDay; - const targetMs = targetDay.getTime(); - for (let attempt = 0; attempt < 4; attempt += 1) { - const actual = wallClock(candidate); - const actualMs = Date.UTC( - Number(actual.year), - Number(actual.month) - 1, - Number(actual.day), - Number(actual.hour), - Number(actual.minute), - ); - const adjustment = targetMs - actualMs; - if (adjustment === 0) break; - candidate = new Date(candidate.getTime() + adjustment); - } - return candidate; - }; - const sameDay = buildRetryAt(0); - return sameDay.getTime() > now.getTime() ? sameDay : buildRetryAt(1); - } catch { - return null; - } -} - export function classifyAdapterFailureForRecovery( latestRun: Pick< NonNullable,