diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index afd5b740f8e..f7b7c588a76 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1105,6 +1105,19 @@ replacement. The run records the refusal by closed labels only (cause label, HTT status, errno-style code), never by provider text. On exhaustion, the task gets a specific recovery notice, and generic recovery does not grant another budget. +An environment provider that refuses a new lease with a transient status (HTTP +408, 425, 429, 500, 502, 503 or 504, or a closed connection errno) before provider +dispatch uses the same bounded bootstrap budget (`environment_provider_unavailable`). +The status comes only from structured fields: the worker's validated acquisition +envelope or the in-process error's status; message text is never parsed, and +other statuses stay `setup_failed`. Each retry is a new creation attempt with its +own durable creation record and provider name. A provisional allocation whose +cleanup could not be confirmed stays owned by its own record, which the creation +reconciler settles; a retry neither reuses nor abandons it. The task shows the +provider failure as the reason, retries automatically, and on exhaustion becomes +`blocked` with a specific recovery notice and a Retry. Because no provider work +started, no execution-reconciliation hold is created. + A saved workspace that cannot be restored into a new allocation before provider dispatch fails with `remote_workspace_restore_failed`. The run error names the server step (`snapshot_lookup`, `snapshot_download`, or `sandbox_restore`) and, diff --git a/packages/plugins/sdk/src/environment-acquire-error.ts b/packages/plugins/sdk/src/environment-acquire-error.ts new file mode 100644 index 00000000000..50a74c3d1f8 --- /dev/null +++ b/packages/plugins/sdk/src/environment-acquire-error.ts @@ -0,0 +1,48 @@ +import { PluginEnvironmentCreationCleanupError, environmentCreationCleanupErrorData } from "./environment-creation-cleanup.js"; +import { + providerErrorDiagnostic, + sanitizeProviderErrorDiagnostic, + type PluginEnvironmentResumeErrorDiagnostic, +} from "./environment-resume-error.js"; + +/** + * Diagnostic evidence for a refused `environmentAcquireLease`: the provider's + * HTTP status or a closed errno code. It crosses the worker RPC boundary as + * structured error data because a JSON-RPC error carries only a message and a + * numeric RPC code. Provider text and identifiers never cross. The host uses it + * only to decide whether a bounded retry is worthwhile; ownership of any + * provisional allocation stays with the creation-cleanup envelope. + */ +export type PluginEnvironmentAcquireErrorDiagnostic = PluginEnvironmentResumeErrorDiagnostic; + +const SCHEMA = "paperclip/environment-acquire-error/v1"; +const CLEANUP_SCHEMA = "paperclip/environment-creation-cleanup/v1"; + +function field(value: unknown, key: string): unknown { + try { return value && typeof value === "object" ? (value as Record)[key] : undefined; } + catch { return undefined; } +} + +/** + * Worker side: the RPC error data for a failed acquisition. A creation whose + * cleanup could not be confirmed keeps its ownership envelope and adds the + * status of the create call only, never of the compensating lookup or delete. + */ +export function environmentAcquireErrorData(error: unknown): unknown { + const createError = error instanceof PluginEnvironmentCreationCleanupError ? error.errors[0] : error; + const providerDiagnostic = providerErrorDiagnostic(createError); + const cleanup = environmentCreationCleanupErrorData(error, true); + if (cleanup && typeof cleanup === "object") { + return providerDiagnostic ? { ...cleanup, providerDiagnostic } : cleanup; + } + return providerDiagnostic ? { schema: SCHEMA, providerDiagnostic } : undefined; +} + +/** Host side: read and re-validate the provider diagnostic from an RPC error. */ +export function readEnvironmentAcquireErrorDiagnostic(error: unknown): PluginEnvironmentAcquireErrorDiagnostic | null { + const data = field(error, "data"); + const schema = field(data, "schema"); + if (schema !== SCHEMA && schema !== CLEANUP_SCHEMA) return null; + const diagnostic = field(data, "providerDiagnostic"); + return sanitizeProviderErrorDiagnostic({ httpStatus: field(diagnostic, "httpStatus"), errorCode: field(diagnostic, "errorCode") }); +} diff --git a/packages/plugins/sdk/src/environment-resume-error.ts b/packages/plugins/sdk/src/environment-resume-error.ts index bfd5f2ed70b..7d52cdaba31 100644 --- a/packages/plugins/sdk/src/environment-resume-error.ts +++ b/packages/plugins/sdk/src/environment-resume-error.ts @@ -21,7 +21,8 @@ function field(value: unknown, key: string): unknown { catch { return undefined; } } -function sanitize(input: { httpStatus?: unknown; errorCode?: unknown }): PluginEnvironmentResumeErrorDiagnostic | null { +/** Shared with the acquisition envelope: the same closed status/errno fields. */ +export function sanitizeProviderErrorDiagnostic(input: { httpStatus?: unknown; errorCode?: unknown }): PluginEnvironmentResumeErrorDiagnostic | null { const httpStatus = Number.isInteger(input.httpStatus) && (input.httpStatus as number) >= 100 && (input.httpStatus as number) <= 599 ? input.httpStatus as number : undefined; const errorCode = typeof input.errorCode === "string" && ERROR_CODES.has(input.errorCode) ? input.errorCode : undefined; @@ -29,8 +30,8 @@ function sanitize(input: { httpStatus?: unknown; errorCode?: unknown }): PluginE return { ...(httpStatus !== undefined ? { httpStatus } : {}), ...(errorCode !== undefined ? { errorCode } : {}) }; } -/** Worker side: derive safe fields from a thrown provider error and its causes. */ -export function environmentResumeErrorData(error: unknown): { schema: string; diagnostic: PluginEnvironmentResumeErrorDiagnostic } | undefined { +/** Derive the closed status/errno fields from a thrown provider error and its causes. */ +export function providerErrorDiagnostic(error: unknown): PluginEnvironmentResumeErrorDiagnostic | null { let httpStatus: unknown; let errorCode: unknown; for (let cause = error, depth = 0; cause && depth < 4; cause = field(cause, "cause"), depth++) { @@ -39,7 +40,12 @@ export function environmentResumeErrorData(error: unknown): { schema: string; di const code = field(cause, "code"); if (errorCode === undefined && typeof code === "string" && ERROR_CODES.has(code)) errorCode = code; } - const diagnostic = sanitize({ httpStatus, errorCode }); + return sanitizeProviderErrorDiagnostic({ httpStatus, errorCode }); +} + +/** Worker side: derive safe fields from a thrown provider error and its causes. */ +export function environmentResumeErrorData(error: unknown): { schema: string; diagnostic: PluginEnvironmentResumeErrorDiagnostic } | undefined { + const diagnostic = providerErrorDiagnostic(error); return diagnostic ? { schema: SCHEMA, diagnostic } : undefined; } @@ -48,5 +54,5 @@ export function readEnvironmentResumeErrorDiagnostic(error: unknown): PluginEnvi const data = field(error, "data"); if (field(data, "schema") !== SCHEMA) return null; const diagnostic = field(data, "diagnostic"); - return sanitize({ httpStatus: field(diagnostic, "httpStatus"), errorCode: field(diagnostic, "errorCode") }); + return sanitizeProviderErrorDiagnostic({ httpStatus: field(diagnostic, "httpStatus"), errorCode: field(diagnostic, "errorCode") }); } diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 2a655ae5603..1cf41d3241f 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -472,6 +472,8 @@ export { PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentSt export type { PluginEnvironmentStopFailure, PluginEnvironmentStopFailureReason } from "./environment-stop-error.js"; export { environmentResumeErrorData, readEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; export type { PluginEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; +export { environmentAcquireErrorData, readEnvironmentAcquireErrorDiagnostic } from "./environment-acquire-error.js"; +export type { PluginEnvironmentAcquireErrorDiagnostic } from "./environment-acquire-error.js"; export { environmentTaskErrorData, readEnvironmentTaskErrorDiagnostic } from "./environment-task-error.js"; export type { PluginEnvironmentTaskErrorDiagnostic } from "./environment-task-error.js"; diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index 91dbec177a4..ec3f1ff2aa5 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -9,6 +9,7 @@ import { environmentSyncErrorData, withEnvironmentSyncErrorCapture } from "./env import { environmentRunnerIngressErrorData } from "./environment-runner-ingress-error.js"; import { environmentStopErrorData } from "./environment-stop-error.js"; import { environmentResumeErrorData } from "./environment-resume-error.js"; +import { environmentAcquireErrorData } from "./environment-acquire-error.js"; import { environmentTaskErrorCodeData, environmentTaskErrorData } from "./environment-task-error.js"; import { workspaceOperationErrorData } from "./workspace-operation-error.js"; import { createInvocationTracer } from "./invocation-tracer.js"; @@ -1650,8 +1651,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost : PLUGIN_RPC_ERROR_CODES.WORKER_ERROR; sendMessage(createErrorResponse(id, errorCode, errorMessage, - method === "environmentAcquireLease" || method === "environmentDestroyLease" - ? environmentCreationCleanupErrorData(err, method === "environmentAcquireLease") + method === "environmentAcquireLease" ? environmentAcquireErrorData(err) + : method === "environmentDestroyLease" ? environmentCreationCleanupErrorData(err) : method === "environmentSyncOut" ? environmentSyncErrorData(err) : method === "environmentRunnerIngressEndpoint" ? environmentRunnerIngressErrorData(err) : method === "environmentStopLease" ? environmentStopErrorData(err) diff --git a/packages/plugins/sdk/tests/environment-acquire-error.test.ts b/packages/plugins/sdk/tests/environment-acquire-error.test.ts new file mode 100644 index 00000000000..5e1bae48bee --- /dev/null +++ b/packages/plugins/sdk/tests/environment-acquire-error.test.ts @@ -0,0 +1,71 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { definePlugin } from "../src/define-plugin.js"; +import { readEnvironmentAcquireErrorDiagnostic } from "../src/environment-acquire-error.js"; +import { PluginEnvironmentCreationCleanupError, readEnvironmentCreationCleanupError } from "../src/environment-creation-cleanup.js"; +import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js"; +import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; + +async function acquire(error: unknown) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({ + async setup() {}, + async onEnvironmentAcquireLease() { throw error; }, + }) }); + try { + const response = new Promise((resolve) => { + lines.on("line", (line) => { + const message = parseMessage(line); + if (isJsonRpcResponse(message)) resolve(message); + }); + }); + stdin.write(serializeMessage(createRequest("environmentAcquireLease", {}, "acquire-test"))); + const result = await response; + if (!("error" in result) || !result.error) throw new Error("Expected RPC error"); + return new JsonRpcCallError(result.error); + } finally { + worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); + } +} + +const cleanup = { + providerLeaseId: "paperclip-create-05f81d77", companyId: "company-1", environmentId: "environment-1", + attemptId: "05f81d77", accountFingerprint: "a".repeat(64), labels: { "paperclip-provider": "daytona" }, +}; + +describe("environment acquisition refusal diagnostics", () => { + it("carries a create status across the real acquire RPC without provider text", async () => { + const providerError = Object.assign(new Error("Bad gateway at https://private.example/api"), { statusCode: 502 }); + const callError = await acquire(providerError); + expect(readEnvironmentAcquireErrorDiagnostic(callError)).toEqual({ httpStatus: 502 }); + expect(JSON.stringify(callError.data)).not.toContain("private.example"); + }); + + it("reports the create status, not the compensating lookup's, and keeps the cleanup envelope", async () => { + const createError = Object.assign(new Error("create failed"), { statusCode: 502 }); + const lookupError = Object.assign(new Error("not found"), { statusCode: 404 }); + const callError = await acquire(new PluginEnvironmentCreationCleanupError([createError, lookupError], + "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-05f81d77", cleanup)); + expect(readEnvironmentAcquireErrorDiagnostic(callError)).toEqual({ httpStatus: 502 }); + // The ownership envelope that drives cleanup reconciliation is unchanged. + expect(readEnvironmentCreationCleanupError(callError)).toMatchObject({ providerLeaseId: cleanup.providerLeaseId, attemptId: cleanup.attemptId }); + }); + + it("carries a closed connection errno and omits unstructured failures", async () => { + expect(readEnvironmentAcquireErrorDiagnostic(await acquire(Object.assign(new Error("reset"), { code: "ECONNRESET" })))) + .toEqual({ errorCode: "ECONNRESET" }); + const plain = await acquire(new Error("Daytona sandbox creation failed (HTTP 502)")); + expect(plain.data).toBeUndefined(); + expect(readEnvironmentAcquireErrorDiagnostic(plain)).toBeNull(); + }); + + it("rejects forged or out-of-range envelopes", () => { + const forged = (data: unknown) => Object.assign(new Error("x"), { data }); + expect(readEnvironmentAcquireErrorDiagnostic(forged({ schema: "other", providerDiagnostic: { httpStatus: 502 } }))).toBeNull(); + expect(readEnvironmentAcquireErrorDiagnostic(forged({ schema: "paperclip/environment-acquire-error/v1", + providerDiagnostic: { httpStatus: 9000, errorCode: "EVIL" } }))).toBeNull(); + }); +}); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index 97a68fac52a..86db90d91ad 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -26,6 +26,9 @@ import { createHash, randomUUID } from "node:crypto"; import { recordLegacyWorkspaceRestoreFailure, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { issueService } from "../services/issues.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; +import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { requestTaskRetry } from "../services/task-retry.js"; import { adapterExecutionControls, createAdapterExecutionControl } from "../services/adapter-execution-control.js"; import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import fs from "node:fs/promises"; @@ -5635,6 +5638,140 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { mockAdapterExecute.mockClear(); }); + describe("transient sandbox provider refusal during lease acquisition", () => { + // Daytona answered sandbox create with HTTP 502 and the provisional + // allocation's cleanup could not be confirmed, exactly as staging recorded. + const daytonaCreate502 = () => new JsonRpcCallError({ code: -32000, + message: "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-05f81d77", + data: { schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 502 } } }); + + function failLeaseAcquisition(times: number) { + let remaining = times; + const acquisitions = { refused: 0, delegated: 0 }; + const originalFactory = environmentOrchestrator.environmentRunOrchestrator; + const factory = vi.spyOn(environmentOrchestrator, "environmentRunOrchestrator").mockImplementation((database, options = {}) => { + const runtime = originalFactory(database, options).runtime; + return originalFactory(database, { ...options, environmentRuntime: { ...runtime, acquireRunLease: async (input) => { + if (remaining > 0) { + remaining -= 1; + acquisitions.refused += 1; + throw daytonaCreate502(); + } + acquisitions.delegated += 1; + return runtime.acquireRunLease(input); + } } }); + }); + return { acquisitions, restore: () => factory.mockRestore() }; + } + + async function runsFor(agentId: string) { + return (await selectRecoveryRunsByAgent(agentId)).sort((left, right) => left.createdAt.getTime() - right.createdAt.getTime()); + } + + async function promoteLatestScheduledRetry(heartbeat: ReturnType, agentId: string) { + const scheduled = (await runsFor(agentId)).filter(run => run.status === "scheduled_retry"); + expect(scheduled).toHaveLength(1); + // Let the backoff elapse, as it would in real time, before promotion. + const due = new Date(); + await db.update(heartbeatRuns).set({ scheduledRetryAt: due }).where(eq(heartbeatRuns.id, scheduled[0]!.id)); + await heartbeat.promoteDueScheduledRetries(due); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + return scheduled[0]!; + } + + it("retries automatically and runs exactly one successful successor without user action", async () => { + const { companyId, agentId, runId, issueId } = await seedQueuedIssueRunFixture(); + const lease = failLeaseAcquisition(1); + try { + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + + const afterFailure = await runsFor(agentId); + expect(afterFailure.map(run => run.status)).toEqual(["failed", "scheduled_retry"]); + expect(afterFailure[0]).toMatchObject({ id: runId, errorCode: "environment_provider_unavailable" }); + expect(afterFailure[0]!.resultJson).toMatchObject({ providerFailure: { provider: "local", httpStatus: 502, + phase: "startup", retryability: "transient" } }); + expect(afterFailure[1]).toMatchObject({ retryOfRunId: runId, scheduledRetryAttempt: 1 }); + // The task is retrying, not blocked: no hold, and the task explains the wait. + const [waiting] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(waiting).toMatchObject({ status: "in_progress", executionRunId: afterFailure[1]!.id }); + expect(await getExecutionBlocker(db, companyId, issueId)).toBeNull(); + expect(await taskInterruptionPresentation(db, waiting!)).toMatchObject({ state: "retry_scheduled", responsibility: "system", + providerFailure: { httpStatus: 502, phase: "startup", retryability: "transient" } }); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + await promoteLatestScheduledRetry(heartbeat, agentId); + const settled = await runsFor(agentId); + // Exactly one successor of the refused run, and it did the work. Later + // runs (for example disposition repair) are ordinary follow-up wakes. + const successors = settled.filter(run => run.retryOfRunId === runId); + expect(successors).toHaveLength(1); + expect(successors[0]).toMatchObject({ id: afterFailure[1]!.id, status: "succeeded" }); + expect(settled.filter(run => run.errorCode === "environment_provider_unavailable").map(run => run.id)).toEqual([runId]); + expect(mockAdapterExecute).toHaveBeenCalled(); + expect(lease.acquisitions.refused).toBe(1); + const [done] = await db.select().from(issues).where(eq(issues.id, issueId)); + expect(done!.status).not.toBe("blocked"); + } finally { + lease.restore(); + mockAdapterExecute.mockClear(); + } + }, 60_000); + + it("explains the block with a working Retry once automatic retries are used up", async () => { + const { companyId, agentId, issueId } = await seedQueuedIssueRunFixture(); + const lease = failLeaseAcquisition(3); + try { + const heartbeat = heartbeatService(db); + await heartbeat.resumeQueuedRuns(); + await heartbeat.drainActiveRunExecutions(); + await promoteLatestScheduledRetry(heartbeat, agentId); + await promoteLatestScheduledRetry(heartbeat, agentId); + + const exhausted = await runsFor(agentId); + expect(exhausted.map(run => [run.status, run.errorCode])).toEqual([ + ["failed", "environment_provider_unavailable"], + ["failed", "environment_provider_unavailable"], + ["failed", "environment_provider_unavailable"], + ]); + expect(lease.acquisitions.refused).toBe(3); + expect(mockAdapterExecute).not.toHaveBeenCalled(); + + const blocked = await waitForValue(async () => { + const [row] = await db.select().from(issues).where(eq(issues.id, issueId)); + return row?.status === "blocked" ? row : null; + }); + expect(blocked).not.toBeNull(); + // The blocked task names its cause, its next action, and a Retry. + const presentation = await taskInterruptionPresentation(db, blocked!); + expect(presentation).toMatchObject({ state: "failed", responsibility: "system", canRetry: true, + observedRunId: exhausted[2]!.id, + providerFailure: { httpStatus: 502, phase: "startup", retryability: "transient" } }); + expect(presentation?.providerFailure?.message).toContain("temporarily unavailable"); + const notices = await db.select().from(issueComments).where(eq(issueComments.issueId, issueId)); + expect(notices.map(comment => (comment.presentation as { title?: string } | null)?.title)) + .toContain("Environment provider was temporarily unavailable"); + // No no-replay hold: provider work never started. + expect(await getExecutionBlocker(db, companyId, issueId)).toBeNull(); + + const receipt = await requestTaskRetry({ db, companyId, issueId, actorId: "responsible-user", + idempotencyKey: randomUUID(), observedRunId: presentation!.observedRunId }); + expect(receipt.accepted).toBe(true); + await heartbeat.resumeTaskRetry(companyId, receipt.requestId!); + await heartbeat.drainActiveRunExecutions(); + const retried = await waitForValue(async () => (await runsFor(agentId)).find(run => run.status === "succeeded") ?? null, 10_000); + expect(retried).not.toBeNull(); + expect(mockAdapterExecute).toHaveBeenCalledTimes(1); + expect(lease.acquisitions.delegated).toBe(1); + } finally { + lease.restore(); + mockAdapterExecute.mockClear(); + } + }, 90_000); + }); + it("classifies only the installed-but-not-ready sandbox provider plugin message as a configuration gap", () => { expect( parseSandboxProviderPluginNotReadyFailureMessage( diff --git a/server/src/__tests__/task-interruption-query.test.ts b/server/src/__tests__/task-interruption-query.test.ts index 626c59d52cc..70cb7cd8ec9 100644 --- a/server/src/__tests__/task-interruption-query.test.ts +++ b/server/src/__tests__/task-interruption-query.test.ts @@ -92,6 +92,33 @@ describe.skipIf(!support.supported)("task interruption latest run query", () => } finally { vi.unstubAllEnvs(); } }); + it("redacts a scheduled retry's predecessor provider explanation with the predecessor's registry", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + try { + const f = await seedInterruptionFixture(db, { selected: 1, history: 1, unrelated: 0, payloadBytes: 0 }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId)); + const predecessorId = randomUUID(); + const secret = `private-predecessor-value-${randomUUID()}`; + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "daytona", phase: "startup", + category: "overloaded", message: `Sandbox refused lease for ${secret}.`, httpStatus: 502, retryability: "transient" }; + await db.insert(heartbeatRuns).values({ id: predecessorId, companyId: f.companyId, agentId: run!.agentId, + status: "failed", errorCode: "environment_provider_unavailable", resultJson: { providerFailure }, + createdAt: new Date(run!.createdAt.getTime() - 60_000) }); + await db.update(heartbeatRuns).set({ status: "scheduled_retry", retryOfRunId: predecessorId, resultJson: {}, + scheduledRetryAt: new Date(Date.now() + 30_000), scheduledRetryAttempt: 1 }).where(eq(heartbeatRuns.id, run!.id)); + await createRunSecretRedactionRegistry(db).register(f.companyId, predecessorId, secret); + const list = await taskInterruptionPresentations(db, f.companyId, f.input); + const detail = await taskInterruptionPresentation(db, f.input[0]!); + for (const presentation of [list.get(f.input[0]!.id), detail]) { + expect(presentation?.observedRunId).toBe(run!.id); + expect(presentation?.providerFailure).toMatchObject({ httpStatus: 502 }); + expect(presentation?.providerFailure?.message).toContain("Sandbox refused lease"); + expect(presentation?.providerFailure?.message).toContain("REDACTED"); + expect(JSON.stringify(presentation)).not.toContain(secret); + } + } finally { vi.unstubAllEnvs(); } + }); + it.each(["decryption failure", "expanded message"])("keeps task reads available after provider display %s", async scenario => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); try { diff --git a/server/src/services/environment-provider-unavailable-error.test.ts b/server/src/services/environment-provider-unavailable-error.test.ts new file mode 100644 index 00000000000..8c42fdd31c7 --- /dev/null +++ b/server/src/services/environment-provider-unavailable-error.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { + EnvironmentProviderUnavailableError, + findEnvironmentProviderUnavailableError, + readTransientEnvironmentProviderFailure, +} from "./environment-provider-unavailable-error.js"; + +function withStatus(status: number, key: "status" | "statusCode" = "status") { + return Object.assign(new Error("provider refused"), { [key]: status }); +} + +describe("readTransientEnvironmentProviderFailure", () => { + it.each([408, 425, 429, 500, 502, 503, 504])("treats HTTP %i as transient", (status) => { + expect(readTransientEnvironmentProviderFailure(withStatus(status))).toEqual({ httpStatus: status }); + }); + + it("reads statusCode as well as status", () => { + expect(readTransientEnvironmentProviderFailure(withStatus(503, "statusCode"))).toEqual({ httpStatus: 503 }); + }); + + it.each(["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "EAI_AGAIN", "ENETUNREACH", + "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET"])("treats errno %s as transient", (code) => { + expect(readTransientEnvironmentProviderFailure(Object.assign(new Error("closed"), { code }))) + .toEqual({ errorCode: code }); + }); + + it.each(["ENOTFOUND", "ABORT_ERR", "EACCES"])("does not retry errno %s", (code) => { + expect(readTransientEnvironmentProviderFailure(Object.assign(new Error("x"), { code }))).toBeNull(); + }); + + it.each([400, 401, 403, 404, 409, 422, 501])("does not retry HTTP %i", (status) => { + expect(readTransientEnvironmentProviderFailure(withStatus(status))).toBeNull(); + }); + + it("finds transient evidence on a wrapped cause", () => { + const wrapped = new Error("outer", { cause: new Error("middle", { cause: withStatus(502) }) }); + expect(readTransientEnvironmentProviderFailure(wrapped)).toEqual({ httpStatus: 502 }); + }); + + it("stops at a definite non-transient status instead of reading deeper causes", () => { + const outer = Object.assign(new Error("bad request", { cause: withStatus(503) }), { status: 400 }); + expect(readTransientEnvironmentProviderFailure(outer)).toBeNull(); + }); + + it("reads the validated plugin RPC envelope", () => { + const rpcError = Object.assign(new Error("rpc failed"), { + data: { schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 429 } }, + }); + expect(readTransientEnvironmentProviderFailure(rpcError)).toEqual({ httpStatus: 429 }); + }); + + it("never parses message text", () => { + expect(readTransientEnvironmentProviderFailure(new Error("HTTP 502 Bad Gateway ECONNRESET"))).toBeNull(); + }); +}); + +describe("EnvironmentProviderUnavailableError", () => { + it.each([ + [{ httpStatus: 429 }, "rate_limit"], + [{ httpStatus: 408 }, "timeout"], + [{ httpStatus: 504 }, "timeout"], + [{ errorCode: "ETIMEDOUT" }, "timeout"], + [{ httpStatus: 502 }, "overloaded"], + [{ errorCode: "ECONNRESET" }, "overloaded"], + ] as const)("classifies %o as %s", (evidence, category) => { + const error = new EnvironmentProviderUnavailableError({ + environmentName: "env", driver: "daytona", evidence, cause: null, + }); + expect(error.providerFailure.category).toBe(category); + expect(error.providerFailure.retryability).toBe("transient"); + }); + + it("falls back to a generic provider label for an unsafe driver key", () => { + const error = new EnvironmentProviderUnavailableError({ + environmentName: "env", driver: "bad driver ", evidence: { httpStatus: 502 }, cause: null, + }); + expect(error.providerFailure.provider).toBe("environment"); + }); + + it("is found through wrapper causes", () => { + const inner = new EnvironmentProviderUnavailableError({ + environmentName: "env", driver: "ssh", evidence: { errorCode: "ECONNREFUSED" }, cause: null, + }); + expect(findEnvironmentProviderUnavailableError(new Error("outer", { cause: inner }))).toBe(inner); + expect(findEnvironmentProviderUnavailableError(new Error("plain"))).toBeNull(); + }); +}); diff --git a/server/src/services/environment-provider-unavailable-error.ts b/server/src/services/environment-provider-unavailable-error.ts new file mode 100644 index 00000000000..f6f0357db9b --- /dev/null +++ b/server/src/services/environment-provider-unavailable-error.ts @@ -0,0 +1,94 @@ +import { readEnvironmentAcquireErrorDiagnostic } from "@paperclipai/plugin-sdk"; +import type { ProviderFailure } from "@paperclipai/shared"; + +/** + * Setup failure code for a lease acquisition that the environment provider + * refused with a transient status (HTTP 408/425/429/5xx or a connection-level + * errno) before any agent work started. The run is retried with the bounded + * bootstrap budget. A provisional allocation whose cleanup could not be + * confirmed stays owned by its own creation record, which the cleanup + * reconciler settles independently; every retry is a new creation attempt + * with its own idempotent identity, so it can neither reuse nor leak it. + */ +export const ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE = "environment_provider_unavailable"; + +const TRANSIENT_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); +// Connection-level failures only. ENOTFOUND (a wrong host) and ABORT_ERR (our +// own cancellation) are excluded: retrying cannot repair either. +const TRANSIENT_ERROR_CODES = new Set([ + "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "EAI_AGAIN", "ENETUNREACH", + "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", +]); + +type TransientProviderEvidence = { httpStatus?: number; errorCode?: string }; + +function own(value: unknown, key: string): unknown { + try { + return value && typeof value === "object" ? Object.getOwnPropertyDescriptor(value, key)?.value : undefined; + } catch { return undefined; } +} + +/** + * Read transient provider evidence from structured fields only: the validated + * plugin RPC envelope, or the status/errno an in-process provider attached to + * its error. Message text is never parsed. + */ +export function readTransientEnvironmentProviderFailure(error: unknown): TransientProviderEvidence | null { + for (let cause = error, depth = 0; cause && depth < 5; cause = own(cause, "cause"), depth++) { + const envelope = readEnvironmentAcquireErrorDiagnostic(cause); + const statusValue = envelope?.httpStatus ?? [own(cause, "status"), own(cause, "statusCode")] + .find(value => Number.isInteger(value)); + const codeValue = envelope?.errorCode ?? own(cause, "code"); + const httpStatus = typeof statusValue === "number" && TRANSIENT_HTTP_STATUSES.has(statusValue) ? statusValue : undefined; + const errorCode = typeof codeValue === "string" && TRANSIENT_ERROR_CODES.has(codeValue) ? codeValue : undefined; + if (httpStatus !== undefined || errorCode !== undefined) { + return { ...(httpStatus !== undefined ? { httpStatus } : {}), ...(errorCode !== undefined ? { errorCode } : {}) }; + } + // A definite non-transient status is authoritative for this layer. + if (typeof statusValue === "number" && statusValue >= 400) return null; + } + return null; +} + +function category(evidence: TransientProviderEvidence): ProviderFailure["category"] { + if (evidence.httpStatus === 429) return "rate_limit"; + if (evidence.httpStatus === 408 || evidence.httpStatus === 504 + || evidence.errorCode === "ETIMEDOUT" || evidence.errorCode === "UND_ERR_CONNECT_TIMEOUT") return "timeout"; + return "overloaded"; +} + +/** A lease acquisition the environment provider refused transiently. */ +export class EnvironmentProviderUnavailableError extends Error { + readonly code = ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE; + readonly httpStatus: number | null; + readonly errorCode: string | null; + /** Read by the setup-failure path and shown on the task as the reason. */ + readonly providerFailure: ProviderFailure; + + constructor(input: { environmentName: string; driver: string; evidence: TransientProviderEvidence; cause: unknown }) { + const status = input.evidence.httpStatus !== undefined ? `HTTP ${input.evidence.httpStatus}` : input.evidence.errorCode!; + super(`Environment provider for "${input.environmentName}" was temporarily unavailable (${status}).`, { cause: input.cause }); + this.name = "EnvironmentProviderUnavailableError"; + this.httpStatus = input.evidence.httpStatus ?? null; + this.errorCode = input.evidence.errorCode ?? null; + this.providerFailure = { + schema: "paperclip.provider-failure.v1", + provider: /^[a-zA-Z0-9_.:/-]{1,80}$/.test(input.driver) ? input.driver : "environment", + phase: "startup", + category: category(input.evidence), + message: `The provider for "${input.environmentName}" was temporarily unavailable while Paperclip was starting it (${status}). ` + + "No agent work had started. Paperclip retries automatically; if those retries are used up, use Retry once the provider is available.", + ...(input.evidence.httpStatus !== undefined ? { httpStatus: input.evidence.httpStatus } : {}), + ...(input.evidence.errorCode !== undefined ? { providerCode: input.evidence.errorCode } : {}), + retryability: "transient", + }; + } +} + +/** Find a transient provider refusal through the orchestrator's error wrappers. */ +export function findEnvironmentProviderUnavailableError(error: unknown): EnvironmentProviderUnavailableError | null { + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + if (cause instanceof EnvironmentProviderUnavailableError) return cause; + } + return null; +} diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index f8fd7da5fb7..0f03cc2f343 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -20,6 +20,9 @@ import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRe import { environmentLeases, type Db } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import { remoteExecutionHasStopped } from "./remote-execution-termination.js"; +import { EnvironmentProviderUnavailableError, readTransientEnvironmentProviderFailure } from "./environment-provider-unavailable-error.js"; +import { findReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js"; +import { findPluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; import type { Environment, EnvironmentLease, @@ -222,13 +225,21 @@ export function environmentRunOrchestrator( try { return await environmentRuntime.acquireRunLease(input); } catch (err) { + // A transient provider refusal (HTTP 5xx/429/timeout) is retried by the + // setup path with its bounded bootstrap budget. Saved-sandbox resume and + // stopped-worker refusals keep their own, more specific classifications. + const transient = findReusableSandboxResumeError(err) || findPluginWorkerUnavailableError(err) + ? null : readTransientEnvironmentProviderFailure(err); throw new EnvironmentRunError( "lease_acquire_failed", `Failed to acquire lease for environment "${input.environment.name}" (${input.environment.driver}): ${err instanceof Error ? err.message : String(err)}`, { environmentId: input.environment.id, driver: input.environment.driver, - cause: err, + cause: transient + ? new EnvironmentProviderUnavailableError({ environmentName: input.environment.name, + driver: input.environment.driver, evidence: transient, cause: err }) + : err, }, ); } diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 122a1c63787..8ca56a7a8ba 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -24,6 +24,7 @@ import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-sched import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "../remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; import { RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY, RUNNER_REVISION_WAIT_RETRY_REASON } from "../runner-revision-mismatch.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; import { randomUUID } from "node:crypto"; @@ -138,7 +139,7 @@ export function isTransientWorkspacePreparationCode(code: string | null | undefi return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE || code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE - || code === RUNNER_REVISION_MISMATCH_CODE; + || code === RUNNER_REVISION_MISMATCH_CODE || code === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE; } export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = @@ -352,6 +353,7 @@ function isRetryableInteractionContinuationInfrastructureFailure( run.errorCode === "process_lost" || run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE || run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE || + run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE || run.errorCode === RUNNER_REVISION_MISMATCH_CODE ) { return true; diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 2d96868fcbe..25a37db2eec 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -27,7 +27,7 @@ import { LegacyControllerLeaseLostError } from "../legacy-controller-lease.js"; import { requestedRunCancellation } from "../run-cancellation.js"; import { ProjectAccessDeniedError } from "../project-access-denial.js"; import { RemoteWorkspaceOperationError, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, RemoteWorkspaceRestoreError } from "../remote-workspace-transport.js"; -import { EnvironmentRunError } from "../environment-run-orchestrator.js"; +import { EnvironmentRunError, environmentRunOrchestrator } from "../environment-run-orchestrator.js"; import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { PluginWorkerUnavailableError } from "../plugin-worker-unavailable-error.js"; import { environmentTaskErrorCodeData, JsonRpcCallError, PluginEnvironmentTaskError, PluginWorkspaceOperationError, workspaceOperationErrorData } from "@paperclipai/plugin-sdk"; @@ -971,6 +971,44 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); }); + // Daytona returned HTTP 502 on sandbox create and cleanup of the + // provisional allocation could not be confirmed. The orchestrator wraps the + // worker refusal exactly as staging recorded it. + const daytonaCreate502 = (withEnvelope = true) => new JsonRpcCallError({ code: -32000, + message: "Daytona sandbox creation failed (HTTP 502); cleanup could not be confirmed for paperclip-create-05f81d77", + data: withEnvelope ? { schema: "paperclip/environment-creation-cleanup/v1", providerDiagnostic: { httpStatus: 502 }, + cleanup: { providerLeaseId: "paperclip-create-05f81d77", companyId: "company", environmentId: "environment", + attemptId: "05f81d77", accountFingerprint: "a".repeat(64), labels: {} } } : undefined }); + const acquireThroughOrchestrator = async (refusal: unknown) => { + const orchestrator = environmentRunOrchestrator(db, { environmentRuntime: { + acquireRunLease: async () => { throw refusal; }, + } as never }); + return orchestrator.acquireLease({ companyId: randomUUID(), environment: { id: randomUUID(), name: "Paperclip Sandbox", driver: "sandbox" } as never, + issueId: null, agentId: randomUUID(), heartbeatRunId: randomUUID(), persistedExecutionWorkspace: null, + executionWorkspaceSettings: null, adapterType: null }).then(() => { throw new Error("expected a refusal"); }, (error: unknown) => error); + }; + + it.each(["fresh", "started", "message_only", "permanent", "retained"] as const)("routes a %s sandbox provider lease refusal through bounded bootstrap recovery", async scenario => { + const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy" }); + const refusal = scenario === "message_only" ? daytonaCreate502(false) + : scenario === "permanent" ? new JsonRpcCallError({ code: -32000, message: "Daytona sandbox creation failed (HTTP 400)", + data: { schema: "paperclip/environment-acquire-error/v1", providerDiagnostic: { httpStatus: 400 } } }) + : daytonaCreate502(); + const error = await acquireThroughOrchestrator(refusal); + f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + await f.completion.failRunSetup(f.setup(error, scenario === "started")); + const failed = await read(f.run.id); + const retried = scenario === "fresh"; + expect(failed).toMatchObject({ status: "failed", errorCode: retried ? "environment_provider_unavailable" : "setup_failed" }); + // The staging message is kept verbatim for diagnostics. + expect(failed.error).toContain("Daytona sandbox creation failed"); + if (retried) { + expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false }, + providerFailure: { provider: "sandbox", phase: "startup", category: "overloaded", httpStatus: 502, retryability: "transient" } }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent); + } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + }); + const runnerMismatch = () => new JsonRpcCallError({ code: -32002, message: "The installed Runner does not match the expected revision", data: environmentTaskErrorCodeData(new PluginEnvironmentTaskError("runner_revision_mismatch")) }); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 84167bae1f6..bb51d76c55a 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -14,6 +14,7 @@ import { RemoteWorkspacePreparationTimeoutError, REMOTE_WORKSPACE_RESTORE_FAILED import { escalateRemoteWorkspaceRestoreFailure } from "../remote-workspace-restore-recovery.js"; import { findReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { findPluginWorkerUnavailableError, PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, findEnvironmentProviderUnavailableError } from "../environment-provider-unavailable-error.js"; import { findWorkspaceProviderFailure } from "../workspace-provider-failure.js"; import { isRunnerRevisionMismatch, RUNNER_REVISION_MISMATCH_CODE, RUNNER_REVISION_MISMATCH_RETRY } from "../runner-revision-mismatch.js"; import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js"; @@ -1380,6 +1381,12 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // bootstrap budget; the worker is restarted or replaced meanwhile. const pluginWorkerUnavailableFailure = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null ? findPluginWorkerUnavailableError(outerErr) : null; + // The environment provider refused lease acquisition with a transient + // status (HTTP 5xx/429/timeout) before dispatch. Each retry is a new, + // separately owned creation attempt; an unconfirmed provisional allocation + // stays with its creation record and the cleanup reconciler. + const environmentProviderUnavailable = !providerDispatchStarted && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null ? findEnvironmentProviderUnavailableError(outerErr) : null; const runnerRevisionMismatch = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null && isRunnerRevisionMismatch(outerErr); const setupFailureErrorCode = @@ -1388,6 +1395,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC reusableSandboxResumeFailure?.code ?? remoteWorkspaceRestoreFailure?.code ?? pluginWorkerUnavailableFailure?.code ?? + environmentProviderUnavailable?.code ?? (runnerRevisionMismatch ? RUNNER_REVISION_MISMATCH_CODE : null) ?? workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? @@ -1552,7 +1560,8 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC } else if (livenessRun.errorCode !== PROJECT_ACCESS_DENIED) { // An accepted interaction keeps its own continuation retry, which // carries the interaction and escalates on exhaustion. - const interactionOwnsRetry = livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE + const interactionOwnsRetry = (livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE + || livenessRun.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE) && isResolvedInteractionContinuationWakeContext(livenessRun.contextSnapshot); await (isTransientWorkspacePreparationCode(livenessRun.errorCode) && !interactionOwnsRetry ? livenessRun.errorCode === RUNNER_REVISION_MISMATCH_CODE diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 0b3255ae97a..2fa69a533d3 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -9,6 +9,7 @@ import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wai import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "./remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "./plugin-worker-unavailable-error.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "./environment-provider-unavailable-error.js"; import { RUNNER_REVISION_MISMATCH_CODE } from "./runner-revision-mismatch.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; @@ -84,7 +85,7 @@ export function legacyExecutionNeedsReconciliation( if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated" || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || run.errorCode === REUSABLE_SANDBOX_RESUME_FAILED_CODE || run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE - || run.errorCode === RUNNER_REVISION_MISMATCH_CODE) && + || run.errorCode === RUNNER_REVISION_MISMATCH_CODE || run.errorCode === ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE) && evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 87fca0f9de0..2f0faafc871 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -11,6 +11,7 @@ import { boundRunIssueId } from "../run-task-binding.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "../remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; +import { ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE } from "../environment-provider-unavailable-error.js"; import { RUNNER_REVISION_MISMATCH_CODE } from "../runner-revision-mismatch.js"; import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js"; import { recoveryRunResultColumn } from "./run-result-read.js"; @@ -563,6 +564,7 @@ const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set([ REUSABLE_SANDBOX_RESUME_FAILED_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE, PLUGIN_WORKER_UNAVAILABLE_CODE, + ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE, RUNNER_REVISION_MISMATCH_CODE, "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index d6f731b9f4e..694a6689aef 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -57,6 +57,7 @@ const WORKSPACE_SCAN_NOTICES: Record issue.companyId === companyId); @@ -196,6 +198,11 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i or ${heartbeatRuns.errorCode} = 'workspace_validation_failed' then ${heartbeatRuns.error} else null end`, errorCode: heartbeatRuns.errorCode, livenessState: heartbeatRuns.livenessState, scheduledRetryAt: heartbeatRuns.scheduledRetryAt, scheduledRetryReason: heartbeatRuns.scheduledRetryReason, + // The run whose resultJson supplied providerFailure. A scheduled retry + // shows its predecessor's reason, so redaction must use that run's registry. + providerFailureRunId: sql`case when ${heartbeatRuns.resultJson}->'providerFailure' is null + and ${heartbeatRuns.status} = 'scheduled_retry' and ${heartbeatRuns.retryOfRunId} is not null + then ${heartbeatRuns.retryOfRunId} else ${heartbeatRuns.id} end`, resultJson: sql>`jsonb_build_object('configurationIncomplete', ${heartbeatRuns.resultJson}->'configurationIncomplete', 'stopReason', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'stopReason'`, 80)}, 'timeoutSource', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'timeoutSource'`, 80)}, @@ -214,7 +221,9 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i 'dispatched', coalesce(${heartbeatRuns.resultJson}->'nativeCancellation'->'dispatched' = 'true'::jsonb, false), 'intentAuditId', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'nativeCancellation'->'intentAuditId'`, 160)}, 'acknowledgementAuditId', ${boundedJsonString(sql`${heartbeatRuns.resultJson}->'nativeCancellation'->'acknowledgementAuditId'`, 160)}), - 'providerFailure', ${heartbeatRuns.resultJson}->'providerFailure')`, + 'providerFailure', coalesce(${heartbeatRuns.resultJson}->'providerFailure', case when ${heartbeatRuns.status} = 'scheduled_retry' + then (select predecessor.result_json->'providerFailure' from ${heartbeatRuns} predecessor + where predecessor.company_id = ${heartbeatRuns.companyId} and predecessor.id = ${heartbeatRuns.retryOfRunId}) end))`, }).from(requested).innerJoinLateral(latestRuns, sql`true`).innerJoin(heartbeatRuns, and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, latestRuns.id))) .leftJoin(agents, and(eq(agents.companyId, heartbeatRuns.companyId), eq(agents.id, heartbeatRuns.agentId))) .orderBy(requestedIssueId), @@ -309,16 +318,24 @@ export async function taskInterruptionPresentations(db: Db, companyId: string, i agentAdapter: run.agentAdapter ?? "unknown" }); } } - const disclosures = [...result.values()].flatMap(presentation => (presentation?.providerFailure || presentation?.workspaceFailure) && presentation.observedRunId - ? [{ id: presentation.observedRunId, providerFailure: presentation.providerFailure, - workspaceFailure: presentation.workspaceFailure, presentation }] : []); + // Redact each explanation with the registry of the run that recorded it: the + // workspace failure is the observed run's, while a scheduled retry's provider + // failure may be its predecessor's. + const disclosures = [...result.entries()].flatMap(([issueId, presentation]) => + (presentation?.providerFailure || presentation?.workspaceFailure) && presentation.observedRunId + ? [{ id: presentation.observedRunId, providerFailureRunId: runMap.get(issueId)?.providerFailureRunId ?? presentation.observedRunId, + providerFailure: presentation.providerFailure, workspaceFailure: presentation.workspaceFailure, presentation }] : []); if (disclosures.length) { try { - const [safe, settings] = await Promise.all([ - createRunSecretRedactionRegistry(db).redactForRuns(companyId, - disclosures.map(({ id, providerFailure, workspaceFailure }) => ({ id, providerFailure, workspaceFailure }))), + const [redacted, settings] = await Promise.all([ + createRunSecretRedactionRegistry(db).redactForRuns(companyId, disclosures.flatMap(({ id, providerFailureRunId, providerFailure, workspaceFailure }) => [ + { id: providerFailureRunId, providerFailure, workspaceFailure: undefined }, + { id, providerFailure: undefined, workspaceFailure }, + ])), instanceSettingsService(db).getGeneral(), ]); + const safe = disclosures.map((_, index) => ({ providerFailure: redacted[2 * index]?.providerFailure, + workspaceFailure: redacted[2 * index + 1]?.workspaceFailure })); const safeMessage = (value: unknown) => typeof value === "string" ? [...redactCurrentUserText(value, { enabled: settings.censorUsernameInLogs })].slice(0, 4000).join("") : ""; disclosures.forEach(({ presentation }, index) => { diff --git a/ui/src/components/ProviderFailureNotice.tsx b/ui/src/components/ProviderFailureNotice.tsx index 9ad8fa86d08..03198ea690b 100644 --- a/ui/src/components/ProviderFailureNotice.tsx +++ b/ui/src/components/ProviderFailureNotice.tsx @@ -12,7 +12,7 @@ const labels: Record = { export function ProviderFailureNotice({ failure, actions, children, className }: { failure: ProviderFailure; actions?: ReactNode; children?: ReactNode; className?: string; }) { - const provider = ({ anthropic: "Claude", openai: "OpenAI" } as Record)[failure.provider] ?? failure.provider; + const provider = ({ anthropic: "Claude", openai: "OpenAI", sandbox: "Sandbox provider" } as Record)[failure.provider] ?? failure.provider; return (