diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 96e3ca64ffb..e840b52b187 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1011,6 +1011,17 @@ identity, unknown failures, active leases, and incomplete cleanup still require their normal reconciliation and termination proof. Unknown historical setup errors remain outside this policy, and generic recovery cannot renew its exhausted budget. +A preserved reusable sandbox that the provider refuses to resume before provider +dispatch uses the same bounded bootstrap budget (`reusable_sandbox_resume_failed`). +A common cause is a follow-up that arrives while the provider is auto-stopping an +idle sandbox. The retry inherits the source wake, so a reply is delivered once, +and it resumes the same recorded lease. A refusal is not proof that the sandbox is +lost, so the retry never destroys it or creates a replacement. Only an explicit +provider not-found or a workspace identity mismatch with a verified backup permits +replacement. The run records the refusal by closed labels only (cause label, HTTP +status, errno-style code), never by provider text. On exhaustion, the task gets a +specific recovery notice, and generic recovery does not grant another budget. + When an admitted continuation reuses a sandbox after verified cleanup, an unpublished preparation uses a directory bound to its writer epoch and lease. Earlier unpublished directories and their receipts stay intact. Repeating the diff --git a/packages/plugins/sdk/src/environment-resume-error.ts b/packages/plugins/sdk/src/environment-resume-error.ts new file mode 100644 index 00000000000..bfd5f2ed70b --- /dev/null +++ b/packages/plugins/sdk/src/environment-resume-error.ts @@ -0,0 +1,52 @@ +/** + * Diagnostic evidence for a refused `environmentResumeLease`. It crosses the + * worker RPC boundary as structured error data, because the JSON-RPC error + * carries only a message and a numeric RPC code. Never use it to authorize + * replacement or cleanup: a refusal does not prove that a sandbox is lost. + */ +export interface PluginEnvironmentResumeErrorDiagnostic { + httpStatus?: number; + errorCode?: string; +} + +const SCHEMA = "paperclip/environment-resume-error/v1"; +// Closed errno-style set. Provider text and identifiers never cross. +const ERROR_CODES = new Set([ + "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "EAI_AGAIN", "ENETUNREACH", + "ABORT_ERR", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", +]); + +function field(value: unknown, key: string): unknown { + try { return value && typeof value === "object" ? (value as Record)[key] : undefined; } + catch { return undefined; } +} + +function sanitize(input: { httpStatus?: unknown; errorCode?: unknown }): PluginEnvironmentResumeErrorDiagnostic | null { + const httpStatus = Number.isInteger(input.httpStatus) && (input.httpStatus as number) >= 100 && (input.httpStatus as number) <= 599 + ? input.httpStatus as number : undefined; + const errorCode = typeof input.errorCode === "string" && ERROR_CODES.has(input.errorCode) ? input.errorCode : undefined; + if (httpStatus === undefined && errorCode === undefined) return null; + return { ...(httpStatus !== undefined ? { httpStatus } : {}), ...(errorCode !== undefined ? { errorCode } : {}) }; +} + +/** Worker side: derive safe fields from a thrown provider error and its causes. */ +export function environmentResumeErrorData(error: unknown): { schema: string; diagnostic: PluginEnvironmentResumeErrorDiagnostic } | undefined { + let httpStatus: unknown; + let errorCode: unknown; + for (let cause = error, depth = 0; cause && depth < 4; cause = field(cause, "cause"), depth++) { + httpStatus ??= [field(cause, "status"), field(cause, "statusCode"), field(field(cause, "response"), "status")] + .find(value => Number.isInteger(value)); + const code = field(cause, "code"); + if (errorCode === undefined && typeof code === "string" && ERROR_CODES.has(code)) errorCode = code; + } + const diagnostic = sanitize({ httpStatus, errorCode }); + return diagnostic ? { schema: SCHEMA, diagnostic } : undefined; +} + +/** Host side: read and re-validate the envelope from an RPC error. */ +export function readEnvironmentResumeErrorDiagnostic(error: unknown): PluginEnvironmentResumeErrorDiagnostic | null { + const data = field(error, "data"); + if (field(data, "schema") !== SCHEMA) return null; + const diagnostic = field(data, "diagnostic"); + return sanitize({ httpStatus: field(diagnostic, "httpStatus"), errorCode: field(diagnostic, "errorCode") }); +} diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index 2c76614c96b..034a931c4c1 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -466,6 +466,8 @@ export { PluginEnvironmentRunnerIngressError, environmentRunnerIngressErrorData, export type { PluginEnvironmentRunnerIngressFailure } from "./environment-runner-ingress-error.js"; export { PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "./environment-stop-error.js"; export type { PluginEnvironmentStopFailure, PluginEnvironmentStopFailureReason } from "./environment-stop-error.js"; +export { environmentResumeErrorData, readEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; +export type { PluginEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; export type { AiConnectionPool, AiConnectionPoolConfig, AiConnectionPoolMember, AiConnectionRouterRequest, AiConnectionRouterResult, AiConnectionRouterSelection } from "@paperclipai/shared"; diff --git a/packages/plugins/sdk/src/worker-rpc-host.ts b/packages/plugins/sdk/src/worker-rpc-host.ts index ac15da62a11..cc8b212230a 100644 --- a/packages/plugins/sdk/src/worker-rpc-host.ts +++ b/packages/plugins/sdk/src/worker-rpc-host.ts @@ -8,6 +8,7 @@ import { environmentCreationCleanupErrorData } from "./environment-creation-clea import { environmentSyncErrorData, withEnvironmentSyncErrorCapture } from "./environment-sync-error.js"; import { environmentRunnerIngressErrorData } from "./environment-runner-ingress-error.js"; import { environmentStopErrorData } from "./environment-stop-error.js"; +import { environmentResumeErrorData } from "./environment-resume-error.js"; import { createInvocationTracer } from "./invocation-tracer.js"; import { createPluginIdleDrain } from "./idle-drain.js"; /** @@ -1648,7 +1649,8 @@ export function startWorkerRpcHost(options: WorkerRpcHostOptions): WorkerRpcHost ? environmentCreationCleanupErrorData(err, method === "environmentAcquireLease") : method === "environmentSyncOut" ? environmentSyncErrorData(err) : method === "environmentRunnerIngressEndpoint" ? environmentRunnerIngressErrorData(err) - : method === "environmentStopLease" ? environmentStopErrorData(err) : undefined)); + : method === "environmentStopLease" ? environmentStopErrorData(err) + : method === "environmentResumeLease" ? environmentResumeErrorData(err) : undefined)); } finally { done?.(); } diff --git a/packages/plugins/sdk/tests/environment-resume-error.test.ts b/packages/plugins/sdk/tests/environment-resume-error.test.ts new file mode 100644 index 00000000000..236454e6f41 --- /dev/null +++ b/packages/plugins/sdk/tests/environment-resume-error.test.ts @@ -0,0 +1,62 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { describe, expect, it } from "vitest"; +import { definePlugin } from "../src/define-plugin.js"; +import { environmentResumeErrorData, readEnvironmentResumeErrorDiagnostic } from "../src/environment-resume-error.js"; +import { createRequest, isJsonRpcResponse, JsonRpcCallError, parseMessage, serializeMessage, type JsonRpcResponse } from "../src/protocol.js"; +import { startWorkerRpcHost } from "../src/worker-rpc-host.js"; + +async function invoke(method: string, error: unknown) { + const stdin = new PassThrough(); + const stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const worker = startWorkerRpcHost({ stdin, stdout, plugin: definePlugin({ + async setup() {}, + async onEnvironmentResumeLease() { throw error; }, + async onEnvironmentProbe() { throw error; }, + }) }); + try { + const response = new Promise((resolve) => { + lines.on("line", (line) => { + const message = parseMessage(line); + if (isJsonRpcResponse(message)) resolve(message); + }); + }); + stdin.write(serializeMessage(createRequest(method, {}, "resume-test"))); + return await response; + } finally { + worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); + } +} + +describe("environment resume refusal diagnostics", () => { + it("carries the provider HTTP status and errno across the real resume RPC without provider text", async () => { + const providerError = Object.assign(new Error("Sandbox private-id is stopping at /home/private"), { statusCode: 409 }); + const wrapped = Object.assign(new Error("resume failed", { cause: Object.assign(providerError, { cause: { code: "ECONNRESET" } }) }), {}); + const response = await invoke("environmentResumeLease", wrapped); + if (!("error" in response) || !response.error) throw new Error("Expected RPC error"); + const callError = new JsonRpcCallError(response.error); + // The JSON-RPC error itself has only a numeric RPC code; the status needs the envelope. + expect(callError).not.toHaveProperty("status"); + expect(readEnvironmentResumeErrorDiagnostic(callError)).toEqual({ httpStatus: 409, errorCode: "ECONNRESET" }); + expect(JSON.stringify(response.error.data)).not.toContain("private"); + }); + + it("reads a response status and omits the envelope when nothing safe is known", async () => { + expect(environmentResumeErrorData({ response: { status: 503 } })).toEqual({ + schema: "paperclip/environment-resume-error/v1", diagnostic: { httpStatus: 503 } }); + const response = await invoke("environmentResumeLease", Object.assign(new Error("private"), { code: "PRIVATE_CODE", status: 42 })); + expect("error" in response && response.error).not.toHaveProperty("data"); + }); + + it("only attaches the envelope to resume calls", async () => { + const response = await invoke("environmentProbe", Object.assign(new Error("x"), { status: 409 })); + expect("error" in response && response.error).not.toHaveProperty("data"); + }); + + it.each([null, {}, { data: { schema: "other", diagnostic: { httpStatus: 409 } } }, + { data: { schema: "paperclip/environment-resume-error/v1", diagnostic: { httpStatus: 9000, errorCode: "secret" } } }])( + "rejects malformed envelopes %j", (value) => { + expect(readEnvironmentResumeErrorDiagnostic(value)).toBeNull(); + }); +}); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 29524aeb8f3..49be61a49c8 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -1,4 +1,4 @@ -import { JsonRpcCallError } from "@paperclipai/plugin-sdk"; +import { JsonRpcCallError, environmentResumeErrorData } from "@paperclipai/plugin-sdk"; import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js"; import { createSandboxOrphanCleanupSpool } from "../services/sandbox-orphan-cleanup-spool.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; @@ -3804,7 +3804,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { isRunning: vi.fn((id: string) => id === pluginId), call: vi.fn(async (_pluginId: string, method: string) => { if (method === "environmentResumeLease") { - throw new Error("stale sandbox"); + // The real worker RPC path: a message and a numeric RPC code, with the + // provider status only in the validated resume-error envelope. + throw new JsonRpcCallError({ code: -32000, message: "stale sandbox /home/daytona/private-path", + data: environmentResumeErrorData(Object.assign(new Error("private"), { statusCode: 409 })) }); } if (method === "environmentDestroyLease") { return undefined; @@ -3840,7 +3843,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { id: executionWorkspaceId, mode: "shared_workspace", }, - })).rejects.toThrow("the lease was preserved and no replacement was created"); + })).rejects.toThrow( + // The provider refusal is named by closed labels; its text stays off the record. + /could not be resumed \(cause=provider_error status=409\); the lease was preserved and no replacement was created\.$/, + ); } finally { now.mockRestore(); } diff --git a/server/src/__tests__/heartbeat-retry-scheduling.test.ts b/server/src/__tests__/heartbeat-retry-scheduling.test.ts index 6907ce47a3e..8f26377c1f6 100644 --- a/server/src/__tests__/heartbeat-retry-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-retry-scheduling.test.ts @@ -493,12 +493,13 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { return { companyId, agentId, issueId, runId, now }; } - it.each(["resume", "paused", "Stop", "dependency"] as const)("bounds fresh remote workspace retries across restart and honors %s", async action => { + it.each(["remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed"].flatMap(errorCode => + (["resume", "paused", "Stop", "dependency"] as const).map(action => [errorCode, action] as const)))("bounds %s bootstrap retries across restart and honors %s", async (errorCode, action) => { const { companyId, agentId, issueId, runId, now } = await seedMaxTurnFixture(); await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex", model: "gpt-6.1-sol" } }) .where(eq(agents.id, agentId)); const resultJson = { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }; - await db.update(heartbeatRuns).set({ errorCode: "remote_workspace_prepare_timeout", resultJson }).where(eq(heartbeatRuns.id, runId)); + await db.update(heartbeatRuns).set({ errorCode, resultJson }).where(eq(heartbeatRuns.id, runId)); let predecessor = runId; for (const attempt of [1, 2]) { const restarted = heartbeatService(db); @@ -526,7 +527,7 @@ describeEmbeddedPostgres("heartbeat bounded retry scheduling", () => { return; } predecessor = child.id; - await db.update(heartbeatRuns).set({ status: "failed", errorCode: "remote_workspace_prepare_timeout", resultJson, finishedAt: now }) + await db.update(heartbeatRuns).set({ status: "failed", errorCode, resultJson, finishedAt: now }) .where(eq(heartbeatRuns.id, predecessor)); } expect(await heartbeatService(db).scheduleBoundedRetry(predecessor, { now })).toMatchObject({ outcome: "retry_exhausted" }); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index c75a714ef37..95c70600037 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,4 +1,5 @@ import { readNativeWorkspaceMigration } from "./native-runtime/native-workspace-migration.js"; +import { ReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js"; import { HttpError } from "../errors.js"; import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; import { createOwnedSandbox, recoverSandboxCreations, recoverableTaskCreation } from "./sandbox-creation-operations.js"; @@ -706,26 +707,7 @@ export interface EnvironmentRuntimeDriver { flushDeferredOrphanCleanups?(): Promise<{ recovered: number; pending: number }>; } -/** A reusable sandbox could not be resumed, but has not been proven lost. */ -export class ReusableSandboxResumeError extends Error { - readonly provider: string; - readonly providerLeaseId: string; - - constructor(input: { - provider: string; - providerLeaseId: string; - cause?: unknown; - }) { - super( - `Reusable sandbox lease "${input.providerLeaseId}" could not be resumed; ` + - "the lease was preserved and no replacement was created.", - input.cause === undefined ? undefined : { cause: input.cause }, - ); - this.name = "ReusableSandboxResumeError"; - this.provider = input.provider; - this.providerLeaseId = input.providerLeaseId; - } -} +export { ReusableSandboxResumeError }; export class RunnerHarnessBackupUnavailableError extends Error { readonly providerLeaseId: string; diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index eda4ed90523..edc1c14aa47 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -22,6 +22,7 @@ import { } from "../execution-recovery-attempt.js"; import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "../remote-workspace-transport.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; import { randomUUID } from "node:crypto"; import { @@ -133,7 +134,7 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; export function isTransientWorkspacePreparationCode(code: string | null | undefined): boolean { return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated - || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE; + || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE; } export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 046d190f90f..6d9b086dd01 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -27,6 +27,8 @@ import { LegacyControllerLeaseLostError } from "../legacy-controller-lease.js"; import { requestedRunCancellation } from "../run-cancellation.js"; import { ProjectAccessDeniedError } from "../project-access-denial.js"; import { RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError } from "../remote-workspace-transport.js"; +import { EnvironmentRunError } from "../environment-run-orchestrator.js"; +import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js"; import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js"; @@ -926,6 +928,27 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); }); + it.each(["fresh", "started", "untrusted", "retained"] as const)("routes a %s preserved sandbox resume failure through bounded bootstrap recovery", async scenario => { + const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy" }); + const providerError = Object.assign(new Error("Sandbox abc is not in a valid state: stopping /home/daytona/secret"), { statusCode: 400 }); + const resume = new ReusableSandboxResumeError({ provider: "daytona", providerLeaseId: "lease-1", cause: providerError }); + // The orchestrator wraps the runtime error, exactly as staging recorded it. + const wrapped = new EnvironmentRunError("lease_acquire_failed", + `Failed to acquire lease for environment "Paperclip Computer" (sandbox): ${resume.message}`, { cause: resume }); + const error = scenario === "untrusted" ? new Error(wrapped.message) : wrapped; + f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + await f.completion.failRunSetup(f.setup(error, scenario === "started")); + const failed = await read(f.run.id); + expect(failed).toMatchObject({ status: "failed", errorCode: scenario === "fresh" ? "reusable_sandbox_resume_failed" : "setup_failed" }); + // The closed-label cause is recorded; provider text never is. + expect(failed.error).toContain("cause=provider_error status=400"); + expect(failed.error).not.toContain("/home/daytona"); + if (scenario === "fresh") { + expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent); + } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + }); + it("records a lease lost before dispatch as a retryable host stop, not an opaque setup failure", async () => { const f = await fixture(); f.controller.abort(new LegacyControllerLeaseLostError("renewal_rejected")); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 8ef5afb87fb..12dde035df8 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -9,6 +9,7 @@ import { NON_RETRYABLE_PREFLIGHT_FAILURE_CODES, isHeartbeatRunTerminalStatus } f import { ChatControlRecoveryUnresolvedError } from "./queue.js"; import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTransientWorkspacePreparationCode } from "./retries.js"; import { RemoteWorkspacePreparationTimeoutError } from "../remote-workspace-transport.js"; +import { findReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js"; import { readGitConnectionFailure } from "../git-connection-failure.js"; import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, ConfigurationIncompleteFailure } from "./run-preparation.js"; @@ -1355,9 +1356,15 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC const remoteWorkspacePreparationTimeout = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null && outerErr instanceof RemoteWorkspacePreparationTimeoutError ? outerErr : null; + // A preserved sandbox that could not be resumed before provider dispatch + // is retried with the same bounded bootstrap budget. The retry resumes the + // same recorded lease; it never creates a replacement or a second executor. + const reusableSandboxResumeFailure = !providerDispatchStarted && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null ? findReusableSandboxResumeError(outerErr) : null; const setupFailureErrorCode = (controllerLeaseLost ? CONTROLLER_LEASE_LOST_FAILURE_CODE : null) ?? remoteWorkspacePreparationTimeout?.code ?? + reusableSandboxResumeFailure?.code ?? workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.ts index a57a29bf7dd..bcfbc9f3655 100644 --- a/server/src/services/legacy-execution-recovery.test.ts +++ b/server/src/services/legacy-execution-recovery.test.ts @@ -10,7 +10,7 @@ const stopped = { }, }; -it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "remote_workspace_prepare_timeout"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { +it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { const run = { runtimeMode: "legacy", status: "failed", errorCode, scheduledRetryAttempt: 2, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; expect(legacyExecutionNeedsReconciliation(run)).toBe(false); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 44bdebd1004..b601258cc87 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -7,6 +7,7 @@ import { hasUnrestoredRemoteWorkspace, preserveLegacyWorkspaceRestoreSources, ty import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "./remote-workspace-transport.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js"; @@ -76,7 +77,7 @@ export function legacyExecutionNeedsReconciliation( // exhaustion needs workspace repair, not reconciliation of provider actions // that the bootstrap evidence proves never started. Keep unknown outcomes held. if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated" - || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE) && + || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || run.errorCode === REUSABLE_SANDBOX_RESUME_FAILED_CODE) && evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts index 253bf1812d0..63323ee317e 100644 --- a/server/src/services/recovery/service.pause-durability.test.ts +++ b/server/src/services/recovery/service.pause-durability.test.ts @@ -5,7 +5,7 @@ const run = (errorCode: string | null) => ({ errorCode } as unknown as Parameters[0]); describe("pause durability: continuation retry classification", () => { - it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "remote_workspace_prepare_timeout"])("does not grant %s another recovery budget", (code) => { + it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed"])("does not grant %s another recovery budget", (code) => { expect(classifyContinuationFailure(run(code))).toMatchObject({ kind: "non_retryable", maxAttempts: 0 }); }); it("agent_paused is retryable so work resumes (Option A: Resume Continues Work)", () => { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 602e98a23b0..c438c49eb5b 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -8,6 +8,7 @@ import { notifyInstructionCollection } from "../instruction-collection-notificat import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { boundRunIssueId } from "../run-task-binding.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE } from "../remote-workspace-transport.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js"; import { recoveryRunResultColumn } from "./run-result-read.js"; import { readLegacyDispositionSource } from "./legacy-disposition-read.js"; @@ -555,6 +556,7 @@ const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set([ "workspace_git_scan_output_limit", "workspace_git_scan_failed", REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, + REUSABLE_SANDBOX_RESUME_FAILED_CODE, "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", "low_trust_boundary_mismatch", diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 0ec705aa7b6..26f4ba493a9 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -50,6 +50,7 @@ const STRANDED_RECOVERY_NOTICE_TITLES_BY_RUN_ERROR_CODE: Record const WORKSPACE_SCAN_NOTICES: Record = { workspace_git_scan_timeout: { title: "Workspace scan timed out", nextAction: "Check repository access and server load, then retry the task." }, remote_workspace_prepare_timeout: { title: "Workspace setup timed out", nextAction: "Inspect the failed run and check repository access and sandbox load before retrying the task." }, + reusable_sandbox_resume_failed: { title: "Saved sandbox could not be resumed", nextAction: "Inspect the failed run and the sandbox provider; the saved sandbox was kept. Retry the task once the provider can start it." }, workspace_git_scan_saturated: { title: "Workspace scan queue is full", nextAction: "Check server load and the workspace scan queue, then retry the task." }, workspace_git_scan_output_limit: { title: "Workspace scan exceeded its limit", nextAction: "Check the repository size and workspace scan output limit before retrying the task." }, workspace_git_scan_failed: { title: "Workspace scan failed", nextAction: "Inspect the failed run and check repository access and integrity before retrying the task." }, diff --git a/server/src/services/remote-workspace-transport.ts b/server/src/services/remote-workspace-transport.ts index 57a03aafaa4..fae86b0e002 100644 --- a/server/src/services/remote-workspace-transport.ts +++ b/server/src/services/remote-workspace-transport.ts @@ -104,18 +104,33 @@ function noteProviderRefusal(operation: RemoteWorkspaceOperation, error: unknown function remoteWorkspaceProviderRefusalCause(operation: RemoteWorkspaceOperation, error: unknown): string | null { // Archive-safety and Git-metadata refusals have their own exact diagnostics. if (["restore_unsafe_archive", "restore_invalid_metadata"].includes(classifyWorkspaceRestoreFailure(error))) return null; + const { label, code } = sandboxProviderFailureLabel(error); + return `remote_workspace_operation_failed: operation=${operation} transport=${label}${code ? ` code=${code}` : ""}`; +} + +/** + * Name a sandbox provider failure by a closed label, an errno-style code, and + * an HTTP-style status. Provider text can carry sandbox paths or identifiers, + * so it only selects the label and is never copied into the result. + */ +export function sandboxProviderFailureLabel(error: unknown): { label: string; code: string | null; status: number | null } { const texts: string[] = []; let code: string | null = null; + let status: number | null = null; for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { if (cause instanceof Error) texts.push(cause.message); else if (typeof cause === "string") texts.push(cause); const candidate = (cause as { code?: unknown }).code; if (!code && typeof candidate === "string" && /^[A-Z][A-Z0-9_]{1,31}$/.test(candidate)) code = candidate; + for (const key of ["status", "statusCode"] as const) { + const value = (cause as Record)[key]; + if (status === null && Number.isInteger(value) && (value as number) >= 100 && (value as number) <= 599) status = value as number; + } } const text = texts.join("\n"); const label = TRANSPORT_FAILURE_LABELS.find(([pattern]) => pattern.test(text) || (code !== null && pattern.test(code)))?.[1] ?? "provider_error"; - return `remote_workspace_operation_failed: operation=${operation} transport=${label}${code ? ` code=${code}` : ""}`; + return { label, code, status }; } async function command(target: RemoteWorkspaceTarget, operation: RemoteWorkspaceOperation, diff --git a/server/src/services/reusable-sandbox-resume-error.ts b/server/src/services/reusable-sandbox-resume-error.ts new file mode 100644 index 00000000000..f7f507ef5e3 --- /dev/null +++ b/server/src/services/reusable-sandbox-resume-error.ts @@ -0,0 +1,52 @@ +import { readEnvironmentResumeErrorDiagnostic } from "@paperclipai/plugin-sdk"; +import { sandboxProviderFailureLabel } from "./remote-workspace-transport.js"; + +/** + * Setup failure code for a preserved reusable sandbox that could not be + * resumed before provider work began. The failure is retried with the bounded + * bootstrap budget; it never authorizes a replacement sandbox. + */ +export const REUSABLE_SANDBOX_RESUME_FAILED_CODE = "reusable_sandbox_resume_failed"; + +/** A reusable sandbox could not be resumed, but has not been proven lost. */ +export class ReusableSandboxResumeError extends Error { + readonly code = REUSABLE_SANDBOX_RESUME_FAILED_CODE; + readonly provider: string; + readonly providerLeaseId: string; + + constructor(input: { + provider: string; + providerLeaseId: string; + cause?: unknown; + }) { + // Record why the provider refused with closed labels only. The provider + // message stays on `cause` and off the run record. + const failure = input.cause === undefined ? null : sandboxProviderFailureLabel(input.cause); + // A plugin worker refusal crosses JSON-RPC with only a message and an RPC + // code; its safe status and errno travel in the validated error envelope. + const envelope = input.cause === undefined ? null : readEnvironmentResumeErrorDiagnostic(input.cause); + if (failure && envelope) { + failure.status ??= envelope.httpStatus ?? null; + failure.code ??= envelope.errorCode ?? null; + } + const detail = failure + ? ` (cause=${failure.label}${failure.status !== null ? ` status=${failure.status}` : ""}${failure.code ? ` code=${failure.code}` : ""})` + : ""; + super( + `Reusable sandbox lease "${input.providerLeaseId}" could not be resumed${detail}; ` + + "the lease was preserved and no replacement was created.", + input.cause === undefined ? undefined : { cause: input.cause }, + ); + this.name = "ReusableSandboxResumeError"; + this.provider = input.provider; + this.providerLeaseId = input.providerLeaseId; + } +} + +/** Find a resume failure through the orchestrator's error wrappers. */ +export function findReusableSandboxResumeError(error: unknown): ReusableSandboxResumeError | null { + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + if (cause instanceof ReusableSandboxResumeError) return cause; + } + return null; +}