diff --git a/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.test.ts b/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.test.ts index de525d3e462..ff114e6ceca 100644 --- a/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.test.ts +++ b/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.test.ts @@ -1,6 +1,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import type { CloudflareDriverConfig } from "./types.js"; -import { createCloudflareBridgeClient, resolveRequestTimeoutMs } from "./bridge-client.js"; +import { environmentResumeErrorData } from "@paperclipai/plugin-sdk"; +import { CloudflareBridgeTimeoutError, createCloudflareBridgeClient, resolveRequestTimeoutMs } from "./bridge-client.js"; const baseConfig: CloudflareDriverConfig = { bridgeBaseUrl: "https://bridge.example.workers.dev", @@ -89,4 +90,16 @@ describe("Cloudflare bridge client timeouts", () => { const init = fetchMock.mock.calls[0]?.[1] as RequestInit; expect(JSON.parse(String(init.body))).toMatchObject({ streamOutput: true }); }); + + it("names a bridge request deadline as a transport timeout in the resume error envelope", async () => { + vi.stubGlobal("fetch", vi.fn((_url: string, init: RequestInit) => new Promise((_resolve, reject) => { + init.signal!.addEventListener("abort", () => reject(Object.assign(new Error("aborted"), { name: "AbortError" }))); + }))); + const client = createCloudflareBridgeClient({ config: { ...baseConfig, bridgeRequestTimeoutMs: 5 } }); + const error = await client.resumeLease({ providerLeaseId: "lease-1" } as never).catch(caught => caught); + expect(error).toBeInstanceOf(CloudflareBridgeTimeoutError); + expect(error).toMatchObject({ code: "ETIMEDOUT", message: "Cloudflare sandbox bridge request timed out after 5ms." }); + expect(environmentResumeErrorData(error)).toEqual({ schema: "paperclip/environment-resume-error/v1", + diagnostic: { errorCode: "ETIMEDOUT" } }); + }); }); diff --git a/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.ts b/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.ts index d5b5c5c97f9..9498817b3c0 100644 --- a/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.ts +++ b/packages/plugins/sandbox-providers/cloudflare/src/bridge-client.ts @@ -35,6 +35,16 @@ function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } +/** A bridge request that hit its own deadline. The errno-style code lets the host retry it as a transport failure. */ +export class CloudflareBridgeTimeoutError extends Error { + readonly code = "ETIMEDOUT"; + + constructor(timeoutMs: number, cause: unknown) { + super(`Cloudflare sandbox bridge request timed out after ${timeoutMs}ms.`, { cause }); + this.name = "CloudflareBridgeTimeoutError"; + } +} + export class CloudflareBridgeError extends Error { readonly status: number; readonly code: string | null; @@ -133,9 +143,7 @@ async function requestJson( } catch (error) { if (error instanceof CloudflareBridgeError) throw error; if ((error as { name?: string } | null)?.name === "AbortError") { - throw new Error( - `Cloudflare sandbox bridge request timed out after ${requestTimeoutMs}ms.`, - ); + throw new CloudflareBridgeTimeoutError(requestTimeoutMs, error); } throw error; } finally { @@ -177,9 +185,7 @@ async function requestResponse( } catch (error) { if (error instanceof CloudflareBridgeError) throw error; if ((error as { name?: string } | null)?.name === "AbortError") { - throw new Error( - `Cloudflare sandbox bridge request timed out after ${requestTimeoutMs}ms.`, - ); + throw new CloudflareBridgeTimeoutError(requestTimeoutMs, error); } throw error; } finally { diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index ae9abc5b08c..342c202753a 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -39,7 +39,7 @@ import plugin, { STUCK_STOPPING_FORCE_AFTER_MS, } from "./plugin.js"; import type { PluginContext } from "@paperclipai/plugin-sdk"; -import { environmentAcquireErrorData, readEnvironmentAcquireErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "@paperclipai/plugin-sdk"; +import { environmentAcquireErrorData, environmentResumeErrorData, readEnvironmentAcquireErrorDiagnostic, environmentSyncErrorData, readEnvironmentSyncErrorDiagnostic, withEnvironmentSyncErrorCapture, environmentCreationCleanupErrorData, readEnvironmentCreationCleanupError, readEnvironmentAcquisitionDiagnostic, PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "@paperclipai/plugin-sdk"; import manifest from "./manifest.js"; import { parseTarVerboseListingLine, splitLinkEntryOnce } from "./file-sync.js"; import * as duplexCommands from "./duplex-command-stream.js"; @@ -5981,9 +5981,13 @@ describe("Daytona sandbox provider plugin", () => { // Idle past half of the default 15-min auto-stop interval so the next // lookup refreshes the stale handle. nowMs += 8 * 60_000; - await expect( - plugin.definition.onEnvironmentExecute?.({ ...execParams("lease-a"), config }), - ).rejects.toThrow(/did not respond within 50 ms/); + const timedOut = await Promise.resolve(plugin.definition.onEnvironmentExecute?.({ ...execParams("lease-a"), config })) + .catch((error: unknown) => error); + expect(timedOut).toBeInstanceOf(Error); + expect((timedOut as Error).message).toMatch(/did not respond within 50 ms/); + expect(timedOut).toMatchObject({ code: "ETIMEDOUT" }); + expect(environmentResumeErrorData(timedOut)).toEqual({ schema: "paperclip/environment-resume-error/v1", + diagnostic: { errorCode: "ETIMEDOUT" } }); } finally { restoreFreshness(); } diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 79c9a68768e..3306c05185c 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -591,6 +591,8 @@ function isValidUrl(value: string): boolean { // The message names the operation and the bound so an operator sees at once // that the sandbox connection is unresponsive, not that the operation is slow. class SandboxLivenessTimeoutError extends Error { + readonly code = "ETIMEDOUT"; + constructor(operation: string, timeoutMs: number) { super( `Daytona sandbox liveness call "${operation}" did not respond within ${timeoutMs} ms; ` diff --git a/packages/plugins/sdk/src/environment-resume-error.ts b/packages/plugins/sdk/src/environment-resume-error.ts index e628864e37c..51ddfcd00e4 100644 --- a/packages/plugins/sdk/src/environment-resume-error.ts +++ b/packages/plugins/sdk/src/environment-resume-error.ts @@ -16,6 +16,33 @@ const ERROR_CODES = new Set([ "ABORT_ERR", "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "CONNECTION_FAILED", ]); +/** + * Where a retry would go. `first_acquisition` may target a host that was never + * reached, so a name that does not resolve is not retried. `known_host` resumes + * an exact lease that this run already reached, so a failed lookup is an outage. + */ +export type EnvironmentRetryScope = "first_acquisition" | "known_host"; + +// The retryable subset of the envelope vocabulary. ABORT_ERR is a caller's own +// cancellation: a provider that hit its own deadline reports ETIMEDOUT instead. +const RETRYABLE_ERROR_CODES = new Set([ + "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "EAI_AGAIN", "ENETUNREACH", + "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "CONNECTION_FAILED", +]); +const KNOWN_HOST_RETRYABLE_ERROR_CODES = new Set([...RETRYABLE_ERROR_CODES, "ENOTFOUND"]); +// 501 and 505 are deterministic. Gateway and edge statuses (520-527, 529, 598, 599) are not. +const RETRYABLE_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 520, 521, 522, 523, 524, 525, 526, 527, 529, 598, 599]); + +/** Whether a provider errno is worth a bounded retry in this scope. */ +export function isRetryableEnvironmentErrorCode(code: unknown, scope: EnvironmentRetryScope = "first_acquisition"): boolean { + return typeof code === "string" && (scope === "known_host" ? KNOWN_HOST_RETRYABLE_ERROR_CODES : RETRYABLE_ERROR_CODES).has(code); +} + +/** Whether a provider HTTP status is worth a bounded retry. */ +export function isRetryableEnvironmentHttpStatus(status: unknown): boolean { + return typeof status === "number" && RETRYABLE_HTTP_STATUSES.has(status); +} + function field(value: unknown, key: string): unknown { try { return value && typeof value === "object" ? (value as Record)[key] : undefined; } catch { return undefined; } diff --git a/packages/plugins/sdk/src/index.ts b/packages/plugins/sdk/src/index.ts index a505f7d7132..7a10d0e8339 100644 --- a/packages/plugins/sdk/src/index.ts +++ b/packages/plugins/sdk/src/index.ts @@ -488,7 +488,8 @@ export { WORKSPACE_OPERATION_FAILURE_REASONS, PluginWorkspaceOperationError, wor export type { PluginWorkspaceOperationFailureReason } from "./workspace-operation-error.js"; export { PluginEnvironmentStopError, environmentStopErrorData, readEnvironmentStopFailure } from "./environment-stop-error.js"; export type { PluginEnvironmentStopFailure, PluginEnvironmentStopFailureReason } from "./environment-stop-error.js"; -export { environmentResumeErrorData, readEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; +export { environmentResumeErrorData, isRetryableEnvironmentErrorCode, isRetryableEnvironmentHttpStatus, readEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; +export type { EnvironmentRetryScope } from "./environment-resume-error.js"; export type { PluginEnvironmentResumeErrorDiagnostic } from "./environment-resume-error.js"; export { environmentAcquireErrorData, readEnvironmentAcquireErrorDiagnostic } from "./environment-acquire-error.js"; export type { PluginEnvironmentAcquireErrorDiagnostic } from "./environment-acquire-error.js"; diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index e41359333c6..d06e916d36d 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -1,4 +1,5 @@ import { RetainedSandboxAbsentError } from "../services/retained-sandbox-absent-error.js"; +import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, environmentResumeErrorData } from "@paperclipai/plugin-sdk"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { randomUUID } from "node:crypto"; import * as remoteWorkspaces from "../services/remote-workspaces.js"; @@ -1138,9 +1139,110 @@ describe("live remote runner lease reattachment", () => { const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => { throw error; }) }); await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) .rejects.toBe(error); + expect(runtime.resumeRunLease).toHaveBeenCalledOnce(); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); expect(runtime.execute).not.toHaveBeenCalled(); }); + it("retries a provider gateway refusal during reattachment and keeps the original allocation", async () => { + const savedLease = originalLease(); + mockGetLease.mockResolvedValue(savedLease); + const gateway = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: "502 Bad Gateway", + data: environmentResumeErrorData(Object.assign(new Error("gateway"), { statusCode: 502 })) }); + const resumeRunLease = vi.fn() + .mockRejectedValueOnce(gateway) + .mockResolvedValueOnce({ providerLeaseId: "original-sandbox", metadata: { remoteCwd: "/remote/workspace" } }); + const runtime = makeMockRuntime({ resumeRunLease }); + const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); + expect(resumeRunLease).toHaveBeenCalledTimes(2); + expect(result.lease).toMatchObject({ id: savedLease.id, providerLeaseId: "original-sandbox" }); + expect(result.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", providerLeaseId: "original-sandbox" }); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + }); + it("gives the first reattach resume the provider deadline and bounds every retry by the budget", async () => { + const gateway = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "web server returned an unknown error", + data: environmentResumeErrorData(Object.assign(new Error("gateway"), { statusCode: 520 })) }); + const budgetMs = 1_500; + const timeouts: Array = []; + const resumeRunLease = vi.fn(async ({ timeoutMs }: { timeoutMs?: number }) => { + timeouts.push(timeoutMs); + await new Promise(resolve => setTimeout(resolve, Math.min(timeoutMs ?? 300, 300))); + throw gateway; + }); + const runtime = makeMockRuntime({ resumeRunLease }); + const startedAt = Date.now(); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime, reattachResumeRetryBudgetMs: budgetMs }) + .acquireForRun(input)).rejects.toBe(gateway); + expect(Date.now() - startedAt).toBeLessThan(300 + budgetMs + 250); + expect(timeouts[0]).toBeUndefined(); + expect(timeouts.length).toBeGreaterThan(2); + const retries = timeouts.slice(1) as number[]; + expect(retries.every(timeout => timeout <= budgetMs)).toBe(true); + expect(retries.every((timeout, index) => index === 0 || timeout < retries[index - 1]!)).toBe(true); + }); + it("does not repeat a reattach resume that used up the provider's start deadline", async () => { + const deadline = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: "environmentResumeLease timed out" }); + const resumeRunLease = vi.fn(async () => { throw deadline; }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)).rejects.toBe(deadline); + expect(resumeRunLease).toHaveBeenCalledOnce(); + }); + it.each([500, 502, 520, 599])("treats a provider HTTP %i refusal during reattachment as transient", async statusCode => { + const savedLease = originalLease(); + mockGetLease.mockResolvedValue(savedLease); + const refusal = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "gateway", + data: environmentResumeErrorData(Object.assign(new Error("gateway"), { statusCode })) }); + const resumeRunLease = vi.fn() + .mockRejectedValueOnce(refusal) + .mockResolvedValueOnce({ providerLeaseId: "original-sandbox", metadata: { remoteCwd: "/remote/workspace" } }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) + .resolves.toMatchObject({ lease: { id: savedLease.id } }); + expect(resumeRunLease).toHaveBeenCalledTimes(2); + }); + it("retries a failed name lookup while reattaching to a host this run already reached", async () => { + const savedLease = originalLease(); + mockGetLease.mockResolvedValue(savedLease); + const lookup = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "getaddrinfo ENOTFOUND app.daytona.io", + data: environmentResumeErrorData(Object.assign(new Error("dns"), { code: "ENOTFOUND" })) }); + const resumeRunLease = vi.fn() + .mockRejectedValueOnce(lookup) + .mockResolvedValueOnce({ providerLeaseId: "original-sandbox", metadata: { remoteCwd: "/remote/workspace" } }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) + .resolves.toMatchObject({ lease: { id: savedLease.id } }); + expect(resumeRunLease).toHaveBeenCalledTimes(2); + }); + it.each([ + ["a caller's cancellation", { code: "ABORT_ERR" }], + ["HTTP 501", { statusCode: 501 }], + ])("does not retry %s during reattachment", async (_label, fields) => { + const refusal = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "refused", + data: environmentResumeErrorData(Object.assign(new Error("refused"), fields)) }); + const resumeRunLease = vi.fn(async () => { throw refusal; }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)).rejects.toBe(refusal); + expect(resumeRunLease).toHaveBeenCalledOnce(); + }); + it("does not retry a definite provider refusal during reattachment", async () => { + const refusal = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "conflict", + data: environmentResumeErrorData(Object.assign(new Error("conflict"), { statusCode: 409 })) }); + const resumeRunLease = vi.fn(async () => { throw refusal; }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)).rejects.toBe(refusal); + expect(resumeRunLease).toHaveBeenCalledOnce(); + }); + it("stops retrying a gateway refusal once the reattachment budget is spent", async () => { + const gateway = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "bad gateway", + data: environmentResumeErrorData(Object.assign(new Error("gateway"), { statusCode: 503 })) }); + const resumeRunLease = vi.fn(async () => { throw gateway; }); + const runtime = makeMockRuntime({ resumeRunLease }); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime, reattachResumeRetryBudgetMs: 1_200 }) + .acquireForRun(input)).rejects.toBe(gateway); + expect(resumeRunLease.mock.calls.length).toBeGreaterThan(1); + expect(resumeRunLease.mock.calls.length).toBeLessThan(6); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + }); it.each(["active", "foreign-company", "revoked", "undispatched", "stopped"])("inspects the original plugin task lease (%s)", async state => { mockGetEnvironment.mockResolvedValue(makeEnvironment("plugin")); const lease = originalLease(); diff --git a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts index 794a9556d1a..9cc7a91bbc0 100644 --- a/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts +++ b/server/src/__tests__/environment-runtime-reusable-resume-replacement.test.ts @@ -358,6 +358,27 @@ describeEmbeddedPostgres("reusable sandbox resume failure replacement", () => { expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentDestroyLease", expect.anything(), expect.anything()); }); + it("applies the idle-start extension to a provider liveness deadline named only by its errno", async () => { + const seeded = await seed({ leaseStatus: "released", dirty: true }); + let clock = Date.now(); + vi.spyOn(Date, "now").mockImplementation(() => clock); + const { call, runtime } = worker(seeded, async (attempt, timeoutMs) => { + if (attempt === 1) { + clock += timeoutMs; + throw new JsonRpcCallError({ code: -32000, + message: `Daytona sandbox liveness call "sandbox.refreshData" did not respond within ${timeoutMs} ms; the sandbox connection is unresponsive.`, + data: environmentResumeErrorData(Object.assign(new Error("private"), { code: "ETIMEDOUT" })) }); + } + return { providerLeaseId: seeded.providerLeaseId, metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }; + }); + + const acquired = await acquire(seeded, runtime); + + expect(acquired.lease.providerLeaseId).toBe(seeded.providerLeaseId); + expect(call.mock.calls.filter(([, method]) => method === "environmentResumeLease")).toHaveLength(2); + expect(call).not.toHaveBeenCalledWith(seeded.pluginId, "environmentAcquireLease", expect.anything(), expect.anything()); + }); + it("waits out the in-flight resume when a retry finds the stopped sandbox still settling", async () => { const seeded = await seed({ leaseStatus: "released", dirty: true }); const clock = rpcTimeoutClock(); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 0e1a32e8997..366c3b24f64 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -4217,6 +4217,18 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(await db.select().from(environmentLeases)).toHaveLength(1); }); + it("caps the exact resume call at the caller's remaining budget", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner", { timeoutMs: 300_000 }); + const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!; + const call = vi.fn(async () => ({ providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } })); + const workerManager = { isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + } as unknown as PluginWorkerManager; + await environmentRuntimeService(db, { pluginWorkerManager: workerManager }) + .resumeRunLease({ environment: seeded.environment, lease, timeoutMs: 4_321 }); + expect(call).toHaveBeenCalledWith(seeded.pluginId, "environmentResumeLease", expect.anything(), 4_321); + }); + it("forwards an explicit export directory to the selected generic plugin", async () => { const seeded = await seedReusablePluginSandboxLease(); const requestedCwd = `/workspace/.paperclip-workspaces/${seeded.executionWorkspaceId}`; diff --git a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts index 7025569275b..c25da97b566 100644 --- a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts +++ b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { and, eq } from "drizzle-orm"; import { activityLog, agents, agentWakeupRequests, authUsers, companyMemberships, environmentLeases, environments, executionWorkspaces, heartbeatRunEvents, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations } from "@paperclipai/db"; import { beforeAll, expect, it, vi } from "vitest"; +import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, environmentResumeErrorData } from "@paperclipai/plugin-sdk"; import { DurablePrpControlPlane } from "../vendor/paperclip-runner/index.js"; import * as storageBackend from "../storage/index.js"; import { getStorageService } from "../storage/index.js"; @@ -172,7 +173,7 @@ describeEmbeddedPostgres("heartbeat native remote preparation failure retention" requestedCwd: remoteCwd, // The reattach resume tells the provider the reconnect grace (RC-54). runnerReconnectGraceMs: REMOTE_RUNNER_RECONNECT_GRACE_MS, - }); + }); expect(acquireAllocation).not.toHaveBeenCalled(); expect(resumed).toHaveBeenCalledOnce(); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); @@ -198,6 +199,43 @@ describeEmbeddedPostgres("heartbeat native remote preparation failure retention" } }, 60_000); + it("keeps reattaching through a gateway refusal during a graceful hand-off and continues the same run", async () => { + const gateway = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: "\r\n502 Bad Gateway\r\n\r\n", + data: environmentResumeErrorData(Object.assign(new Error("gateway"), { statusCode: 502 })) }); + const continued = new Error("fixture stops after the reattached runner continues"); + let claimed: { db: Awaited>["db"]; runId: string } | undefined; + const reached = vi.fn(async (): Promise => { + const [run] = await claimed!.db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, claimed!.runId)); + const [owner] = await claimed!.db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, claimed!.runId)); + expect(run.status).toBe("running"); + expect(owner.controllerGeneration).toBe(2); + throw continued; + }); + let refusals = 0; + let arranged: Awaited> | undefined; + try { + arranged = await arrangeRemoteRestart("GATEWAY-DURING-HANDOFF", { + resumeLease: async ({ lease }) => { if (refusals++ === 0) throw gateway; return lease; }, + execute: reached, + }); + const { f, db, heartbeat, acquireAllocation, readyAllocation } = arranged; + claimed = { db, runId: f.runId }; + const { claims } = await heartbeat.recoverNativeRunsAfterRestart(new Date(), [f.runId]); + expect(claims).toEqual([expect.objectContaining({ kind: "reattach_remote_runner", runId: f.runId })]); + await heartbeat.drainActiveRunExecutions(); + expect(readyAllocation).toHaveBeenCalledTimes(2); + expect(acquireAllocation).not.toHaveBeenCalled(); + expect(reached).toHaveBeenCalledOnce(); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(run.error).toBe(continued.message); + expect(run.error).not.toContain("Bad Gateway"); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, f.issueId))).toHaveLength(1); + } finally { + await arranged?.restore(); + } + }, 60_000); + // S51: a user's message started a remote native turn. After a server restart, // recovery claimed the run to reattach its original sandbox, but the sandbox // allocation was gone. The restart recovery path itself must request the diff --git a/server/src/services/environment-provider-unavailable-error.test.ts b/server/src/services/environment-provider-unavailable-error.test.ts index d79f2f20fcd..2871ff9f247 100644 --- a/server/src/services/environment-provider-unavailable-error.test.ts +++ b/server/src/services/environment-provider-unavailable-error.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, readEnvironmentAcquisitionDiagnostic } from "@paperclipai/plugin-sdk"; +import { isRetryableEnvironmentErrorCode, JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, readEnvironmentAcquisitionDiagnostic } from "@paperclipai/plugin-sdk"; import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; import { EnvironmentProviderUnavailableError, @@ -13,7 +13,7 @@ function withStatus(status: number, key: "status" | "statusCode" = "status") { } describe("readTransientEnvironmentProviderFailure", () => { - it.each([408, 425, 429, 500, 502, 503, 504])("treats HTTP %i as transient", (status) => { + it.each([408, 425, 429, 500, 502, 503, 504, 520, 524, 529, 599])("treats HTTP %i as transient", (status) => { expect(readTransientEnvironmentProviderFailure(withStatus(status))).toEqual({ httpStatus: status }); }); @@ -31,7 +31,18 @@ describe("readTransientEnvironmentProviderFailure", () => { expect(readTransientEnvironmentProviderFailure(Object.assign(new Error("x"), { code }))).toBeNull(); }); - it.each([400, 401, 403, 404, 409, 422, 501])("does not retry HTTP %i", (status) => { + it.each([ + ["ENOTFOUND", false, true], + ["EAI_AGAIN", true, true], + ["ABORT_ERR", false, false], + ["EACCES", false, false], + ] as const)("scopes errno %s: first acquisition %s, reattach to a reached host %s", (code, acquisition, reattach) => { + expect(isRetryableEnvironmentErrorCode(code)).toBe(acquisition); + expect(isRetryableEnvironmentErrorCode(code, "first_acquisition")).toBe(acquisition); + expect(isRetryableEnvironmentErrorCode(code, "known_host")).toBe(reattach); + }); + + it.each([400, 401, 403, 404, 409, 422, 501, 505])("does not retry HTTP %i", (status) => { expect(readTransientEnvironmentProviderFailure(withStatus(status))).toBeNull(); }); diff --git a/server/src/services/environment-provider-unavailable-error.ts b/server/src/services/environment-provider-unavailable-error.ts index 1d4eaefa8c9..852c39b0b91 100644 --- a/server/src/services/environment-provider-unavailable-error.ts +++ b/server/src/services/environment-provider-unavailable-error.ts @@ -1,4 +1,4 @@ -import { JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, readEnvironmentAcquireErrorDiagnostic, readEnvironmentAcquisitionDiagnostic } from "@paperclipai/plugin-sdk"; +import { isRetryableEnvironmentErrorCode, isRetryableEnvironmentHttpStatus, JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, readEnvironmentAcquireErrorDiagnostic, readEnvironmentAcquisitionDiagnostic } from "@paperclipai/plugin-sdk"; import type { ProviderFailure } from "@paperclipai/shared"; import { getEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; @@ -14,14 +14,6 @@ import { getEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-d */ export const ENVIRONMENT_PROVIDER_UNAVAILABLE_CODE = "environment_provider_unavailable"; -const TRANSIENT_HTTP_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); -// Connection-level failures only. ENOTFOUND (a wrong host) and ABORT_ERR (our -// own cancellation) are excluded: retrying cannot repair either. -const TRANSIENT_ERROR_CODES = new Set([ - "ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "EAI_AGAIN", "ENETUNREACH", - "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "CONNECTION_FAILED", -]); - /** * `reason` names evidence that has neither a provider status nor a deadline: * the provider reported a provisional creation that did not finish, or a @@ -50,8 +42,8 @@ export function readTransientEnvironmentProviderFailure(error: unknown): Transie const statusValue = envelope?.httpStatus ?? [own(cause, "status"), own(cause, "statusCode")] .find(value => Number.isInteger(value)); const codeValue = envelope?.errorCode ?? own(cause, "code"); - const httpStatus = typeof statusValue === "number" && TRANSIENT_HTTP_STATUSES.has(statusValue) ? statusValue : undefined; - const errorCode = typeof codeValue === "string" && TRANSIENT_ERROR_CODES.has(codeValue) ? codeValue : undefined; + const httpStatus = typeof statusValue === "number" && isRetryableEnvironmentHttpStatus(statusValue) ? statusValue : undefined; + const errorCode = typeof codeValue === "string" && isRetryableEnvironmentErrorCode(codeValue) ? codeValue : undefined; if (httpStatus !== undefined || errorCode !== undefined) { return { ...(httpStatus !== undefined ? { httpStatus } : {}), ...(errorCode !== undefined ? { errorCode } : {}) }; } @@ -66,7 +58,7 @@ function definiteNonTransientStatus(error: unknown): boolean { for (let cause = error, depth = 0; cause && depth < 5; cause = own(cause, "cause"), depth++) { const status = readEnvironmentAcquireErrorDiagnostic(cause)?.httpStatus ?? [own(cause, "status"), own(cause, "statusCode")].find(value => Number.isInteger(value)); - if (typeof status === "number" && status >= 400 && !TRANSIENT_HTTP_STATUSES.has(status)) return true; + if (typeof status === "number" && status >= 400 && !isRetryableEnvironmentHttpStatus(status)) return true; } return false; } diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 75a28cc58ef..6b4951d63c0 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -49,6 +49,7 @@ import { type EnvironmentRuntimeService, type ProviderResourceDisposition, } from "./environment-runtime.js"; +import { REUSABLE_SANDBOX_RESUME_BUDGET_MS, retryTransientResumeCall } from "./reusable-sandbox-resume-replacement.js"; import { ENVIRONMENT_DRIVER_TRAITS } from "./environment-driver-traits.js"; import type { PluginTaskDescription } from "./environment-task-runtime.js"; import { unlaunchedPortableTaskLease } from "./plugin-task-ownership.js"; @@ -177,8 +178,10 @@ export function environmentRunOrchestrator( options: { pluginWorkerManager?: PluginWorkerManager; environmentRuntime?: EnvironmentRuntimeService; + reattachResumeRetryBudgetMs?: number; } = {}, ) { + const reattachResumeRetryBudgetMs = options.reattachResumeRetryBudgetMs ?? REUSABLE_SANDBOX_RESUME_BUDGET_MS; const environmentsSvc = environmentService(db); const executionWorkspacesSvc = executionWorkspaceService(db); const environmentRuntime = options.environmentRuntime ?? environmentRuntimeService(db, { @@ -418,11 +421,12 @@ export function environmentRunOrchestrator( // controller was offline. Ready the exact lease under its lifecycle // budget before short preparation commands (such as project-auth // checks) can run. This API cannot acquire a replacement. - const resumed = await environmentRuntime.resumeRunLease({ - environment, lease, requestedCwd: expected.remoteCwd, - ...(warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs } : {}), - ...(runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs } : {}), - }).catch((error: unknown) => { + const resumed = await retryTransientResumeCall({ budgetMs: reattachResumeRetryBudgetMs, scope: "known_host", + call: timeoutMs => environmentRuntime.resumeRunLease({ + environment, lease, requestedCwd: expected.remoteCwd, ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...(warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs } : {}), + ...(runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs } : {}), + }) }).catch((error: unknown) => { // Confirmed provider absence is a named lease mismatch; other // failures (for example a transient refusal) keep their own error. if (error instanceof RetainedSandboxAbsentError) throw remoteRecoveryLeaseMismatch("environment_resume", "lease_expired"); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 11ab4a89e3b..c8c7d501201 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -16,7 +16,11 @@ import { reusableSandboxNotKeptAfterReplacementSql, reusableSandboxReplacementPlan, type ReusableSandboxReplacementPlan, - sandboxStateTransitionFailure, + jitteredResumeRetryDelayMs, + nextResumeRetryBaseDelayMs, + RESUME_RETRY_FIRST_DELAY_MS, + resumeFailureTimedOut, + transientSandboxResumeFailure, } from "./reusable-sandbox-resume-replacement.js"; import { PluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; import { HttpError } from "../errors.js"; @@ -590,6 +594,9 @@ export interface EnvironmentDriverResumeInput extends EnvironmentDriverLeaseInpu warmRunnerIdleTimeoutMs?: number; /** The run's remote runner reconnect grace, forwarded to the provider worker only. */ runnerReconnectGraceMs?: number; + /** Upper bound for the provider resume call, so a caller's retry budget bounds a retry. + * Only the sandbox plugin driver honors it; other drivers keep their own deadline. */ + timeoutMs?: number; } export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput { @@ -802,26 +809,6 @@ function delay(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } -function transientSandboxResumeFailure(error: unknown): boolean { - const candidate = error && typeof error === "object" - ? error as Record - : {}; - const status = typeof candidate.status === "number" - ? candidate.status - : typeof candidate.statusCode === "number" - ? candidate.statusCode - : null; - if (status === 429 || (status !== null && status >= 500)) return true; - // A sandbox still stopping, archiving, or starting can be started shortly. - if (sandboxStateTransitionFailure(error)) return true; - const code = typeof candidate.code === "string" ? candidate.code.toUpperCase() : ""; - if (["ETIMEDOUT", "ECONNRESET", "ECONNREFUSED", "EAI_AGAIN", "ENETUNREACH"].includes(code)) { - return true; - } - const message = error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase(); - return /\b(timeout|timed out|rate limit|temporar|network|connection reset|service unavailable)\b/.test(message); -} - function getLeaseDriverKey( lease: Pick, environment: Pick | null, @@ -1919,7 +1906,7 @@ function createSandboxEnvironmentDriver( configuredResumeTimeoutMs, Math.min(providerStartTimeoutMs, REUSABLE_SANDBOX_IDLE_RESUME_BUDGET_MS), ); - let retryDelayMs = 250; + let retryDelayMs = RESUME_RETRY_FIRST_DELAY_MS; let idleStartWaitExtended = false; let everyResumeTimedOut = true; let resumed: PluginEnvironmentLease; @@ -1945,11 +1932,11 @@ function createSandboxEnvironmentDriver( break; } catch (error) { if (!transientSandboxResumeFailure(error)) throw error; - if (sandboxProviderFailureLabel(error).label !== "transport_timeout") everyResumeTimedOut = false; + if (!resumeFailureTimedOut(error)) everyResumeTimedOut = false; const retryAt = Date.now() + retryDelayMs * 1.25; if (retryAt >= resumeDeadline) { const waitForIdleStart = resumeDeadline < idleResumeDeadline && retryAt < idleResumeDeadline - && sandboxProviderFailureLabel(error).label === "transport_timeout" + && resumeFailureTimedOut(error) && await reusableSandboxLeaseIsIdle({ db, lease: reusableLease, companyId: input.companyId, heartbeatRunId: input.heartbeatRunId, }); @@ -1960,12 +1947,8 @@ function createSandboxEnvironmentDriver( resumeDeadline = idleResumeDeadline; idleStartWaitExtended = true; } - const jitteredDelayMs = Math.max( - 1, - Math.round(retryDelayMs * (0.75 + Math.random() * 0.5)), - ); - await delay(jitteredDelayMs); - retryDelayMs = Math.min(retryDelayMs * 2, 5_000); + await delay(jitteredResumeRetryDelayMs(retryDelayMs)); + retryDelayMs = nextResumeRetryBaseDelayMs(retryDelayMs); } } providerLease = @@ -2779,7 +2762,7 @@ function createSandboxEnvironmentDriver( issueId: input.lease.issueId, config, providerLeaseId: input.lease.providerLeaseId, leaseMetadata: input.lease.metadata ?? undefined, ...(input.requestedCwd !== undefined ? { requestedCwd: input.requestedCwd } : {}), - }, resolvePluginSandboxRpcTimeoutMs(config) ?? 60_000); + }, Math.min(resolvePluginSandboxRpcTimeoutMs(config) ?? 60_000, input.timeoutMs ?? Number.POSITIVE_INFINITY)); if (resumed?.providerLeaseId !== input.lease.providerLeaseId) { if (resumeReplyConfirmsAbsence(resumed)) throw new RetainedSandboxAbsentError(); throw new Error("The provider did not confirm the exact retained sandbox. No replacement was acquired."); diff --git a/server/src/services/reusable-sandbox-resume-replacement.ts b/server/src/services/reusable-sandbox-resume-replacement.ts index 401355a013f..8132e5c2377 100644 --- a/server/src/services/reusable-sandbox-resume-replacement.ts +++ b/server/src/services/reusable-sandbox-resume-replacement.ts @@ -1,5 +1,5 @@ import { and, eq, inArray, sql } from "drizzle-orm"; -import { readEnvironmentResumeErrorDiagnostic } from "@paperclipai/plugin-sdk"; +import { type EnvironmentRetryScope, isRetryableEnvironmentErrorCode, isRetryableEnvironmentHttpStatus, JsonRpcCallError, PLUGIN_RPC_ERROR_CODES, readEnvironmentResumeErrorDiagnostic } from "@paperclipai/plugin-sdk"; import type { Db } from "@paperclipai/db"; import { environmentLeases, executionWorkspaces, executionWorkspaceSnapshots, heartbeatRuns } from "@paperclipai/db"; import { readRemoteWorkspaceAuthority } from "@paperclipai/shared"; @@ -47,6 +47,77 @@ export function sandboxStateTransitionFailure(error: unknown): boolean { return false; } +export const RESUME_RETRY_FIRST_DELAY_MS = 250; +export const RESUME_RETRY_MAX_DELAY_MS = 5_000; + +/** The jittered wait before the next resume attempt, from the current base delay. */ +export function jitteredResumeRetryDelayMs(baseDelayMs: number): number { + return Math.max(1, Math.round(baseDelayMs * (0.75 + Math.random() * 0.5))); +} + +export function nextResumeRetryBaseDelayMs(baseDelayMs: number): number { + return Math.min(baseDelayMs * 2, RESUME_RETRY_MAX_DELAY_MS); +} + +export function isPluginRpcDeadline(error: unknown): boolean { + return error instanceof JsonRpcCallError && error.code === PLUGIN_RPC_ERROR_CODES.TIMEOUT; +} + +/** + * Whether a refused resume may pass when repeated. Only structured evidence + * counts: the provider status or errno (on the error or in the validated plugin + * envelope), the plugin RPC deadline, or a sandbox still changing state. Free + * text counts only for a few transport phrases. The status and errno policy is + * the SDK's, shared with lease acquisition; `scope` says whether this run + * already reached the host. + */ +export function transientSandboxResumeFailure(error: unknown, scope: EnvironmentRetryScope = "first_acquisition"): boolean { + const candidate = error && typeof error === "object" ? error as Record : {}; + if (isRetryableEnvironmentHttpStatus(candidate.status) || isRetryableEnvironmentHttpStatus(candidate.statusCode)) return true; + const envelope = readEnvironmentResumeErrorDiagnostic(error); + if (isRetryableEnvironmentHttpStatus(envelope?.httpStatus) || isRetryableEnvironmentErrorCode(envelope?.errorCode, scope)) return true; + if (isPluginRpcDeadline(error)) return true; + if (sandboxStateTransitionFailure(error)) return true; + if (typeof candidate.code === "string" && isRetryableEnvironmentErrorCode(candidate.code.toUpperCase(), scope)) return true; + const message = error instanceof Error ? error.message : String(error); + return /\b(rate limit(ed)?|connection reset|socket hang up|service unavailable)\b/i.test(message); +} + +/** Whether a refused resume was a transport deadline, from its failure label or its envelope errno. */ +export function resumeFailureTimedOut(error: unknown): boolean { + const code = readEnvironmentResumeErrorDiagnostic(error)?.errorCode; + return sandboxProviderFailureLabel(error).label === "transport_timeout" || code === "ETIMEDOUT" || code === "UND_ERR_CONNECT_TIMEOUT"; +} + +/** + * Resume the exact lease, repeating transient refusals. The first attempt keeps + * the provider's configured start deadline. When it ran out that deadline, the + * failure is final. Otherwise `budgetMs`, counted from the first failure, bounds + * every later wait and call. + */ +export async function retryTransientResumeCall(input: { + budgetMs: number; + scope: EnvironmentRetryScope; + call: (timeoutMs: number | undefined) => Promise; + sleep?: (ms: number) => Promise; +}): Promise { + const sleep = input.sleep ?? (ms => new Promise(resolve => setTimeout(resolve, ms))); + let deadline: number | null = null; + let baseDelayMs = RESUME_RETRY_FIRST_DELAY_MS; + while (true) { + try { + return await input.call(deadline === null ? undefined : Math.max(1, deadline - Date.now())); + } catch (error) { + if (!transientSandboxResumeFailure(error, input.scope) || (deadline === null && isPluginRpcDeadline(error))) throw error; + deadline ??= Date.now() + input.budgetMs; + const waitMs = jitteredResumeRetryDelayMs(baseDelayMs); + if (Date.now() + waitMs >= deadline) throw error; + await sleep(waitMs); + baseDelayMs = nextResumeRetryBaseDelayMs(baseDelayMs); + } + } +} + export async function reusableSandboxLeaseIsIdle(input: { db: Db; lease: EnvironmentLease; diff --git a/server/src/services/reusable-sandbox-resume-retry.test.ts b/server/src/services/reusable-sandbox-resume-retry.test.ts new file mode 100644 index 00000000000..44e65d508b0 --- /dev/null +++ b/server/src/services/reusable-sandbox-resume-retry.test.ts @@ -0,0 +1,93 @@ +import { environmentResumeErrorData, JsonRpcCallError, PLUGIN_RPC_ERROR_CODES } from "@paperclipai/plugin-sdk"; +import { describe, expect, it, vi } from "vitest"; +import { readTransientEnvironmentProviderFailure } from "./environment-provider-unavailable-error.js"; +import { retryTransientResumeCall, transientSandboxResumeFailure } from "./reusable-sandbox-resume-replacement.js"; + +const pluginRefusal = (provider: Error) => new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, + message: provider.message, data: environmentResumeErrorData(provider) }); +const withFields = (message: string, fields: Record) => Object.assign(new Error(message), fields); + +describe("transientSandboxResumeFailure", () => { + it.each([ + ["envelope HTTP 502", pluginRefusal(withFields("502 Bad Gateway", { statusCode: 502 })), true], + ["envelope HTTP 520", pluginRefusal(withFields("unknown error", { statusCode: 520 })), true], + ["envelope HTTP 429", pluginRefusal(withFields("slow down", { statusCode: 429 })), true], + ["envelope HTTP 408", pluginRefusal(withFields("request timeout", { statusCode: 408 })), true], + ["envelope HTTP 409", pluginRefusal(withFields("conflict", { statusCode: 409 })), false], + ["envelope HTTP 404", pluginRefusal(withFields("missing", { statusCode: 404 })), false], + ["envelope HTTP 501", pluginRefusal(withFields("not implemented", { statusCode: 501 })), false], + ["envelope ENOTFOUND", pluginRefusal(withFields("getaddrinfo ENOTFOUND api", { code: "ENOTFOUND" })), false], + ["envelope ECONNRESET", pluginRefusal(withFields("socket closed", { code: "ECONNRESET" })), true], + ["envelope EAI_AGAIN", pluginRefusal(withFields("dns", { code: "EAI_AGAIN" })), true], + ["envelope ABORT_ERR", pluginRefusal(withFields("aborted", { code: "ABORT_ERR" })), false], + ["envelope CONNECTION_FAILED", pluginRefusal(withFields("connect failed", { code: "CONNECTION_FAILED" })), true], + ["plugin RPC deadline", new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: "deadline" }), true], + ["plugin worker error without evidence", new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.WORKER_ERROR, message: "requested directory does not match" }), false], + ["direct status 503", withFields("unavailable", { status: 503 }), true], + ["direct errno ETIMEDOUT", withFields("connect", { code: "ETIMEDOUT" }), true], + ["direct errno ENOTFOUND", withFields("dns", { code: "ENOTFOUND" }), false], + ["sandbox state transition", new Error("Sandbox state change in progress"), true], + ["deterministic startup timeout text", new Error("retained sandbox startup timed out"), false], + ["deterministic readiness text", new Error("Sandbox failed to become ready within the timeout period"), false], + ["transport phrase", new Error("socket hang up"), true], + ["rate limit phrase", new Error("Rate limited by provider"), true], + ] as const)("classifies %s for a first acquisition", (_name, error, transient) => { + expect(transientSandboxResumeFailure(error)).toBe(transient); + }); + + it.each([ + ["envelope ENOTFOUND", pluginRefusal(withFields("dns", { code: "ENOTFOUND" })), true], + ["envelope EAI_AGAIN", pluginRefusal(withFields("dns", { code: "EAI_AGAIN" })), true], + ["direct errno ENOTFOUND", withFields("dns", { code: "ENOTFOUND" }), true], + ["envelope ABORT_ERR", pluginRefusal(withFields("aborted", { code: "ABORT_ERR" })), false], + ["envelope HTTP 501", pluginRefusal(withFields("not implemented", { statusCode: 501 })), false], + ["envelope HTTP 520", pluginRefusal(withFields("gateway", { statusCode: 520 })), true], + ] as const)("classifies %s for a reattach to a host this run already reached", (_name, error, transient) => { + expect(transientSandboxResumeFailure(error, "known_host")).toBe(transient); + }); + + it.each([ + ...[400, 401, 403, 404, 408, 409, 422, 425, 429, 500, 501, 502, 503, 504, 505, 520, 524, 529, 599].map(statusCode => ({ statusCode })), + ...["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE", "EAI_AGAIN", "ENETUNREACH", "ENOTFOUND", "ABORT_ERR", + "UND_ERR_CONNECT_TIMEOUT", "UND_ERR_SOCKET", "CONNECTION_FAILED", "EACCES"].map(code => ({ code })), + ])("agrees with the lease acquisition policy for %j", fields => { + const error = withFields("provider", fields); + expect(transientSandboxResumeFailure(error)).toBe(readTransientEnvironmentProviderFailure(error) !== null); + }); +}); + +describe("retryTransientResumeCall", () => { + it("gives the first call no cap and every retry the remaining budget", async () => { + const gateway = pluginRefusal(withFields("gateway", { statusCode: 502 })); + const timeouts: Array = []; + const call = vi.fn(async (timeoutMs: number | undefined) => { + timeouts.push(timeoutMs); + if (timeouts.length < 3) throw gateway; + return "resumed"; + }); + await expect(retryTransientResumeCall({ budgetMs: 10_000, scope: "known_host", call, sleep: async () => undefined })).resolves.toBe("resumed"); + expect(timeouts[0]).toBeUndefined(); + expect(timeouts.slice(1).every(timeout => typeof timeout === "number" && timeout <= 10_000)).toBe(true); + }); + + it("does not repeat a first call that used up the provider deadline", async () => { + const deadline = new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: "deadline" }); + const call = vi.fn(async () => { throw deadline; }); + await expect(retryTransientResumeCall({ budgetMs: 10_000, scope: "known_host", call, sleep: async () => undefined })).rejects.toBe(deadline); + expect(call).toHaveBeenCalledOnce(); + }); + + it("stops once the next wait would pass the budget", async () => { + const gateway = pluginRefusal(withFields("gateway", { statusCode: 503 })); + let now = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + const call = vi.fn(async () => { throw gateway; }); + await expect(retryTransientResumeCall({ budgetMs: 2_000, scope: "known_host", call, sleep: async ms => { now += ms; } })).rejects.toBe(gateway); + expect(now).toBeLessThan(2_000); + expect(call.mock.calls.length).toBeGreaterThan(1); + } finally { + clock.mockRestore(); + } + }); +}); diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts index 81f7d9cba1f..865754db98e 100644 --- a/tests/restart-chaos/staging-mix.test.ts +++ b/tests/restart-chaos/staging-mix.test.ts @@ -4,7 +4,7 @@ import { parseCommand } from "./config.js"; import { agentRunCaps, assignAdapters, childLine, childNonce, doneMarker, expectedRoundHash, followUpMarker, gradeStagingMix, holdsRetry, planArrivals, arrivalEndMs, followUpRequestId, planStagingMixWave, plannedTaskCount, posixCksum, retryDue, roundCommand, ROUND_SECONDS, setOverlapProbe, stagingMixDescription, - slotTimesMs, stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, + bannerShownMs, noteRetryBanner, slotTimesMs, stagingMixBudgetMs, stagingMixStep, SUBTASK_ROUNDS, verifyLedger, type StagingMixTask, } from "./staging-mix.js"; import { descendantsOf, parseMemInfo, parsePs, parseVmStat, StagingMixRun, stagingMixSeedOptions, type ResourceSample } from "./staging-mix-run.js"; import type { OracleArtifacts, TaskRecord, TaskSample } from "./types.js"; @@ -404,6 +404,22 @@ describe("staging-mix board step", () => { expect(rec.stagingMix!.retryBanner).toMatchObject({ shownMs: 19_000, visibleSince: null }); }); + it("counts a banner at most two minutes past its last sample when it was seen hidden late", () => { + const state: Parameters[0] = {}; + noteRetryBanner(state, true, 1_000, "setup_failed"); + noteRetryBanner(state, true, 273_000, "setup_failed"); + noteRetryBanner(state, false, 10_483_000, null); + expect(state.retryBanner).toMatchObject({ firstShownAt: 1_000, lastShownAt: 273_000, shownMs: 392_000, visibleSince: null }); + }); + + it("bounds a banner that is still open when sampling stopped", () => { + const state: Parameters[0] = {}; + noteRetryBanner(state, true, 1_000, "setup_failed"); + noteRetryBanner(state, true, 273_000, "setup_failed"); + expect(bannerShownMs(state.retryBanner!, 10_483_000)).toBe(392_000); + expect(bannerShownMs(state.retryBanner!, 300_000)).toBe(299_000); + }); + it("with --mix-retry-during-wave, waits for an overlap to click a Retry, and holds the observer's own click until the hold runs out", async () => { const rec = record(); const step = (r: TaskRecord, sample: TaskSample, now: number, overlap: boolean) => stepIn(r, sample, now, overlap, task({ followUpAfterMs: 60_000, retryDuringWave: true })); diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 287309d54a5..baf041cb290 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -672,10 +672,12 @@ export function latestFailureCode(sample: Pick): string | nu return failed[0]?.errorCode ?? failed[0]?.failureCause ?? null; } +export const BANNER_MAX_UNSAMPLED_MS = 2 * 60_000; + export function noteRetryBanner(state: StagingMixRecord, shown: boolean, now: number, errorCode: string | null): void { const banner = state.retryBanner; if (!shown) { - if (banner?.visibleSince != null) { banner.shownMs += now - banner.visibleSince; banner.visibleSince = null; } + if (banner?.visibleSince != null) { banner.shownMs = bannerShownMs(banner, now); banner.visibleSince = null; } return; } if (!banner) { state.retryBanner = { firstShownAt: now, lastShownAt: now, shownMs: 0, visibleSince: now, errorCode }; return; } @@ -685,7 +687,9 @@ export function noteRetryBanner(state: StagingMixRecord, shown: boolean, now: nu } export function bannerShownMs(banner: RetryBanner, now: number): number { - return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince); + if (banner.visibleSince === null) return banner.shownMs; + const until = Math.min(now, banner.lastShownAt + BANNER_MAX_UNSAMPLED_MS); + return banner.shownMs + Math.max(0, until - banner.visibleSince); } export function collectStagingMixArtifacts(record: TaskRecord, diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 70d372906e9..7154238ec2f 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -8,7 +8,7 @@ "packages/plugins/sdk/src/dev-server.ts": "2eadff81b74ca0706aec53aaf8637797fe4b4093", "packages/plugins/sdk/src/environment-acquire-error.ts": "50a74c3d1f83150f4215d44a763aa8ce4d9469c2", "packages/plugins/sdk/src/environment-creation-cleanup.ts": "35f23cd19ab0dd4148731846419a1c9b9942786b", - "packages/plugins/sdk/src/environment-resume-error.ts": "e628864e37c649ede89052e723e3a9dace352716", + "packages/plugins/sdk/src/environment-resume-error.ts": "51ddfcd00e4e8c8a8b6bb39152655926b10aacad", "packages/plugins/sdk/src/environment-runner-ingress-error.ts": "fdb47797c2a1d2df82390e42b9593d4fa4cf49e7", "packages/plugins/sdk/src/environment-stop-error.ts": "b0e03f8aa2b2170ba095da5c7138050fced3f3cb", "packages/plugins/sdk/src/environment-sync-error.ts": "7d6a51a95da49a7da4525aebbab6382dc5c44ae8", @@ -17,7 +17,7 @@ "packages/plugins/sdk/src/host-client-factory.ts": "e0301c390ea13f2340b669383f07af2455e48c8a", "packages/plugins/sdk/src/idle-drain.test.ts": "cc59f7881e7d46c89395a1d93d74f6c6b9b0a64a", "packages/plugins/sdk/src/idle-drain.ts": "ae6e4c6be4657a3d541396922b3bc551b8db557f", - "packages/plugins/sdk/src/index.ts": "a505f7d7132a406e2ccdbfd92bee0de06159bdca", + "packages/plugins/sdk/src/index.ts": "7a10d0e833956d6265250eedbb768c7971c36d09", "packages/plugins/sdk/src/invocation-tracer.ts": "b28ac4aa4bc3e82658abac4a8081ed4508008696", "packages/plugins/sdk/src/protocol.duplex-channel.test.ts": "18255eb3d63038dc459240c111cbcb29fcba25cc", "packages/plugins/sdk/src/protocol.ts": "4861d78f1c57f1485a79e037932f39e8c91446b8",