diff --git a/server/src/services/__tests__/run-failure-report-sandbox-resume.test.ts b/server/src/services/__tests__/run-failure-report-sandbox-resume.test.ts new file mode 100644 index 00000000000..6544d1a5baf --- /dev/null +++ b/server/src/services/__tests__/run-failure-report-sandbox-resume.test.ts @@ -0,0 +1,131 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRuns } from "@paperclipai/db"; +import { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, +} from "../../__tests__/helpers/embedded-postgres.js"; + +const mockCaptureRunFailure = vi.hoisted(() => vi.fn()); + +vi.mock("../../sentry.js", () => ({ + captureRunFailure: mockCaptureRunFailure, +})); + +import { environmentResumeErrorData, JsonRpcCallError } from "@paperclipai/plugin-sdk"; +import { reportRunFailure } from "../run-failure-report.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE, ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; + +const testDatabaseUrl = process.env.PAPERCLIP_TEST_DATABASE_URL; +const embeddedPostgresSupport = testDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); +const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; + +describeEmbeddedPostgres("reportRunFailure sandbox resume provider outages", () => { + let db!: ReturnType; + let tempDb: Awaited> | null = null; + let companyId!: string; + let agentId!: string; + let inheritedEnv: NodeJS.ProcessEnv; + + beforeAll(async () => { + if (testDatabaseUrl) { + await applyPendingMigrations(testDatabaseUrl); + db = createDb(testDatabaseUrl); + } else { + tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-failure-sandbox-resume-"); + db = createDb(tempDb.connectionString); + } + }, 20_000); + + beforeEach(async () => { + inheritedEnv = process.env; + process.env = Object.fromEntries( + ["PATH", "HOME", "USER", "USERNAME", "LOGNAME", "USERPROFILE", "TMPDIR", "TEMP", "TMP"] + .flatMap((key) => inheritedEnv[key] === undefined ? [] : [[key, inheritedEnv[key]]]), + ); + process.env.PAPERCLIP_SENTRY_RUN_FAILURES = "unexpected"; + companyId = randomUUID(); + agentId = randomUUID(); + await db.insert(companies).values({ + id: companyId, + name: "Paperclip", + issuePrefix: `T${companyId.replace(/-/g, "").slice(0, 6).toUpperCase()}`, + requireBoardApprovalForNewAgents: false, + defaultResponsibleUserId: "responsible-user", + }); + await db.insert(agents).values({ + id: agentId, + companyId, + name: "SandboxBot", + role: "engineer", + status: "active", + adapterType: "paperclip_runner", + adapterConfig: {}, + runtimeConfig: {}, + permissions: {}, + }); + }); + + afterEach(async () => { + vi.clearAllMocks(); + vi.unstubAllEnvs(); + process.env = inheritedEnv; + await db.delete(heartbeatRuns); + await db.delete(agents); + await db.delete(companies); + }); + + afterAll(async () => { + if (testDatabaseUrl) await closeRegisteredClients(testDatabaseUrl); + await tempDb?.cleanup(); + }); + + // The sandbox provider answered the saved-sandbox resume with a gateway + // page. The plugin worker sends only the closed status across JSON-RPC. + const providerOutage = () => new ReusableSandboxResumeError({ provider: "daytona", providerLeaseId: "lease-1", + cause: new JsonRpcCallError({ code: -32002, message: "502 Bad Gateway", + data: environmentResumeErrorData({ status: 502 }) }) }); + + function resumeFailureRun( + overrides: Partial = {}, + ): typeof heartbeatRuns.$inferSelect { + return { + id: randomUUID(), + companyId, + agentId, + status: "failed", + error: "Reusable sandbox lease could not be resumed", + errorCode: REUSABLE_SANDBOX_RESUME_FAILED_CODE, + runtimeMode: "legacy", + executionStage: "preparing", + nativeIssueId: null, + contextSnapshot: null, + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }, + ...overrides, + } as unknown as typeof heartbeatRuns.$inferSelect; + } + + it("keeps a provider outage local until its retry decision, and after a scheduled retry", async () => { + const run = resumeFailureRun(); + const error = providerOutage(); + await reportRunFailure(db, run, { phase: "setup", error, sandboxResumeRetry: "pending" }); + await reportRunFailure(db, run, { phase: "setup", error, sandboxResumeRetry: "scheduled" }); + // The setup path reports once more with its decision, so a pending write stays local in every mode. + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", "all"); + await reportRunFailure(db, run, { phase: "setup", error, sandboxResumeRetry: "pending" }); + expect(mockCaptureRunFailure).not.toHaveBeenCalled(); + }); + + it.each([ + ["no retry was scheduled", {}, "not_scheduled", "unexpected"], + ["the decision is unknown", {}, undefined, "unexpected"], + ["every run failure is reported", {}, "scheduled", "all"], + ["the run failed for another reason", { errorCode: "setup_failed" }, "scheduled", "unexpected"], + ] as const)("reports a sandbox resume provider outage when %s", async (_label, overrides, sandboxResumeRetry, mode) => { + vi.stubEnv("PAPERCLIP_SENTRY_RUN_FAILURES", mode); + const run = resumeFailureRun(overrides); + await reportRunFailure(db, run, { phase: "setup", error: providerOutage(), ...(sandboxResumeRetry ? { sandboxResumeRetry } : {}) }); + expect(mockCaptureRunFailure).toHaveBeenCalledOnce(); + expect(mockCaptureRunFailure.mock.calls[0]![0]).toMatchObject({ errorCode: run.errorCode, runStatus: "failed" }); + }); +}); diff --git a/server/src/services/heartbeat/run-completion-reviewed-chat-report.test.ts b/server/src/services/heartbeat/run-completion-reviewed-chat-report.test.ts index 2d049495938..f75e6054f28 100644 --- a/server/src/services/heartbeat/run-completion-reviewed-chat-report.test.ts +++ b/server/src/services/heartbeat/run-completion-reviewed-chat-report.test.ts @@ -4,6 +4,9 @@ import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vites import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; import { ReviewedChatBindingNotReadyError } from "../reviewed-chat-binding-diagnostics.js"; +import { environmentResumeErrorData, JsonRpcCallError } from "@paperclipai/plugin-sdk"; +import { EnvironmentRunError } from "../environment-run-orchestrator.js"; +import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; import { reportRunFailure } from "../run-failure-report.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; import { createHeartbeatRunPreparation } from "./run-preparation.js"; @@ -67,7 +70,7 @@ function callbacks(db: Db) { } const support = await getEmbeddedPostgresTestSupport(); -describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)("reviewed chat setup wait reporting", () => { +describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)("setup failure reporting after a bounded retry decision", () => { let database: Awaited> | undefined; let db: Db; beforeAll(async () => { @@ -126,4 +129,42 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect.objectContaining({ phase: "setup", error, reviewedChatRetry: scenario === "scheduled" || scenario === "interaction_scheduled" ? "scheduled" : "not_scheduled" })); }); + + // The sandbox provider answered a saved-sandbox resume with HTTP 502, and the + // orchestrator wrapped the refusal exactly as staging recorded it. + const resumeRefusal = (cause: unknown) => new EnvironmentRunError("lease_acquire_failed", "Failed to acquire lease", { + cause: new ReusableSandboxResumeError({ provider: "daytona", providerLeaseId: "lease-1", cause }) }); + const rpcRefusal = (httpStatus: number, title = `${httpStatus} Bad Gateway`) => new JsonRpcCallError({ code: -32002, + message: `${title}`, data: environmentResumeErrorData({ status: httpStatus }) }); + + it.each(["scheduled", "gateway_timeout", "not_scheduled", "exhausted", "scheduler_failed", "provider_refusal", "unclassified"] as const)( + "reports a sandbox resume provider outage only after its retry decision: %s", async scenario => { + const f = await fixture(); + if (scenario === "scheduler_failed") f.deps.scheduleBoundedRetryForRun.mockRejectedValueOnce(new Error("fixture scheduler failure")); + else f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce(["scheduled", "gateway_timeout", "provider_refusal", "unclassified"].includes(scenario) + ? { outcome: "scheduled" } as never + : scenario === "exhausted" ? { outcome: "retry_exhausted", attempt: 4, maxAttempts: 4 } as never + : { outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + // A 4xx names a refusal of this sandbox, and a failure without a status + // is unproven; neither is a provider outage. + const error = resumeRefusal(scenario === "provider_refusal" ? rpcRefusal(400) + : scenario === "unclassified" ? new Error("worker refused the resume") + : scenario === "gateway_timeout" ? rpcRefusal(504, "504 Gateway Time-out") : rpcRefusal(502)); + await f.completion.failRunSetup(f.setup(error)); + const write = f.deps.setRunStatusIfRunning.mock.calls.find(call => call[1] === "failed"); + expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "reusable_sandbox_resume_failed" }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledOnce(); + if (scenario === "provider_refusal" || scenario === "unclassified") { + // Other resume refusals keep their report on the failed-status write. + expect(write?.[3]).toEqual(expect.not.objectContaining({ sandboxResumeRetry: expect.anything() })); + expect(reportRunFailure).not.toHaveBeenCalled(); + return; + } + expect(write?.[3]).toMatchObject({ phase: "setup", sandboxResumeRetry: "pending" }); + expect(reportRunFailure).toHaveBeenCalledOnce(); + expect(reportRunFailure).toHaveBeenCalledWith(db, + expect.objectContaining({ id: f.run.id, status: "failed", errorCode: "reusable_sandbox_resume_failed" }), + expect.objectContaining({ phase: "setup", error, + sandboxResumeRetry: scenario === "scheduled" || scenario === "gateway_timeout" ? "scheduled" : "not_scheduled" })); + }); }); diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 40e289dbc66..e7234a825a3 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1704,6 +1704,10 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC (sandboxDiskFull ? NATIVE_SANDBOX_DISK_FULL : null) ?? "setup_failed"; const failureCause = GENERIC_RUN_FAILURE_CODES.has(setupFailureErrorCode) ? thrownFailureCause : null; + // The bounded retry resumes the same lease once the provider recovers, so + // a provider outage is reported only after that retry decision. + const sandboxResumeOutage = setupFailureErrorCode === reusableSandboxResumeFailure?.code + && reusableSandboxResumeFailure.providerOutage; // Bounded kind and reason only: a workspace provider's own text never crosses into Core. const workspaceProviderFailure = findWorkspaceProviderFailure(outerErr)?.summary() ?? null; // A wrapped query error hides its driver cause (for example @@ -1795,8 +1799,9 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC }, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets(), // The Sentry report waits for the bounded retry decision below. ...(reviewedChatBindingNotReady ? { reviewedChatRetry: "pending" as const } : {}), + ...(sandboxResumeOutage ? { sandboxResumeRetry: "pending" as const } : {}), }, { setupFailure: true }).catch(() => ({ run: null, updated: false as const })); - let reviewedChatRetryScheduled = false; + let setupRetryScheduled = false; if (!setupFailureWrite.updated) { logger.info( { @@ -1900,7 +1905,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ); return null; }); - reviewedChatRetryScheduled = retry?.outcome === "scheduled"; + setupRetryScheduled = retry?.outcome === "scheduled"; storageWait = retry; storageWaitScheduled = livenessRun.errorCode === STORAGE_UNAVAILABLE_CODE && (retry?.outcome === "scheduled" || retry?.outcome === "retry_exhausted"); @@ -1932,11 +1937,12 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC ); } // This path owned the failed-status write, which deferred the report. - const reviewedChatReportRun = reviewedChatBindingNotReady && setupFailureWrite.updated + const deferredReportRun = (reviewedChatBindingNotReady || sandboxResumeOutage) && setupFailureWrite.updated ? failedRun ?? setupFailureWrite.run : null; - if (reviewedChatReportRun) { - void reportRunFailure(db, reviewedChatReportRun, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets(), - reviewedChatRetry: reviewedChatRetryScheduled ? "scheduled" : "not_scheduled" }); + if (deferredReportRun) { + const retryDecision = setupRetryScheduled ? "scheduled" as const : "not_scheduled" as const; + void reportRunFailure(db, deferredReportRun, { error: outerErr, phase: "setup", secretValues: readFailureReportSecrets(), + ...(reviewedChatBindingNotReady ? { reviewedChatRetry: retryDecision } : { sandboxResumeRetry: retryDecision }) }); } // Ensure the agent is not left stuck in "running" if the setup-failure // path owned the terminal transition. If another path already finalized diff --git a/server/src/services/reusable-sandbox-resume-error.ts b/server/src/services/reusable-sandbox-resume-error.ts index 1ef63ed2746..8fa27f29864 100644 --- a/server/src/services/reusable-sandbox-resume-error.ts +++ b/server/src/services/reusable-sandbox-resume-error.ts @@ -14,6 +14,11 @@ export class ReusableSandboxResumeError extends Error { readonly code = REUSABLE_SANDBOX_RESUME_FAILED_CODE; readonly provider: string; readonly providerLeaseId: string; + /** + * The provider answered the resume with an HTTP 5xx. That is an outage of + * the provider, not a refusal of this sandbox, so the bounded retry owns it. + */ + readonly providerOutage: boolean; constructor(input: { provider: string; @@ -41,6 +46,8 @@ export class ReusableSandboxResumeError extends Error { this.name = "ReusableSandboxResumeError"; this.provider = input.provider; this.providerLeaseId = input.providerLeaseId; + // Only the closed status decides: a gateway page's text can also read as a timeout. + this.providerOutage = failure?.status != null && failure.status >= 500 && failure.status <= 599; } } diff --git a/server/src/services/run-failure-diagnostics.ts b/server/src/services/run-failure-diagnostics.ts index 4059248a106..2409b15723f 100644 --- a/server/src/services/run-failure-diagnostics.ts +++ b/server/src/services/run-failure-diagnostics.ts @@ -46,6 +46,11 @@ export interface RunFailureReportOptions { * decision; a scheduled retry owns recovery and reports its own outcome. */ reviewedChatRetry?: "pending" | "scheduled" | "not_scheduled"; + /** + * The bounded retry decision for a saved-sandbox resume that the provider + * answered with an HTTP 5xx, with the same `pending` handshake. + */ + sandboxResumeRetry?: "pending" | "scheduled" | "not_scheduled"; } export interface RunFailureException { diff --git a/server/src/services/run-failure-report.ts b/server/src/services/run-failure-report.ts index c4d27b3d675..a69cd67b484 100644 --- a/server/src/services/run-failure-report.ts +++ b/server/src/services/run-failure-report.ts @@ -23,6 +23,7 @@ import { NATIVE_PROVIDER_OVERLOADED_CODE } from "./native-runtime/native-provide import { NATIVE_TOOL_RECEIPT_LIMIT_CODE } from "./native-runtime/native-tool-receipt-limit.js"; import { PROJECT_ACCESS_DENIED } from "./project-access-denial.js"; import { REVIEWED_CHAT_BINDING_NOT_READY } from "./reviewed-chat-binding-diagnostics.js"; +import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; import { readProviderFailure } from "@paperclipai/shared"; type HeartbeatRun = typeof heartbeatRuns.$inferSelect; @@ -347,6 +348,17 @@ function isScheduledReviewedChatRetry(run: HeartbeatRun, options: RunFailureRepo options.reviewedChatRetry === "scheduled"; } +/** + * The sandbox provider answered a saved-sandbox resume with an HTTP 5xx before + * provider dispatch, and the setup path scheduled the bounded retry of the same + * lease. The retry owns recovery; an exhausted or unscheduled retry, and any + * other resume refusal, stays reportable. + */ +function isScheduledSandboxResumeRetry(run: HeartbeatRun, options: RunFailureReportOptions): boolean { + return run.status === "failed" && run.errorCode === REUSABLE_SANDBOX_RESUME_FAILED_CODE && + options.sandboxResumeRetry === "scheduled"; +} + const providerAuthErrorCodes = new Set(["claude_auth_required", "acpx_auth_required"]); /** @@ -401,7 +413,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR // The terminal write that ends the retry reports instead (`isNewTerminalRunFailure`). if (isPendingNativeRetry(run)) return Promise.resolve(); // The setup path reports this run again once it has decided the retry. - if (options.reviewedChatRetry === "pending") return Promise.resolve(); + if (options.reviewedChatRetry === "pending" || options.sandboxResumeRetry === "pending") return Promise.resolve(); if (mode === "unexpected" && ( isMissingSecretConfigurationBlocker(run, options) || isAiConnectionSelectionBlocker(run, options) || @@ -416,6 +428,7 @@ export function reportRunFailure(db: Db, run: HeartbeatRun, options: RunFailureR isCodexAcpModelCapacityFailure(run, options) || isAcpProviderAccessFailure(run, options) || isScheduledReviewedChatRetry(run, options) || + isScheduledSandboxResumeRetry(run, options) || isRestartInterruptedSetup(run) )) return Promise.resolve(); const runStatus = run.status;