diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index a354d195eae..639d97fcce1 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -1,5 +1,6 @@ import { JsonRpcCallError, environmentResumeErrorData } from "@paperclipai/plugin-sdk"; import { collectRunFailureDiagnostics } from "../services/run-failure-diagnostics.js"; +import { PluginWorkerUnavailableError } from "../services/plugin-worker-unavailable-error.js"; import { createSandboxOrphanCleanupSpool } from "../services/sandbox-orphan-cleanup-spool.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; @@ -1084,8 +1085,9 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { pluginWorkerManager: offlineManager, pluginWorkerReadyTimeoutMs: 0, }); - await expect(runtimeWithStoppedWorker.acquireRunLease(input)) - .rejects.toThrow("its worker is not running"); + const refusal = await runtimeWithStoppedWorker.acquireRunLease(input).catch(error => error); + expect(refusal).toBeInstanceOf(PluginWorkerUnavailableError); + expect((refusal as Error).message).toContain("its worker is not running"); expect(offlineManager.call).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/heartbeat-process-recovery.test.ts b/server/src/__tests__/heartbeat-process-recovery.test.ts index d7edf39fe74..1560c6e3695 100644 --- a/server/src/__tests__/heartbeat-process-recovery.test.ts +++ b/server/src/__tests__/heartbeat-process-recovery.test.ts @@ -5594,7 +5594,7 @@ describeEmbeddedPostgres("heartbeat orphaned process recovery", () => { const failedRun = runs?.find((row) => row.id === runId); const retryRun = runs?.find((row) => row.id !== runId); expect(failedRun?.status).toBe("failed"); - expect(failedRun?.errorCode).toBe("setup_failed"); + expect(failedRun?.errorCode).toBe("plugin_worker_unavailable"); expect(failedRun?.error).toContain("worker is not running"); expect(retryRun).toMatchObject({ status: "scheduled_retry", diff --git a/server/src/__tests__/plugin-worker-manager.test.ts b/server/src/__tests__/plugin-worker-manager.test.ts index 6a76a509a16..5f8ee1d604c 100644 --- a/server/src/__tests__/plugin-worker-manager.test.ts +++ b/server/src/__tests__/plugin-worker-manager.test.ts @@ -27,6 +27,7 @@ vi.mock("../middleware/logger.js", () => { }); import { logger } from "../middleware/logger.js"; +import { PluginWorkerUnavailableError, PLUGIN_WORKER_UNAVAILABLE_CODE } from "../services/plugin-worker-unavailable-error.js"; import { withRunExecutionTiming } from "../services/execution-preparation-timing.js"; import { appendStderrExcerpt, @@ -80,6 +81,24 @@ describe("worker configuration completion", () => { }); }); +describe("plugin worker unavailable refusals", () => { + it("refuses calls to a stopped or missing worker with a typed, unsent error", async () => { + const handle = createPluginWorkerHandle("test.plugin", { + entrypointPath: DELAYED_WORKER_ENTRYPOINT, manifest: TEST_MANIFEST, config: {}, + instanceInfo: { instanceId: "instance-1", hostVersion: "1.0.0" }, apiVersion: 1, hostHandlers: {}, + }); + const settled = vi.fn(); + const stopped = await handle.call("environmentExecute", {} as never, undefined, undefined, settled).catch(error => error); + expect(stopped).toBeInstanceOf(PluginWorkerUnavailableError); + expect(stopped).toMatchObject({ code: PLUGIN_WORKER_UNAVAILABLE_CODE, pluginId: "test.plugin", workerStatus: "stopped", + message: 'Cannot call "environmentExecute" — worker for "test.plugin" is stopped' }); + expect(settled).toHaveBeenCalledTimes(1); + const missing = await createPluginWorkerManager().call("missing.plugin", "environmentExecute", {} as never).catch(error => error); + expect(missing).toBeInstanceOf(PluginWorkerUnavailableError); + expect(missing).toMatchObject({ workerStatus: null, message: 'No worker registered for plugin "missing.plugin"' }); + }); +}); + describe("resolveRpcCallTimeoutMs", () => { const MAX_RPC_TIMEOUT_MS = 15 * 60 * 1_000; const MAX_NODE_TIMER_TIMEOUT_MS = 2_147_483_647; diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 45964277672..b4fffc7ada9 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -1,5 +1,6 @@ import { readNativeWorkspaceMigration } from "./native-runtime/native-workspace-migration.js"; import { ReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js"; +import { PluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; import { HttpError } from "../errors.js"; import { captureEnvironmentAcquisitionDiagnostic } from "./environment-acquisition-diagnostics.js"; import { createOwnedSandbox, recoverSandboxCreations, recoverableTaskCreation } from "./sandbox-creation-operations.js"; @@ -1638,9 +1639,10 @@ function createSandboxEnvironmentDriver( ); } if (pluginProvider.state === "worker_unavailable") { - throw new Error( - `Sandbox provider "${parsed.config.provider}" is installed via plugin "${pluginProvider.resolved.plugin.pluginKey}", but its worker is not running.`, - ); + // No request reached the worker, so setup may retry this refusal + // within its bootstrap budget once the worker runs again. + throw new PluginWorkerUnavailableError({ pluginId: pluginProvider.resolved.plugin.id, workerStatus: null, + message: `Sandbox provider "${parsed.config.provider}" is installed via plugin "${pluginProvider.resolved.plugin.pluginKey}", but its worker is not running.` }); } const workerConfig = { diff --git a/server/src/services/heartbeat/retries.ts b/server/src/services/heartbeat/retries.ts index 6fb3111a7b8..40cbaf4731a 100644 --- a/server/src/services/heartbeat/retries.ts +++ b/server/src/services/heartbeat/retries.ts @@ -23,6 +23,7 @@ import { import { WORKSPACE_GIT_SCAN_ERROR_CODES } from "../workspace-git-operation-scheduler.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "../remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; +import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { CONTROLLER_LEASE_LOST_FAILURE_CODE } from "../legacy-controller-lease.js"; import { randomUUID } from "node:crypto"; import { @@ -135,7 +136,7 @@ const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_WAKE_REASON = "transient_failure_retry"; export function isTransientWorkspacePreparationCode(code: string | null | undefined): boolean { return code === WORKSPACE_GIT_SCAN_ERROR_CODES.timeout || code === WORKSPACE_GIT_SCAN_ERROR_CODES.saturated || code === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || code === REUSABLE_SANDBOX_RESUME_FAILED_CODE - || code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE; + || code === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || code === PLUGIN_WORKER_UNAVAILABLE_CODE; } export const BOUNDED_TRANSIENT_HEARTBEAT_RETRY_MAX_ATTEMPTS = @@ -341,7 +342,8 @@ function isRetryableInteractionContinuationInfrastructureFailure( if ( run.errorCode === WORKSPACE_VALIDATION_FAILURE_CODE || run.errorCode === "process_lost" || - run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE + run.errorCode === CONTROLLER_LEASE_LOST_FAILURE_CODE || + run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE ) { return true; } diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 6c06858564b..2323daf486f 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -29,6 +29,7 @@ import { ProjectAccessDeniedError } from "../project-access-denial.js"; import { RemoteWorkspaceOperationError, RemoteWorkspaceOperationTimeoutError, RemoteWorkspacePreparationTimeoutError, RemoteWorkspaceRestoreError } from "../remote-workspace-transport.js"; import { EnvironmentRunError } from "../environment-run-orchestrator.js"; import { ReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; +import { PluginWorkerUnavailableError } from "../plugin-worker-unavailable-error.js"; import { createHeartbeatLifecycle, type HeartbeatLifecycleDependencies } from "./run-lifecycle.js"; import { createHeartbeatRunPreparation, ConfigurationIncompleteFailure } from "./run-preparation.js"; import { createHeartbeatRunState, getAdapterSessionCodec } from "./run-state.js"; @@ -949,6 +950,35 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); }); + it.each(["fresh", "started", "untrusted", "retained"] as const)("routes a %s stopped plugin worker refusal through bounded bootstrap recovery", async scenario => { + const f = await fixture({ runtimeMode: scenario === "retained" ? "native" : "legacy" }); + const refusal = new PluginWorkerUnavailableError({ pluginId: "plugin-1", workerStatus: "stopped", + message: 'Cannot call "environmentExecute" — worker for "plugin-1" is stopped' }); + const error = scenario === "untrusted" ? new Error(refusal.message) + : new EnvironmentRunError("workspace_realization_failed", refusal.message, { cause: refusal }); + f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + await f.completion.failRunSetup(f.setup(error, scenario === "started")); + const failed = await read(f.run.id); + expect(failed).toMatchObject({ status: "failed", errorCode: scenario === "fresh" ? "plugin_worker_unavailable" : "setup_failed" }); + if (scenario === "fresh") { + expect(failed.resultJson).toMatchObject({ executionRecovery: { kind: "bootstrap", providerWorkStarted: false } }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent); + } else expect(f.deps.scheduleBoundedRetryForRun).not.toHaveBeenCalled(); + }); + + it("routes a lease acquisition refused by a stopped provider worker through bounded bootstrap recovery", async () => { + const f = await fixture(); + const refusal = new PluginWorkerUnavailableError({ pluginId: "plugin-1", workerStatus: null, + message: 'Sandbox provider "daytona" is installed via plugin "paperclip.daytona-sandbox-provider", but its worker is not running.' }); + const wrapped = new EnvironmentRunError("lease_acquire_failed", + `Failed to acquire lease for environment "Paperclip Computer" (sandbox): ${refusal.message}`, { cause: refusal }); + f.deps.scheduleBoundedRetryForRun.mockResolvedValueOnce({ outcome: "not_scheduled", reason: "Fixture scheduler boundary", issueId: null }); + await f.completion.failRunSetup(f.setup(wrapped)); + expect(await read(f.run.id)).toMatchObject({ status: "failed", errorCode: "plugin_worker_unavailable", + resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }); + expect(f.deps.scheduleBoundedRetryForRun).toHaveBeenCalledWith(expect.objectContaining({ id: f.run.id }), f.agent); + }); + it.each(["retried", "exhausted", "refused", "started", "in progress"] as const)("names a %s saved workspace restore failure and leaves recovery", async scenario => { const f = await fixture(); const [issue] = await db.insert(issues).values({ companyId: f.run.companyId, title: "Parent task", status: scenario === "in progress" ? "in_progress" : "blocked", diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 4ca7c47d87f..c0ecf1746d3 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -12,6 +12,7 @@ import { readTransientRecoveryContractFromRun, readHeartbeatRunErrorFamily, isTr import { RemoteWorkspacePreparationTimeoutError, REMOTE_WORKSPACE_RESTORE_FAILED_CODE, findRemoteWorkspaceRestoreError } from "../remote-workspace-transport.js"; import { escalateRemoteWorkspaceRestoreFailure } from "../remote-workspace-restore-recovery.js"; import { findReusableSandboxResumeError } from "../reusable-sandbox-resume-error.js"; +import { findPluginWorkerUnavailableError, PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { type RunSessionOutcome, resolveNextSessionState, normalizeUsageTotals, resolveCacheAdjustedCostUsd, resolveLedgerBiller, resolveLedgerCostStatus, normalizeLedgerBillingType } from "./run-state.js"; import { readGitConnectionFailure } from "../git-connection-failure.js"; import { CONFIGURATION_INCOMPLETE_FAILURE_CODE, ConfigurationIncompleteFailure } from "./run-preparation.js"; @@ -40,7 +41,7 @@ import { resolveChatRunPresentationAuthorizationReason } from "../chat-run-publi import { hasChatRunOwnedProviderInteraction } from "../chat-interaction-arbitration.js"; import { readAcceptedTaskContinuationSummary } from "../heartbeat-run-summary.js"; import { SANDBOX_PROVIDER_PLUGIN_NOT_READY_REASON } from "../recovery/stranded-notice.js"; -import { MAX_TURN_CONTINUATION_RETRY_REASON } from "../../modules/run-dispatch/index.js"; +import { MAX_TURN_CONTINUATION_RETRY_REASON, isResolvedInteractionContinuationWakeContext } from "../../modules/run-dispatch/index.js"; import { redactCurrentUserText } from "../../log-redaction.js"; import type { createHeartbeatLifecycle } from "./run-lifecycle.js"; import type { createHeartbeatRunPreparation } from "./run-preparation.js"; @@ -1370,11 +1371,17 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC // on exhaustion, the board gets a named recovery action. const remoteWorkspaceRestoreFailure = !providerDispatchStarted && run.runtimeMode === "legacy" && run.nativeSessionId === null && run.runnerInstanceId === null ? findRemoteWorkspaceRestoreError(outerErr) : null; + // A plugin worker that was not running refused the call before sending it, + // so no provider work or plugin side effect started. Retry within the + // bootstrap budget; the worker is restarted or replaced meanwhile. + const pluginWorkerUnavailableFailure = !providerDispatchStarted && run.runtimeMode === "legacy" + && run.nativeSessionId === null && run.runnerInstanceId === null ? findPluginWorkerUnavailableError(outerErr) : null; const setupFailureErrorCode = (controllerLeaseLost ? CONTROLLER_LEASE_LOST_FAILURE_CODE : null) ?? remoteWorkspacePreparationTimeout?.code ?? reusableSandboxResumeFailure?.code ?? remoteWorkspaceRestoreFailure?.code ?? + pluginWorkerUnavailableFailure?.code ?? workspaceGitScanFailure?.code ?? workspaceValidationSetupFailure?.code ?? configurationIncompleteSetupFailure?.code ?? @@ -1532,7 +1539,11 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC }); } } else if (livenessRun.errorCode !== PROJECT_ACCESS_DENIED) { - await (isTransientWorkspacePreparationCode(livenessRun.errorCode) + // An accepted interaction keeps its own continuation retry, which + // carries the interaction and escalates on exhaustion. + const interactionOwnsRetry = livenessRun.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE + && isResolvedInteractionContinuationWakeContext(livenessRun.contextSnapshot); + await (isTransientWorkspacePreparationCode(livenessRun.errorCode) && !interactionOwnsRetry ? scheduleBoundedRetryForRun(livenessRun, failedAgent) : scheduleInteractionContinuationInfrastructureRetryIfEligible(livenessRun, failedAgent) ).catch((retryError) => { diff --git a/server/src/services/legacy-execution-recovery.test.ts b/server/src/services/legacy-execution-recovery.test.ts index a4902b6226a..2d632370260 100644 --- a/server/src/services/legacy-execution-recovery.test.ts +++ b/server/src/services/legacy-execution-recovery.test.ts @@ -10,7 +10,7 @@ const stopped = { }, }; -it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { +it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed", "plugin_worker_unavailable"])("does not invent unknown provider actions after exhausted %s bootstrap retries", (errorCode) => { const run = { runtimeMode: "legacy", status: "failed", errorCode, scheduledRetryAttempt: 2, resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } }; expect(legacyExecutionNeedsReconciliation(run)).toBe(false); diff --git a/server/src/services/legacy-execution-recovery.ts b/server/src/services/legacy-execution-recovery.ts index 294f20d9b4b..df6cfaac8f8 100644 --- a/server/src/services/legacy-execution-recovery.ts +++ b/server/src/services/legacy-execution-recovery.ts @@ -8,6 +8,7 @@ import { isPreDispatchReviewWaitVerified } from "./pre-dispatch-review-wait.js"; import { isPreDispatchExecutionWaitVerified } from "./pre-dispatch-execution-wait.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "./remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "./reusable-sandbox-resume-error.js"; +import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "./plugin-worker-unavailable-error.js"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { normalizeMaxTurnStopReason } from "./heartbeat-stop-metadata.js"; import { claimedAdapterType, hasConversationContinuationPolicy } from "./conversation-continuation.js"; @@ -78,7 +79,7 @@ export function legacyExecutionNeedsReconciliation( // that the bootstrap evidence proves never started. Keep unknown outcomes held. if ((run.errorCode === "workspace_git_scan_timeout" || run.errorCode === "workspace_git_scan_saturated" || run.errorCode === REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE || run.errorCode === REUSABLE_SANDBOX_RESUME_FAILED_CODE - || run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE) && + || run.errorCode === REMOTE_WORKSPACE_RESTORE_FAILED_CODE || run.errorCode === PLUGIN_WORKER_UNAVAILABLE_CODE) && evidence?.kind === "bootstrap" && evidence.providerWorkStarted === false) return false; if (executionFailureRetryCount(run) >= 2) return true; return !( diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 221be9cad40..40ce4870c78 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -1,5 +1,6 @@ import { pluginIdleManifestGeneration, type IdlePluginCatalogEntry } from "./plugin-idle-catalog.js"; import { createHash, randomUUID } from "node:crypto"; +import { PluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; /** * PluginWorkerManager — spawns and manages out-of-process plugin worker child * processes, routes JSON-RPC 2.0 calls over stdio, and handles lifecycle @@ -3590,9 +3591,10 @@ export function createPluginWorkerHandle( if (!childProcess?.stdin?.writable) { onWorkerSettled?.(); reject( - new Error( - `Cannot call "${method}" — worker for "${pluginId}" is not running`, - ), + // The status can still read "running" while the child's stdin is + // closed, so no status is reported for this refusal. + new PluginWorkerUnavailableError({ pluginId, workerStatus: null, + message: `Cannot call "${method}" — worker for "${pluginId}" is not running` }), ); return; } @@ -3833,9 +3835,8 @@ export function createPluginWorkerHandle( if (status !== "running" && status !== "starting") { onWorkerSettled?.(); return Promise.reject( - new Error( - `Cannot call "${method}" — worker for "${pluginId}" is ${status}`, - ), + new PluginWorkerUnavailableError({ pluginId, workerStatus: status, + message: `Cannot call "${method}" — worker for "${pluginId}" is ${status}` }), ); } return callInternal(method, params, timeoutMs, executeLogSink, onWorkerSettled, startupContextId, observeResult); @@ -4226,7 +4227,8 @@ export function createPluginWorkerManager( if (!handle) { onWorkerSettled?.(); return Promise.reject( - new Error(`No worker registered for plugin "${pluginId}"`), + new PluginWorkerUnavailableError({ pluginId, workerStatus: null, + message: `No worker registered for plugin "${pluginId}"` }), ); } let finishInspection: (() => void) | undefined; @@ -4256,7 +4258,8 @@ export function createPluginWorkerManager( const handle = workers.get(pluginId); if (!handle) { return Promise.reject( - new Error(`No worker registered for plugin "${pluginId}"`), + new PluginWorkerUnavailableError({ pluginId, workerStatus: null, + message: `No worker registered for plugin "${pluginId}"` }), ); } return handle.openLoginPtySession(input); diff --git a/server/src/services/plugin-worker-unavailable-error.ts b/server/src/services/plugin-worker-unavailable-error.ts new file mode 100644 index 00000000000..89e421b39b8 --- /dev/null +++ b/server/src/services/plugin-worker-unavailable-error.ts @@ -0,0 +1,30 @@ +/** + * Setup failure code for a call that reached a plugin worker which was not + * running (stopped, crashed, restarting, or not registered). No request was + * sent to the worker, so the run is retried with the bounded bootstrap budget + * once the worker is available again. + */ +export const PLUGIN_WORKER_UNAVAILABLE_CODE = "plugin_worker_unavailable"; + +/** A host call was refused because its plugin worker was not running. */ +export class PluginWorkerUnavailableError extends Error { + readonly code = PLUGIN_WORKER_UNAVAILABLE_CODE; + readonly pluginId: string; + /** The worker status at refusal, or null when no worker was registered. */ + readonly workerStatus: string | null; + + constructor(input: { pluginId: string; workerStatus: string | null; message: string }) { + super(input.message); + this.name = "PluginWorkerUnavailableError"; + this.pluginId = input.pluginId; + this.workerStatus = input.workerStatus; + } +} + +/** Find a worker refusal through the orchestrator's error wrappers. */ +export function findPluginWorkerUnavailableError(error: unknown): PluginWorkerUnavailableError | null { + for (let cause = error, depth = 0; cause && depth < 4; cause = (cause as { cause?: unknown }).cause, depth++) { + if (cause instanceof PluginWorkerUnavailableError) return cause; + } + return null; +} diff --git a/server/src/services/recovery/service.pause-durability.test.ts b/server/src/services/recovery/service.pause-durability.test.ts index 899d93d6aeb..7d9e7407f97 100644 --- a/server/src/services/recovery/service.pause-durability.test.ts +++ b/server/src/services/recovery/service.pause-durability.test.ts @@ -5,7 +5,7 @@ const run = (errorCode: string | null) => ({ errorCode } as unknown as Parameters[0]); describe("pause durability: continuation retry classification", () => { - it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed"])("does not grant %s another recovery budget", (code) => { + it.each(["workspace_git_scan_timeout", "workspace_git_scan_saturated", "workspace_git_scan_failed", "workspace_git_scan_output_limit", "workspace_git_scan_cancelled", "remote_workspace_prepare_timeout", "reusable_sandbox_resume_failed", "remote_workspace_restore_failed", "plugin_worker_unavailable"])("does not grant %s another recovery budget", (code) => { expect(classifyContinuationFailure(run(code))).toMatchObject({ kind: "non_retryable", maxAttempts: 0 }); }); it("agent_paused is retryable so work resumes (Option A: Resume Continues Work)", () => { diff --git a/server/src/services/recovery/service.ts b/server/src/services/recovery/service.ts index 971c45f4636..6c7c37eaf0e 100644 --- a/server/src/services/recovery/service.ts +++ b/server/src/services/recovery/service.ts @@ -9,6 +9,7 @@ import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js" import { boundRunIssueId } from "../run-task-binding.js"; import { REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE } from "../remote-workspace-transport.js"; import { REUSABLE_SANDBOX_RESUME_FAILED_CODE } from "../reusable-sandbox-resume-error.js"; +import { PLUGIN_WORKER_UNAVAILABLE_CODE } from "../plugin-worker-unavailable-error.js"; import { recoveryRunContextColumn, recoveryRunVersionColumn } from "./run-context-read.js"; import { recoveryRunResultColumn } from "./run-result-read.js"; import { readLegacyDispositionSource } from "./legacy-disposition-read.js"; @@ -559,6 +560,7 @@ const NON_RETRYABLE_CONTINUATION_ERROR_CODES = new Set([ REMOTE_WORKSPACE_PREPARATION_TIMEOUT_CODE, REUSABLE_SANDBOX_RESUME_FAILED_CODE, REMOTE_WORKSPACE_RESTORE_FAILED_CODE, + PLUGIN_WORKER_UNAVAILABLE_CODE, "low_trust_isolation_unavailable", "low_trust_requires_isolated_workspace", "low_trust_boundary_mismatch", diff --git a/server/src/services/recovery/stranded-notice.test.ts b/server/src/services/recovery/stranded-notice.test.ts index 7cf6541d25d..71d3c4c759d 100644 --- a/server/src/services/recovery/stranded-notice.test.ts +++ b/server/src/services/recovery/stranded-notice.test.ts @@ -136,6 +136,17 @@ describe("buildStrandedRecoveryEscalationNotice", () => { expect(notice.body).not.toContain("retried continuation"); expect(JSON.stringify(notice.metadata)).toContain("repository access and server load"); }); + it("names an environment plugin that was not running after its bounded retries", () => { + const notice = buildStrandedRecoveryEscalationNotice({ + seed: buildImmediateExecutionPathRecoveryNoticeSeed({ status: "in_progress" }), + recoveryActionId: "worker-recovery", + recoveryOwner: null, + sourceRun: { id: "failed-run", status: "failed", errorCode: "plugin_worker_unavailable" }, + }); + expect(notice.presentation.title).toBe("Environment plugin was not running"); + expect(notice.body).toContain("before the agent started"); + expect(JSON.stringify(notice.metadata)).toContain("worker status"); + }); const actionId = "6a2f8e64-6f5e-4b58-b7fd-111111111111"; const owner = { id: "9b1c2d3e-4f50-4a61-8b72-222222222222", name: "CTO" }; const sourceRun = { diff --git a/server/src/services/recovery/stranded-notice.ts b/server/src/services/recovery/stranded-notice.ts index 411fcc15d16..c1fbd1877be 100644 --- a/server/src/services/recovery/stranded-notice.ts +++ b/server/src/services/recovery/stranded-notice.ts @@ -56,6 +56,7 @@ const WORKSPACE_SCAN_NOTICES: Record