diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index d5c04deec3e..fcf339efa67 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1470,6 +1470,14 @@ recovered workspace hold offers Retry (`canRetry`) and says "Retry or send a message to continue." An intentional Stop keeps "send a message" as its only continuation. +`native_remote_recovery_lease_mismatch` carries a closed detail: +`native_remote_recovery_lease_mismatch: stage= field=`. The stage +is `workspace_preparation`, `environment_acquire`, `environment_resume`, +`workspace_sync` or `runner_verification`. The field names the identity that +differed, for example `lease_status`, `adapter_type`, `provider_lease_id`, `remote_cwd` or +`identity_run_id`. The detail contains no identifiers. Consumers match the code +exactly or as the `code: ` prefix. + When a native attempt fails before launch because its predecessor's session is quarantined, explicit task Retry verifies the retained journal's actual owner. That owner must belong to the same company, task, agent, workspace, native session, diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 38a2e1f1383..a5c368142e4 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -989,13 +989,13 @@ describe("live remote runner lease reattachment", () => { expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); }); it.each([ - null, - { providerLeaseId: "replacement", metadata: { remoteCwd: "/remote/workspace" } }, - { providerLeaseId: "original-sandbox", metadata: { expired: true } }, - ])("holds recovery when the retained lease resume cannot confirm its original allocation (%j)", async receipt => { + [null, "lease"], + [{ providerLeaseId: "replacement", metadata: { remoteCwd: "/remote/workspace" } }, "provider_lease_id"], + [{ providerLeaseId: "original-sandbox", metadata: { expired: true } }, "lease_expired"], + ] as const)("holds recovery when the retained lease resume cannot confirm its original allocation (%j)", async (receipt, field) => { const runtime = makeMockRuntime({ resumeRunLease: vi.fn(async () => receipt) }); await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) - .rejects.toThrow("native_remote_recovery_lease_mismatch"); + .rejects.toThrow(new Error(`native_remote_recovery_lease_mismatch: stage=environment_resume field=${field}`)); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); expect(runtime.execute).not.toHaveBeenCalled(); expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); @@ -1036,22 +1036,25 @@ describe("live remote runner lease reattachment", () => { ...input, reattachRemoteLease: { kind: "plugin_task", leaseId: lease.id }, }); if (state === "active") await expect(acquired).resolves.toMatchObject({ lease }); - else await expect(acquired).rejects.toThrow("native_remote_recovery_lease_mismatch"); + else await expect(acquired).rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=environment_acquire field=" + + (state === "foreign-company" ? "lease_scope" : state === "revoked" ? "lease_status" : "plugin_task"))); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); expect(runtime.resumeRunLease).not.toHaveBeenCalled(); }); it.each([ - { companyId: "foreign" }, { environmentId: "foreign" }, { heartbeatRunId: "another-run" }, - { issueId: "another-task" }, { executionWorkspaceId: "another-workspace" }, - { providerLeaseId: "replacement" }, { provider: "another-provider" }, - { status: "released" }, { expiresAt: new Date(0) }, { releasedAt: new Date() }, { cleanupStatus: "pending" }, - { metadata: { agentId: "another-agent", remoteCwd: "/remote/workspace" } }, - { metadata: { agentId: "agent-1", remoteCwd: "/other/workspace" } }, - ] as Partial[])("fails closed before provider acquisition for a mismatched lease %j", async (override) => { + [{ companyId: "foreign" }, "lease_scope"], [{ environmentId: "foreign" }, "lease_scope"], + [{ heartbeatRunId: "another-run" }, "lease_scope"], [{ issueId: "another-task" }, "lease_scope"], + [{ executionWorkspaceId: "another-workspace" }, "lease_scope"], + [{ providerLeaseId: "replacement" }, "provider_lease_id"], [{ provider: "another-provider" }, "provider"], + [{ status: "released" }, "lease_status"], [{ expiresAt: new Date(0) }, "lease_expired"], + [{ releasedAt: new Date() }, "lease_status"], [{ cleanupStatus: "pending" }, "lease_status"], + [{ metadata: { agentId: "another-agent", remoteCwd: "/remote/workspace" } }, "lease_scope"], + [{ metadata: { agentId: "agent-1", remoteCwd: "/other/workspace" } }, "remote_cwd"], + ] as [Partial, string][])("fails closed before provider acquisition for a mismatched lease %j", async (override, field) => { mockGetLease.mockResolvedValue({ ...originalLease(), ...override }); const runtime = makeMockRuntime(); await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) - .rejects.toThrow("native_remote_recovery_lease_mismatch"); + .rejects.toThrow(new Error(`native_remote_recovery_lease_mismatch: stage=environment_acquire field=${field}`)); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); expect(runtime.resumeRunLease).not.toHaveBeenCalled(); }); @@ -1059,7 +1062,18 @@ describe("live remote runner lease reattachment", () => { mockGetLease.mockResolvedValue(null); const runtime = makeMockRuntime(); await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input)) - .rejects.toThrow("native_remote_recovery_lease_mismatch"); + .rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=environment_acquire field=lease")); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + }); + it("names a changed adapter type, not the environment driver, for a plugin task reattachment", async () => { + mockGetEnvironment.mockResolvedValue(makeEnvironment("plugin")); + const lease = originalLease(); + lease.metadata = { agentId: "agent-1", runnerTaskLease: true, runnerTaskDispatched: true }; + mockGetLease.mockResolvedValue(lease); + const runtime = makeMockRuntime(); + await expect(environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun({ + ...input, adapterType: "codex_local", reattachRemoteLease: { kind: "plugin_task", leaseId: lease.id }, + })).rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=environment_acquire field=adapter_type")); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); }); }); diff --git a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts index 88d1448a4f4..7025569275b 100644 --- a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts +++ b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts @@ -227,7 +227,9 @@ describeEmbeddedPostgres("heartbeat native remote preparation failure retention" await heartbeat.drainActiveRunExecutions(); expect(readyAllocation).toHaveBeenCalledOnce(); const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); - expect(run).toMatchObject({ status: "failed", error: "native_remote_recovery_lease_mismatch" }); + // The run records which check failed: the provider no longer resumes the lease. + expect(run).toMatchObject({ status: "failed", + error: "native_remote_recovery_lease_mismatch: stage=environment_resume field=lease" }); const retryKey = `task-retry:restart-interrupted-turn:${f.runId}`; await vi.waitFor(async () => { const retries = await db.select().from(agentWakeupRequests).where(and( diff --git a/server/src/__tests__/native-workspace-sync.test.ts b/server/src/__tests__/native-workspace-sync.test.ts index f690abcd5be..563b4aa0572 100644 --- a/server/src/__tests__/native-workspace-sync.test.ts +++ b/server/src/__tests__/native-workspace-sync.test.ts @@ -109,7 +109,7 @@ describe("native workspace sync durable metadata", () => { restartRecovery: { kind: "reattach_remote_runner", runId: "run", leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "graceful", recoveryRequestId: null, remote: { providerLeaseId: "original", remoteCwd: "/workspace" } }, - })).rejects.toThrow("native_remote_recovery_lease_mismatch"); + })).rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=workspace_sync field=provider_lease_id")); expect(select).not.toHaveBeenCalled(); }); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 4e3f725cfa5..de083eed785 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -20,6 +20,7 @@ import { prepareRemoteWorkspace, readRemoteWorkspaceAuthority, remoteWorkspaceRe import { environmentLeases, type Db } from "@paperclipai/db"; import { and, eq } from "drizzle-orm"; import { remoteExecutionHasStopped } from "./remote-execution-termination.js"; +import { remoteRecoveryLeaseMismatch, type RemoteRecoveryLeaseMismatchField } from "./native-runtime/remote-recovery-lease-mismatch.js"; import { EnvironmentProviderUnavailableError, readTransientEnvironmentProviderFailure } from "./environment-provider-unavailable-error.js"; import { findReusableSandboxResumeError } from "./reusable-sandbox-resume-error.js"; import { findPluginWorkerUnavailableError } from "./plugin-worker-unavailable-error.js"; @@ -356,20 +357,24 @@ export function environmentRunOrchestrator( // Never reacquire: acquisition can replace a non-reusable sandbox. const expected = input.reattachRemoteLease; const lease = await environmentsSvc.getLeaseById(expected.leaseId); - if (!lease || - lease.companyId !== input.companyId || lease.environmentId !== environment.id || + const mismatch: RemoteRecoveryLeaseMismatchField | null = !lease ? "lease" + : lease.companyId !== input.companyId || lease.environmentId !== environment.id || lease.heartbeatRunId !== input.heartbeatRunId || lease.issueId !== input.issueId || lease.executionWorkspaceId !== (input.persistedExecutionWorkspace?.id ?? null) || - lease.metadata?.agentId !== input.agentId || lease.status !== "active" || - lease.releasedAt !== null || lease.cleanupStatus !== null || - (lease.expiresAt !== null && new Date(lease.expiresAt).getTime() <= Date.now()) || - (expected.kind === "plugin_task" - ? environment.driver !== "plugin" || input.adapterType !== "paperclip_runner" - || lease.metadata?.runnerTaskLease !== true || lease.metadata?.runnerTaskDispatched !== true - || lease.metadata?.runnerTaskExecutionStopped === true - : environment.driver !== "sandbox" || lease.provider !== environment.config.provider - || lease.providerLeaseId !== expected.providerLeaseId || lease.metadata?.remoteCwd !== expected.remoteCwd)) { - throw new Error("native_remote_recovery_lease_mismatch"); + lease.metadata?.agentId !== input.agentId ? "lease_scope" + : lease.status !== "active" || lease.releasedAt !== null || lease.cleanupStatus !== null ? "lease_status" + : lease.expiresAt !== null && new Date(lease.expiresAt).getTime() <= Date.now() ? "lease_expired" + : expected.kind === "plugin_task" + ? environment.driver !== "plugin" ? "environment_driver" + : input.adapterType !== "paperclip_runner" ? "adapter_type" + : lease.metadata?.runnerTaskLease !== true || lease.metadata?.runnerTaskDispatched !== true + || lease.metadata?.runnerTaskExecutionStopped === true ? "plugin_task" : null + : environment.driver !== "sandbox" ? "environment_driver" + : lease.provider !== environment.config.provider ? "provider" + : lease.providerLeaseId !== expected.providerLeaseId ? "provider_lease_id" + : lease.metadata?.remoteCwd !== expected.remoteCwd ? "remote_cwd" : null; + if (mismatch || !lease) { + throw remoteRecoveryLeaseMismatch("environment_acquire", mismatch ?? "lease"); } if (expected.kind !== "plugin_task") { // The provider may have archived this retained sandbox while its @@ -381,10 +386,11 @@ export function environmentRunOrchestrator( ...(warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs } : {}), ...(runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs } : {}), }); - if (!resumed || resumed.providerLeaseId !== expected.providerLeaseId || - resumed.metadata?.expired === true) { - throw new Error("native_remote_recovery_lease_mismatch"); + if (!resumed) throw remoteRecoveryLeaseMismatch("environment_resume", "lease"); + if (resumed.providerLeaseId !== expected.providerLeaseId) { + throw remoteRecoveryLeaseMismatch("environment_resume", "provider_lease_id"); } + if (resumed.metadata?.expired === true) throw remoteRecoveryLeaseMismatch("environment_resume", "lease_expired"); // Resume confirms allocation readiness. Its optional metadata may // describe the allocation root; keep the validated lease's workspace. } diff --git a/server/src/services/heartbeat/restart-interrupted-turn.ts b/server/src/services/heartbeat/restart-interrupted-turn.ts index ba4f1ca07b1..df89505a90e 100644 --- a/server/src/services/heartbeat/restart-interrupted-turn.ts +++ b/server/src/services/heartbeat/restart-interrupted-turn.ts @@ -5,11 +5,12 @@ import { isUserCommentContinuationReason } from "../explicit-native-continuation import { requestTaskRetry } from "../task-retry.js"; import { HttpError } from "../../errors.js"; import { logger } from "../../middleware/logger.js"; +import { NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH } from "../native-runtime/remote-recovery-lease-mismatch.js"; /** Reattachment could not find the run's original runner or allocation: the * sandbox lease or plugin task is gone, or the remote runner process is not * there to verify. This is infrastructure, not the task's own failure. */ -const REATTACHMENT_LOST_ERRORS = ["native_remote_recovery_lease_mismatch", "native_plugin_recovery_lease_mismatch", +const REATTACHMENT_LOST_ERRORS = [NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH, "native_plugin_recovery_lease_mismatch", "runner_remote_process_identity_unavailable", "runner_continuity_break_archive_failed"]; /** A failure message is one of the closed codes, optionally followed by diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 612075612bb..1edc7bfe73d 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -2058,7 +2058,8 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace remote: { providerLeaseId: "original-provider-lease", remoteCwd: "/original" }, }; const acquire = vi.spyOn(envOrchestrator, "acquireForRun"); - await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("native_remote_recovery_lease_mismatch"); + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow( + new Error("native_remote_recovery_lease_mismatch: stage=workspace_preparation field=workspace_reference")); expect(acquire).not.toHaveBeenCalled(); expect(await leases(input.run.id)).toHaveLength(0); expect(input.nativeRunnerPreparationSpans).toMatchObject([{ name: "environment.acquire", outcome: "failed" }]); diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index d00ff0cd91d..7ff85efacfd 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1,3 +1,4 @@ +import { remoteRecoveryLeaseMismatch } from "../native-runtime/remote-recovery-lease-mismatch.js"; import { resolveProviderApiPlacement } from "../native-runtime/provider-api-placement.js"; import { assertEphemeralWorkspaceExecution, readRemoteWorkspaceAuthority, RemoteWorkspaceWriterActiveError, reserveRemoteWorkspace, shouldUseRemoteWorkspaces } from "../remote-workspaces.js"; import { WORKSPACE_BUSY_HOLDER_STALE_AFTER_MS, WorkspaceBusyDeferral } from "./retries.js"; @@ -1296,12 +1297,14 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace ? readNativeWorkspaceSyncReference(parseObject(run.runnerProfileJson).nativeWorkspaceSync) : null; const remoteRecoveryLocation = remoteAuthority ? readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata)?.location ?? null : null; - if (remoteRecovery && (remoteRecovery.runId !== run.id || (sandboxRecovery && (remoteAuthority - ? !remoteRecoveryLocation || remoteRecoveryLocation.providerLeaseId !== sandboxRecovery.providerLeaseId - || remoteRecoveryLocation.remoteCwd !== sandboxRecovery.remoteCwd - : !recoveryWorkspace || recoveryWorkspace.providerLeaseId !== sandboxRecovery.providerLeaseId - || recoveryWorkspace.remoteCwd !== sandboxRecovery.remoteCwd)))) { - throw new Error("native_remote_recovery_lease_mismatch"); + if (remoteRecovery) { + const saved = remoteAuthority ? remoteRecoveryLocation : recoveryWorkspace; + const mismatch = remoteRecovery.runId !== run.id ? "run_id" as const + : !sandboxRecovery ? null + : !saved ? "workspace_reference" as const + : saved.providerLeaseId !== sandboxRecovery.providerLeaseId ? "provider_lease_id" as const + : saved.remoteCwd !== sandboxRecovery.remoteCwd ? "remote_cwd" as const : null; + if (mismatch) throw remoteRecoveryLeaseMismatch("workspace_preparation", mismatch); } acquiredEnvironment = await timeWorkspace("workspace_environment_acquire", () => envOrchestrator.acquireForRun({ companyId: agent.companyId, diff --git a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts index 65e6b98e908..49984b10497 100644 --- a/server/src/services/native-runtime/native-controller-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-controller-recovery.integration.test.ts @@ -154,7 +154,8 @@ describe.skipIf(!recoveryPostgresSupport.supported)("native controller durable d runnerLifecycle: "ready", runnerSettled: true }); expect(nativeSessionFailureSourceCode(failure)).toBe("native_session_interrupted"); await expect(verifyRemoteRunnerReattachment({ claim: { ...sourceClaim, runId }, target: target as never, - identity, runId, normalizedSessionId: fixture.scope.normalizedSessionId })).rejects.toThrow("native_remote_recovery_lease_mismatch"); + identity, runId, normalizedSessionId: fixture.scope.normalizedSessionId })) + .rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=runner_verification field=identity_run_id")); const recovery = await nativeProviderRecoveryEvidence({ db, runId, executionTargetKind: target.kind, sourceFailureCode: nativeSessionFailureSourceCode(failure) }); expect(recovery).toMatchObject({ providerSessionEstablished: true, providerEventsExist: false, checkpointExists: true }); expect(recovery.recoveryMode).toBe("ambiguous_state"); diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 6aacc8b8e9a..09108ef78cf 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -740,7 +740,20 @@ describe("remote controller restart adoption", () => { runId: "run", normalizedSessionId: "session", }), - ).rejects.toThrow("native_remote_recovery_lease_mismatch"); + ).rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=runner_verification field=provider_lease_id")); + expect(execute).not.toHaveBeenCalled(); + }); + it("names the missing durable runner identity before reading the sandbox", async () => { + const { execute, target } = fixture(); + await expect( + verifyRemoteRunnerReattachment({ + claim, + target: target as never, + identity: {}, + runId: "run", + normalizedSessionId: "session", + }), + ).rejects.toThrow(new Error("native_remote_recovery_lease_mismatch: stage=runner_verification field=identity_run_id")); expect(execute).not.toHaveBeenCalled(); }); it.each([ diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e3b0d16f6ab..c18f97a92c6 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,3 +1,4 @@ +import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; import { captureRemoteNativeProviderArtifacts } from "./native-provider-artifact-capture.js"; import { NativeSessionDiagnostics, nativeSessionDiagnostics, type NativeDiagnosticLease } from "./native-session-diagnostics.js"; import { recordAgentProviderRateLimits } from "../agent-provider-rate-limits.js"; @@ -13368,21 +13369,17 @@ export async function verifyRemoteRunnerReattachment(input: { recoveryDeadline?: number; }) { const { claim, target, identity } = input; - if ( - claim.remote.kind === "plugin_task" || - target?.kind !== "remote" || - target.transport !== "sandbox" || - !target.runner || - claim.runId !== input.runId || - identity.runId !== input.runId || - identity.normalizedSessionId !== input.normalizedSessionId || - claim.remote.providerLeaseId !== - target.sandboxLeaseAcquisition?.providerLeaseId || - target.sandboxLeaseAcquisition.outcome === "replacement" || - claim.remote.remoteCwd !== target.remoteCwd - ) { - throw new Error("native_remote_recovery_lease_mismatch"); + if (claim.remote.kind === "plugin_task") throw remoteRecoveryLeaseMismatch("runner_verification", "plugin_task"); + if (target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) { + throw remoteRecoveryLeaseMismatch("runner_verification", "target"); } + const mismatch = claim.runId !== input.runId ? "run_id" as const + : identity.runId !== input.runId ? "identity_run_id" as const + : identity.normalizedSessionId !== input.normalizedSessionId ? "identity_session_id" as const + : claim.remote.providerLeaseId !== target.sandboxLeaseAcquisition?.providerLeaseId ? "provider_lease_id" as const + : target.sandboxLeaseAcquisition?.outcome === "replacement" ? "replacement_sandbox" as const + : claim.remote.remoteCwd !== target.remoteCwd ? "remote_cwd" as const : null; + if (mismatch) throw remoteRecoveryLeaseMismatch("runner_verification", mismatch); const runner = target.runner; const probeDeadline = Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY, Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS); diff --git a/server/src/services/native-runtime/native-workspace-sync.ts b/server/src/services/native-runtime/native-workspace-sync.ts index a684714b53a..82f703e98ca 100644 --- a/server/src/services/native-runtime/native-workspace-sync.ts +++ b/server/src/services/native-runtime/native-workspace-sync.ts @@ -1,3 +1,4 @@ +import { remoteRecoveryLeaseMismatch } from "./remote-recovery-lease-mismatch.js"; import { hasMigratedLegacyWorkspaceSource } from "../legacy-workspace-recovery-migration.js"; import { notifyCancelledWorkspacePreservation } from "../cancelled-workspace-preservation-notifications.js"; import { notifyEnvironmentCleanup } from "../environment-cleanup-notifications.js"; @@ -915,11 +916,13 @@ export async function prepareNativeWorkspaceSync(input: { const target = input.target; const providerLeaseId = providerLeaseIdFor({ target, lease: input.lease }); // Reattachment may never seed a replacement sandbox over a still-live run. - if (input.restartRecovery?.kind === "reattach_remote_runner" && - (input.restartRecovery.remote.kind === "plugin_task" || input.restartRecovery.runId !== input.runId || - input.restartRecovery.remote.providerLeaseId !== providerLeaseId || - input.restartRecovery.remote.remoteCwd !== target.remoteCwd)) { - throw new Error("native_remote_recovery_lease_mismatch"); + if (input.restartRecovery?.kind === "reattach_remote_runner") { + const { remote, runId } = input.restartRecovery; + const mismatch = remote.kind === "plugin_task" ? "plugin_task" as const + : runId !== input.runId ? "run_id" as const + : remote.providerLeaseId !== providerLeaseId ? "provider_lease_id" as const + : remote.remoteCwd !== target.remoteCwd ? "remote_cwd" as const : null; + if (mismatch) throw remoteRecoveryLeaseMismatch("workspace_sync", mismatch); } const run = await measureRunPhase("workspace_sync.run_read", () => input.db .select({ companyId: heartbeatRuns.companyId, runnerProfileJson: heartbeatRuns.runnerProfileJson }) diff --git a/server/src/services/native-runtime/remote-recovery-lease-mismatch.ts b/server/src/services/native-runtime/remote-recovery-lease-mismatch.ts new file mode 100644 index 00000000000..84102e142bd --- /dev/null +++ b/server/src/services/native-runtime/remote-recovery-lease-mismatch.ts @@ -0,0 +1,42 @@ +/** + * Reattaching a remote native run proves that the original sandbox lease, its + * workspace, and its runner are still the ones the run saved. When a check + * fails, the run fails with `native_remote_recovery_lease_mismatch`. The detail + * names the step and the identity field that differed, using closed values + * only (no identifiers), so an operator can tell which check failed. + * + * Consumers match the code exactly or as the `code: detail` prefix. + */ +export const NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH = "native_remote_recovery_lease_mismatch"; + +export type RemoteRecoveryLeaseMismatchStage = + | "workspace_preparation" + | "environment_acquire" + | "environment_resume" + | "workspace_sync" + | "runner_verification"; + +export type RemoteRecoveryLeaseMismatchField = + | "run_id" + | "workspace_reference" + | "lease" + | "lease_scope" + | "lease_status" + | "lease_expired" + | "environment_driver" + | "adapter_type" + | "provider" + | "provider_lease_id" + | "remote_cwd" + | "plugin_task" + | "target" + | "replacement_sandbox" + | "identity_run_id" + | "identity_session_id"; + +export function remoteRecoveryLeaseMismatch( + stage: RemoteRecoveryLeaseMismatchStage, + field: RemoteRecoveryLeaseMismatchField, +): Error { + return new Error(`${NATIVE_REMOTE_RECOVERY_LEASE_MISMATCH}: stage=${stage} field=${field}`); +} diff --git a/server/src/services/workspace-recovery-retry-copyback.test.ts b/server/src/services/workspace-recovery-retry-copyback.test.ts index 4b6b8bb218f..101ae89912b 100644 --- a/server/src/services/workspace-recovery-retry-copyback.test.ts +++ b/server/src/services/workspace-recovery-retry-copyback.test.ts @@ -121,6 +121,7 @@ describe("automatic retained workspace recovery", () => { { title: "a restart lost its sandbox before the turn started", turn: "none" as const, error: "native_remote_recovery_lease_mismatch", initial: false }, { title: "a same-run retry found the settled turn's remote runner gone", turn: "settled" as const, error: "runner_remote_process_identity_unavailable", initial: false }, { title: "a same-run retry found the settled turn's sandbox lease gone", turn: "settled" as const, error: "native_remote_recovery_lease_mismatch", initial: false }, + { title: "a same-run retry named the lease check that failed", turn: "settled" as const, error: "native_remote_recovery_lease_mismatch: stage=environment_acquire field=lease_status", initial: false }, ])("continues a user turn once, without a second message, after $title and recovery saved the work", async ({ turn, error, initial }) => { const f = await restartAllocationLostSource(turn, error); const service = heartbeatService(db, { environmentRuntime: f.environmentRuntime });