diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 0ef42ec0b23..74e550ad52a 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -6216,3 +6216,11 @@ with company/run identifiers to the process logger; it cannot append provider te to a completed run or borrow a successor run's recorder. Canonical native output export, source coverage, secret redaction and persistence-failure warnings retain their existing contracts. + +Exact sandbox reattachment reports the current acquisition as `resumed` only +when the recorded lease passes scope and identity checks and the provider confirms +that same allocation. The lease's original acquisition can have been `replacement`; +that historical label cannot describe the later reattachment. This changes the +operation receipt passed separately through preparation to the execution target, +not persisted lease metadata or the Runner verification guard. Missing, +expired, foreign or changed allocations retain their named recovery refusals. diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 735e03ec415..75f8532993b 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -984,9 +984,73 @@ describe("live remote runner lease reattachment", () => { runnerReconnectGraceMs: REMOTE_RUNNER_RECONNECT_GRACE_MS, }); expect(result.lease).toEqual(savedLease); + expect(result.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", providerLeaseId: "original-sandbox" }); expect(result.leaseContext.executionWorkspaceId).toBe("ew-1"); expect(result.environment.config.reuseLease).toBe(false); }); + it.each(["created", "replacement", "resumed"] as const)( + "reports exact-lease reattachment as resumed after historical %s acquisition", async outcome => { + const savedLease = originalLease(); + savedLease.metadata!.sandboxLeaseAcquisition = { + outcome, providerLeaseId: "original-sandbox", previousProviderLeaseId: "older-sandbox", reason: "resume_failed", + }; + const before = structuredClone(savedLease); + mockGetLease.mockResolvedValue(savedLease); + const runtime = makeMockRuntime(); + const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); + expect(result.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", providerLeaseId: "original-sandbox" }); + expect(result.lease.id).toBe(savedLease.id); + expect(result.lease.metadata?.remoteCwd).toBe("/remote/workspace"); + expect(savedLease).toEqual(before); + expect(runtime.resumeRunLease).toHaveBeenCalledOnce(); + expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); + }, + ); + it.each([false, true])("preserves acquisition history when realization persists workspace metadata (remote authority: %s)", async remoteAuthority => { + const savedLease = originalLease(); + savedLease.metadata!.sandboxLeaseAcquisition = { + outcome: "replacement", providerLeaseId: "original-sandbox", + previousProviderLeaseId: "older-sandbox", reason: "resume_failed", + }; + const history = structuredClone(savedLease.metadata!.sandboxLeaseAcquisition); + mockGetLease.mockResolvedValue(savedLease); + mockUpdateLeaseMetadata.mockImplementation(async (_id, metadata) => ({ ...savedLease, metadata })); + mockBuildWorkspaceRealizationRequest.mockReturnValue({ runtimeOverlay: {} }); + const { resolveEnvironmentExecutionTarget } = await vi.importActual("../services/environment-execution-target.js"); + mockResolveEnvironmentExecutionTarget.mockImplementation(resolveEnvironmentExecutionTarget); + const config = await import("../services/environment-config.js"); + const resolveConfig = vi.spyOn(config, "resolveEnvironmentDriverConfigForRuntime").mockResolvedValue({ + driver: "sandbox", config: { provider: "daytona", reuseLease: false }, + } as never); + const workspace = makePersistedExecutionWorkspace({ metadata: remoteAuthority ? { + remoteWorkspace: { schema: "paperclip.remote-workspace.v1", epoch: 1, source: { kind: "empty" }, + writerRunId: randomUUID(), dirty: false, snapshotId: null, location: null }, + } : null }); + const prepare = vi.spyOn(remoteWorkspaces, "prepareRemoteWorkspace").mockImplementation(async args => ({ + workspace, lease: args.lease, remoteRoot: "/remote/workspace", reference: {} as never, + } as never)); + mockGetStoredWorkspace.mockResolvedValue(workspace); + try { + const runtime = makeMockRuntime({ supportsSync: () => false }); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: runtime }); + const acquisition = await orchestrator.acquireForRun(input); + const result = await orchestrator.realizeForRun({ + ...makeRealizeInput(), ...acquisition, adapterType: input.adapterType, persistedExecutionWorkspace: workspace, + }); + expect(mockUpdateLeaseMetadata).toHaveBeenCalled(); + for (const [, metadata] of mockUpdateLeaseMetadata.mock.calls) { + expect(metadata.sandboxLeaseAcquisition).toEqual(history); + } + expect(result.lease.metadata!.sandboxLeaseAcquisition).toEqual(history); + expect(result.executionTarget).toMatchObject({ kind: "remote", transport: "sandbox", + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "original-sandbox" } }); + } finally { + prepare.mockRestore(); + resolveConfig.mockRestore(); + mockResolveEnvironmentExecutionTarget.mockReset(); + } + }); it("forwards the admitted warm idle window to the recovery resume", async () => { const savedLease = originalLease(); mockGetLease.mockResolvedValue(savedLease); @@ -1021,7 +1085,10 @@ describe("live remote runner lease reattachment", () => { expect(state).toBe("pending"); expect(ready).toBe(false); } finally { release(); } - expect((await pending).lease).toEqual(before); + const result = await pending; + expect(result.lease).toEqual(before); + expect(result.sandboxLeaseAcquisition).toEqual({ outcome: "resumed", providerLeaseId: "original-sandbox" }); + expect(savedLease).toEqual(before); expect(ready).toBe(true); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); expect(runtime.execute).not.toHaveBeenCalled(); @@ -1047,7 +1114,7 @@ describe("live remote runner lease reattachment", () => { providerLeaseId: "original-sandbox", metadata, })) }); const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); - expect(result.lease).toBe(savedLease); + expect(result.lease).toMatchObject({ id: savedLease.id, providerLeaseId: savedLease.providerLeaseId }); expect(result.lease.metadata?.remoteCwd).toBe("/remote/workspace"); expect(runtime.resumeRunLease).toHaveBeenCalledOnce(); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 2c920bd7a3e..941af0cbb5f 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -61,6 +61,7 @@ import { type AdapterExecutionTarget, type AdapterRemoteExecutionSpec, type AdapterWorkspaceRealization, + type SandboxLeaseAcquisition, } from "@paperclipai/adapter-utils/execution-target"; import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability"; import { buildWorkspaceRealizationRequest } from "./workspace-realization.js"; @@ -124,6 +125,7 @@ export interface EnvironmentAcquisitionResult { leaseContext: ReturnType; executionTransport: Record | null; pluginTaskDescription?: PluginTaskDescription; + sandboxLeaseAcquisition?: SandboxLeaseAcquisition; } export interface EnvironmentRealizationResult { @@ -379,6 +381,7 @@ export function environmentRunOrchestrator( runnerReconnectGraceMs, }; let leaseRecord: EnvironmentRuntimeLeaseRecord; + let sandboxLeaseAcquisition: SandboxLeaseAcquisition | undefined; const expected = input.reattachRemoteLease; if (expected) { // Reattachment keeps the original lease, including ephemeral leases. @@ -432,6 +435,7 @@ export function environmentRunOrchestrator( if (resumed.metadata?.expired === true) throw remoteRecoveryLeaseMismatch("environment_resume", "lease_expired"); // Resume confirms allocation readiness. Its optional metadata may // describe the allocation root; keep the validated lease's workspace. + sandboxLeaseAcquisition = { outcome: "resumed", providerLeaseId: expected.providerLeaseId }; } leaseRecord = { environment, lease, leaseContext: buildEnvironmentLeaseContext(input) }; } else { @@ -473,6 +477,7 @@ export function environmentRunOrchestrator( lease: leaseRecord.lease, leaseContext: leaseRecord.leaseContext, executionTransport, + ...(sandboxLeaseAcquisition ? { sandboxLeaseAcquisition } : {}), }; } @@ -517,6 +522,7 @@ export function environmentRunOrchestrator( async function realizeForRun(input: { environment: Environment; lease: EnvironmentLease; + sandboxLeaseAcquisition?: SandboxLeaseAcquisition; adapterType: string; companyId: string; issueId: string | null; @@ -544,15 +550,22 @@ export function environmentRunOrchestrator( effectiveExecutionWorkspaceMode, } = input; let { lease, persistedExecutionWorkspace } = input; + const withAcquisitionReceipt = (target: AdapterExecutionTarget | null): AdapterExecutionTarget | null => { + if (!input.sandboxLeaseAcquisition || target?.kind !== "remote" || target.transport !== "sandbox") return target; + if (lease.providerLeaseId !== input.sandboxLeaseAcquisition.providerLeaseId) { + throw remoteRecoveryLeaseMismatch("environment_resume", "provider_lease_id"); + } + return { ...target, sandboxLeaseAcquisition: input.sandboxLeaseAcquisition }; + }; const remoteAuthority = readRemoteWorkspaceAuthority(persistedExecutionWorkspace?.metadata); if (remoteAuthority && persistedExecutionWorkspace) { if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") { throw new EnvironmentRunError("unsupported_adapter_environment", "Remote workspace storage requires a qualified native sandbox"); } - const target = await resolveEnvironmentExecutionTarget({ + const target = withAcquisitionReceipt(await resolveEnvironmentExecutionTarget({ db, companyId, adapterType, environment, leaseId: lease.id, leaseMetadata: lease.metadata, lease, environmentRuntime, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null, - }); + })); if (!target || target.kind !== "remote" || target.transport !== "sandbox" || !target.runner || target.runnerTask) { throw new EnvironmentRunError("transport_resolution_failed", "Remote workspace storage requires a sandbox file transport"); } @@ -753,7 +766,7 @@ export function environmentRunOrchestrator( // Step 4: Resolve execution target for the adapter let executionTarget: AdapterExecutionTarget | null; try { - executionTarget = await resolveEnvironmentExecutionTarget({ + executionTarget = withAcquisitionReceipt(await resolveEnvironmentExecutionTarget({ db, companyId, adapterType, @@ -764,7 +777,7 @@ export function environmentRunOrchestrator( environmentRuntime, signal: input.signal, duplexObservabilityRecorder: input.duplexObservabilityRecorder ?? null, - }); + })); // The native plugin task bridge owns guest paths. Never fall back to host execution. if (executionWorkspace.providerBinding && (executionTarget?.kind !== "remote" || !("runnerTask" in executionTarget) || typeof executionTarget.runnerTask !== "function")) { diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index 99403f393b7..c773418f22b 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -1562,6 +1562,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace realizationResult = await timeWorkspace("workspace_environment_realize", () => envOrchestrator.realizeForRun({ environment: selectedEnvironment, lease: activeEnvironmentLease.lease, + sandboxLeaseAcquisition: acquiredEnvironment.sandboxLeaseAcquisition, adapterType: agent.adapterType, companyId: agent.companyId, issueId: issueId ?? null,