test: verify saved warm files with retained workspace binding

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -104,9 +104,10 @@ its DB checkpoint alone cannot replace those bytes. Per-turn application
 qualification under this disk loss awaits the provider-backup owner's durable
 storage contract; native controller recovery does not waive that dependency.
 
-`rolling-web-deployment` definition 4 selects every registered native Daytona
+`rolling-web-deployment` definition 6 selects every registered native Daytona
 profile in disposable per-turn environments; `rolling-web-deployment-warm`
-uses the same profiles in reusable warm environments. Their
+selects providers supported by the production managed warm-session capability
+and explicitly sets both agent and environment policy to warm. Their
 provider-backup path uses encrypted object storage and a fenced database ledger.
 The earlier per-turn failure remains failed evidence; this definition requires
 a fresh passing run, including the strict post-replacement continuation oracle.
@@ -115,7 +116,9 @@ the replacement time to become ready while useful agent work remains active.
 The observer rechecks the original live owner immediately before switching
 traffic and fails if that turn already settled. A completed turn cannot count
 as active takeover. Per-turn file inspection requires the exact committed saved
-snapshot; the warm lane still requires the live workspace.
+snapshot. Completed warm turns also read that exact saved snapshot, while their
+workspace location must remain bound to the retained sandbox. The API's saved
+file response is independent of whether the sandbox is retained or retired.
 Cold restores retain the published workspace directory name, including after
 the old allocation is destroyed, because provider recovery binds the working
 directory into session identity. Unpublished preparations retain distinct

tests/runner-e2e/catalog.ts

@@ -1270,9 +1270,9 @@ const rollingDeploymentSuites: RunnerSuiteFixture[] = [
   description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.",
   profiles: id === ROLLING_WEB_DEPLOYMENT_WARM_SUITE ? rollingWarmDeploymentProfiles : rollingDeploymentProfiles, environments: [environment],
   expectedMatrixSize: (id === ROLLING_WEB_DEPLOYMENT_WARM_SUITE ? rollingWarmDeploymentProfiles : rollingDeploymentProfiles).length * remoteWorkspaceStorageSuite.tasks.length,
-  definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 5, controllerDowntimeMs: 0,
+  definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 6, controllerDowntimeMs: 0,
     activeWorkBoundary: "foreground-180-second-cli-reliability-soak-and-live-owner-recheck",
-    workspaceFileSource: environment.configurationKey ? "live" : "saved-exact-snapshot-after-per-turn-retirement",
+    workspaceFileSource: environment.configurationKey ? "saved-exact-snapshot-with-retained-location" : "saved-exact-snapshot-after-per-turn-retirement",
     lifecycleCoverage: environment.configurationKey ? "warm-reusable" : "per-turn-disposable", providerCoverage: "registered-native-daytona-profiles",
     replacement: "ready-before-traffic-switch-sigterm-delete-entire-home",
     upgradeReadiness: "active-remote-workspace-recoverable-before-handoff",

tests/runner-e2e/everyday-flow.ts

@@ -186,8 +186,12 @@ export async function runEverydayFlow(input: Input) {
   const controllerOutage = controllerOutageForExecution(execution, process.env);
   const requireRemoteRestart = controllerOutage !== undefined || isRollingWebDeployment(execution);
   const remoteWorkspaceStorage = isRemoteWorkspaceStorageExecution(execution);
-  if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [],
-    expectedFileSource: execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable" ? "saved" : "live" };
+  if (remoteWorkspaceStorage) {
+    const perTurn = execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable";
+    ev.remoteWorkspaceStorage = { browserReadPhases: [],
+      expectedFileSource: perTurn || isRollingWebDeployment(execution) ? "saved" : "live",
+      expectedLocation: perTurn ? "released" : "retained" };
+  }
   const providerChoice = caseId === "provider-decline" || caseId === "provider-second";
   const nativeProviderCase = caseId === "provider-native";
   let aggregatorFixture: Awaited<ReturnType<typeof setupAggregatorFixture>> | undefined;
@@ -564,7 +568,8 @@ export async function runEverydayFlow(input: Input) {
       label: `remote workspace ${phase} saved version`, deadlineAt: input.deadlineAt,
       load: () => readRemoteWorkspaceStorageCheckpoint({ api, companyId: fixtures.company.id,
         agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id,
-        expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource }),
+        expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource,
+        expectedLocation: ev.remoteWorkspaceStorage!.expectedLocation }),
       accept: observed => Boolean(observed.point),
     });
     ev.remoteWorkspaceStorage![phase] = point.point!;

tests/runner-e2e/remote-workspace-storage.test.ts

@@ -86,6 +86,34 @@ describe("remote workspace Product E2E qualification", () => {
     expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false);
   });
 
+  it("reads a completed warm turn from its saved snapshot while retaining the bound workspace location", async () => {
+    const f = checkpointApi();
+    const savedFile = { ...f.file, resource: { ...f.file.resource,
+      source: { kind: "saved", snapshotId: f.receipt.snapshotId } } };
+    const location = f.workspace.workspace.metadata.remoteWorkspace.location;
+    const originalGet = f.api.get;
+    const api = { get: async <T>(route: string): Promise<T> => {
+      if (route.includes("file-resources/content")) return savedFile as T;
+      if (route.endsWith("/leases")) return [{ id: location.leaseId, companyId: f.context.companyId,
+        issueId: f.context.issueId, providerLeaseId: location.providerLeaseId }] as T;
+      return originalGet<T>(route);
+    } };
+    const input = { ...f.context, api, expectedFileSource: "saved" as const, expectedLocation: "retained" as const };
+    expect((await readRemoteWorkspaceStorageCheckpoint(input)).point).toMatchObject({
+      fileSource: "saved", fileSnapshotId: f.receipt.snapshotId, locationState: "retained" });
+    expect(await readRemoteWorkspaceStorageCheckpoint({ ...input, expectedLocation: "released" })).toEqual({});
+    location.environmentId = id("5");
+    expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({});
+    const evidence = goodEvidence();
+    evidence.expectedFileSource = "saved";
+    evidence.expectedLocation = "retained";
+    for (const point of [evidence.initial!, evidence.final!]) Object.assign(point, {
+      fileSource: "saved", fileSnapshotId: point.snapshotId, locationState: "retained" });
+    expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(true);
+    evidence.final!.locationState = "released";
+    expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false);
+  });
+
   it.each(["run-id", "company", "agent", "native-issue", "runtime", "status", "schema", "workspace",
     "epoch", "snapshot", "save-state", "lease", "provider", "remote-root"] as const)(
     "rejects a full-run %s mismatch even when its public summary succeeds", async mismatch => {

tests/runner-e2e/remote-workspace-storage.ts

@@ -25,10 +25,12 @@ export interface RemoteWorkspaceStorageCheckpoint {
   proofSha256: string;
   fileSource: string;
   fileSnapshotId?: string;
+  locationState?: "retained" | "released";
 }
 
 export interface RemoteWorkspaceStorageEvidence {
   expectedFileSource?: "live" | "saved";
+  expectedLocation?: "retained" | "released";
   initial?: RemoteWorkspaceStorageCheckpoint;
   final?: RemoteWorkspaceStorageCheckpoint;
   browserReadPhases: string[];
@@ -51,6 +53,7 @@ export function remoteWorkspaceProofDigest(content: string): string {
 export async function readRemoteWorkspaceStorageCheckpoint(input: {
   api: Pick<RunnerApi, "get">; companyId: string; agentId: string; issueId: string; environmentId: string;
   expectedFileSource?: "live" | "saved";
+  expectedLocation?: "retained" | "released";
 }): Promise<{ point?: RemoteWorkspaceStorageCheckpoint; content?: string }> {
   const view = await input.api.get<{ issueId: string; workspace: ExecutionWorkspace | null }>(`/api/issues/${input.issueId}/workspace`);
   const workspace = view.workspace;
@@ -59,9 +62,10 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: {
     || workspace.providerType !== "cloud_sandbox" || !authorityResult.success) return {};
   const authority = authorityResult.data;
   const expectedSource = input.expectedFileSource ?? "live";
+  const expectedLocation = input.expectedLocation ?? (expectedSource === "saved" ? "released" : "retained");
   if (!authority.snapshotId || authority.dirty || authority.writerRunId) return {};
-  if (expectedSource === "live" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {};
-  if (expectedSource === "saved" && authority.location !== null) return {};
+  if (expectedLocation === "retained" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {};
+  if (expectedLocation === "released" && authority.location !== null) return {};
   const runs = await input.api.get<RunSummary[]>(`/api/issues/${input.issueId}/runs`);
   const latest = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded"
     && run.agentId === input.agentId && typeof run.runId === "string")
@@ -95,6 +99,7 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: {
     authoritySchema: authority.schema, authorityDirty: authority.dirty,
     authorityWriter: authority.writerRunId, providerType: workspace.providerType,
     proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind, fileSnapshotId,
+    locationState: authority.location ? "retained" : "released",
   } };
 }
 
@@ -106,6 +111,7 @@ export function gradeRemoteWorkspaceStorage(evidence: RemoteWorkspaceStorageEvid
     && point.authoritySchema === "paperclip.remote-workspace.v1" && point.authorityDirty === false
     && point.authorityWriter === null && point.providerType === "cloud_sandbox"
     && point.fileSource === (evidence.expectedFileSource ?? "live")
+    && (!evidence.expectedLocation || point.locationState === evidence.expectedLocation)
     && (point.fileSource !== "saved" || point.fileSnapshotId === point.snapshotId)
     && /^[a-f0-9]{64}$/.test(point.proofSha256));
   return [