diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index cd87a9cdc2b..491417336c2 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -104,9 +104,10 @@ its DB checkpoint alone cannot replace those bytes. Per-turn application qualification under this disk loss awaits the provider-backup owner's durable storage contract; native controller recovery does not waive that dependency. -`rolling-web-deployment` definition 4 selects every registered native Daytona +`rolling-web-deployment` definition 6 selects every registered native Daytona profile in disposable per-turn environments; `rolling-web-deployment-warm` -uses the same profiles in reusable warm environments. Their +selects providers supported by the production managed warm-session capability +and explicitly sets both agent and environment policy to warm. Their provider-backup path uses encrypted object storage and a fenced database ledger. The earlier per-turn failure remains failed evidence; this definition requires a fresh passing run, including the strict post-replacement continuation oracle. @@ -115,7 +116,9 @@ the replacement time to become ready while useful agent work remains active. The observer rechecks the original live owner immediately before switching traffic and fails if that turn already settled. A completed turn cannot count as active takeover. Per-turn file inspection requires the exact committed saved -snapshot; the warm lane still requires the live workspace. +snapshot. Completed warm turns also read that exact saved snapshot, while their +workspace location must remain bound to the retained sandbox. The API's saved +file response is independent of whether the sandbox is retained or retired. Cold restores retain the published workspace directory name, including after the old allocation is destroyed, because provider recovery binds the working directory into session identity. Unpublished preparations retain distinct diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index e3e3162ac9c..08bcdfb8ffc 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1270,9 +1270,9 @@ const rollingDeploymentSuites: RunnerSuiteFixture[] = [ description: "Overlap independent ephemeral web servers while a remote execution stays active; keep ordinary reads, saves, files and a later native continuation working.", profiles: id === ROLLING_WEB_DEPLOYMENT_WARM_SUITE ? rollingWarmDeploymentProfiles : rollingDeploymentProfiles, environments: [environment], expectedMatrixSize: (id === ROLLING_WEB_DEPLOYMENT_WARM_SUITE ? rollingWarmDeploymentProfiles : rollingDeploymentProfiles).length * remoteWorkspaceStorageSuite.tasks.length, - definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 5, controllerDowntimeMs: 0, + definitionMetadata: { ...remoteWorkspaceStorageSuite.definitionMetadata, version: 6, controllerDowntimeMs: 0, activeWorkBoundary: "foreground-180-second-cli-reliability-soak-and-live-owner-recheck", - workspaceFileSource: environment.configurationKey ? "live" : "saved-exact-snapshot-after-per-turn-retirement", + workspaceFileSource: environment.configurationKey ? "saved-exact-snapshot-with-retained-location" : "saved-exact-snapshot-after-per-turn-retirement", lifecycleCoverage: environment.configurationKey ? "warm-reusable" : "per-turn-disposable", providerCoverage: "registered-native-daytona-profiles", replacement: "ready-before-traffic-switch-sigterm-delete-entire-home", upgradeReadiness: "active-remote-workspace-recoverable-before-handoff", diff --git a/tests/runner-e2e/everyday-flow.ts b/tests/runner-e2e/everyday-flow.ts index ccd87e9ef89..84f0f328778 100644 --- a/tests/runner-e2e/everyday-flow.ts +++ b/tests/runner-e2e/everyday-flow.ts @@ -186,8 +186,12 @@ export async function runEverydayFlow(input: Input) { const controllerOutage = controllerOutageForExecution(execution, process.env); const requireRemoteRestart = controllerOutage !== undefined || isRollingWebDeployment(execution); const remoteWorkspaceStorage = isRemoteWorkspaceStorageExecution(execution); - if (remoteWorkspaceStorage) ev.remoteWorkspaceStorage = { browserReadPhases: [], - expectedFileSource: execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable" ? "saved" : "live" }; + if (remoteWorkspaceStorage) { + const perTurn = execution.suite.definitionMetadata?.lifecycleCoverage === "per-turn-disposable"; + ev.remoteWorkspaceStorage = { browserReadPhases: [], + expectedFileSource: perTurn || isRollingWebDeployment(execution) ? "saved" : "live", + expectedLocation: perTurn ? "released" : "retained" }; + } const providerChoice = caseId === "provider-decline" || caseId === "provider-second"; const nativeProviderCase = caseId === "provider-native"; let aggregatorFixture: Awaited> | undefined; @@ -564,7 +568,8 @@ export async function runEverydayFlow(input: Input) { label: `remote workspace ${phase} saved version`, deadlineAt: input.deadlineAt, load: () => readRemoteWorkspaceStorageCheckpoint({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: parent!.id, environmentId: fixtures.environment.id, - expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource }), + expectedFileSource: ev.remoteWorkspaceStorage!.expectedFileSource, + expectedLocation: ev.remoteWorkspaceStorage!.expectedLocation }), accept: observed => Boolean(observed.point), }); ev.remoteWorkspaceStorage![phase] = point.point!; diff --git a/tests/runner-e2e/remote-workspace-storage.test.ts b/tests/runner-e2e/remote-workspace-storage.test.ts index 4fa50e72462..88701da4359 100644 --- a/tests/runner-e2e/remote-workspace-storage.test.ts +++ b/tests/runner-e2e/remote-workspace-storage.test.ts @@ -86,6 +86,34 @@ describe("remote workspace Product E2E qualification", () => { expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false); }); + it("reads a completed warm turn from its saved snapshot while retaining the bound workspace location", async () => { + const f = checkpointApi(); + const savedFile = { ...f.file, resource: { ...f.file.resource, + source: { kind: "saved", snapshotId: f.receipt.snapshotId } } }; + const location = f.workspace.workspace.metadata.remoteWorkspace.location; + const originalGet = f.api.get; + const api = { get: async (route: string): Promise => { + if (route.includes("file-resources/content")) return savedFile as T; + if (route.endsWith("/leases")) return [{ id: location.leaseId, companyId: f.context.companyId, + issueId: f.context.issueId, providerLeaseId: location.providerLeaseId }] as T; + return originalGet(route); + } }; + const input = { ...f.context, api, expectedFileSource: "saved" as const, expectedLocation: "retained" as const }; + expect((await readRemoteWorkspaceStorageCheckpoint(input)).point).toMatchObject({ + fileSource: "saved", fileSnapshotId: f.receipt.snapshotId, locationState: "retained" }); + expect(await readRemoteWorkspaceStorageCheckpoint({ ...input, expectedLocation: "released" })).toEqual({}); + location.environmentId = id("5"); + expect(await readRemoteWorkspaceStorageCheckpoint(input)).toEqual({}); + const evidence = goodEvidence(); + evidence.expectedFileSource = "saved"; + evidence.expectedLocation = "retained"; + for (const point of [evidence.initial!, evidence.final!]) Object.assign(point, { + fileSource: "saved", fileSnapshotId: point.snapshotId, locationState: "retained" }); + expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(true); + evidence.final!.locationState = "released"; + expect(gradeRemoteWorkspaceStorage(evidence).every(check => check.passed)).toBe(false); + }); + it.each(["run-id", "company", "agent", "native-issue", "runtime", "status", "schema", "workspace", "epoch", "snapshot", "save-state", "lease", "provider", "remote-root"] as const)( "rejects a full-run %s mismatch even when its public summary succeeds", async mismatch => { diff --git a/tests/runner-e2e/remote-workspace-storage.ts b/tests/runner-e2e/remote-workspace-storage.ts index 3f17cb67ee8..2e2924a10c6 100644 --- a/tests/runner-e2e/remote-workspace-storage.ts +++ b/tests/runner-e2e/remote-workspace-storage.ts @@ -25,10 +25,12 @@ export interface RemoteWorkspaceStorageCheckpoint { proofSha256: string; fileSource: string; fileSnapshotId?: string; + locationState?: "retained" | "released"; } export interface RemoteWorkspaceStorageEvidence { expectedFileSource?: "live" | "saved"; + expectedLocation?: "retained" | "released"; initial?: RemoteWorkspaceStorageCheckpoint; final?: RemoteWorkspaceStorageCheckpoint; browserReadPhases: string[]; @@ -51,6 +53,7 @@ export function remoteWorkspaceProofDigest(content: string): string { export async function readRemoteWorkspaceStorageCheckpoint(input: { api: Pick; companyId: string; agentId: string; issueId: string; environmentId: string; expectedFileSource?: "live" | "saved"; + expectedLocation?: "retained" | "released"; }): Promise<{ point?: RemoteWorkspaceStorageCheckpoint; content?: string }> { const view = await input.api.get<{ issueId: string; workspace: ExecutionWorkspace | null }>(`/api/issues/${input.issueId}/workspace`); const workspace = view.workspace; @@ -59,9 +62,10 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: { || workspace.providerType !== "cloud_sandbox" || !authorityResult.success) return {}; const authority = authorityResult.data; const expectedSource = input.expectedFileSource ?? "live"; + const expectedLocation = input.expectedLocation ?? (expectedSource === "saved" ? "released" : "retained"); if (!authority.snapshotId || authority.dirty || authority.writerRunId) return {}; - if (expectedSource === "live" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {}; - if (expectedSource === "saved" && authority.location !== null) return {}; + if (expectedLocation === "retained" && (!authority.location || authority.location.environmentId !== input.environmentId)) return {}; + if (expectedLocation === "released" && authority.location !== null) return {}; const runs = await input.api.get(`/api/issues/${input.issueId}/runs`); const latest = runs.filter(run => run.runtimeMode === "native" && run.status === "succeeded" && run.agentId === input.agentId && typeof run.runId === "string") @@ -95,6 +99,7 @@ export async function readRemoteWorkspaceStorageCheckpoint(input: { authoritySchema: authority.schema, authorityDirty: authority.dirty, authorityWriter: authority.writerRunId, providerType: workspace.providerType, proofSha256: remoteWorkspaceProofDigest(file.content.data), fileSource: file.resource?.source?.kind, fileSnapshotId, + locationState: authority.location ? "retained" : "released", } }; } @@ -106,6 +111,7 @@ export function gradeRemoteWorkspaceStorage(evidence: RemoteWorkspaceStorageEvid && point.authoritySchema === "paperclip.remote-workspace.v1" && point.authorityDirty === false && point.authorityWriter === null && point.providerType === "cloud_sandbox" && point.fileSource === (evidence.expectedFileSource ?? "live") + && (!evidence.expectedLocation || point.locationState === evidence.expectedLocation) && (point.fileSource !== "saved" || point.fileSnapshotId === point.snapshotId) && /^[a-f0-9]{64}$/.test(point.proofSha256)); return [