diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index fdc9b560867..84ec0f5c73e 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -313,6 +313,15 @@ The general everyday suite retains its 125-second default. These intervals are fault tests, not permitted HTTP deployment downtime. Both fixtures retain the workspace and remote sandbox; they do not prove whole-webserver disk loss. +The dedicated suite's version 6 uses the existing reusable warm environment, +with a 15-minute provider idle window and unchanged 60/300/420-second ownership, +outage and reconnect limits. Its prior per-turn/no-reuse measurement failed +retained native/provider identity and remains failed. The test retires the whole +`runtime/paperclip-runner` directory, including provider failover-backup bytes. +Per-turn application qualification after sandbox disposal and this disk loss +therefore awaits the provider-backup owner's durable storage contract; logical +native checkpoints do not replace provider history or weaken the identity oracle. + The rollback reader must support both the recovery envelope and the settled PRP v2 idle goal suffix. Record its actual source commit and installed tree hash. An older reader rejecting that journal is an unsupported rollback, not a pass. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 7c0ee87b9f4..431e03883b8 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -10,7 +10,8 @@ qualification. The explicit-only `daytona-restart-continuity` suite runs the existing `recover-controller` journey with the recorded Codex and Claude staging models -on fresh Daytona fixtures. After a completed task tool in an active revision +on fresh reusable warm Daytona fixtures. The sandbox, Runner and provider history +must survive both controller replacements. After a completed task tool in an active revision turn, it keeps the owned development controller offline for 300 seconds. This exceeds the 60-second ownership lease and remains within the Runner's 420-second reconnect grace. The existing independent oracle @@ -28,6 +29,15 @@ ownership/reconnect limits, archive/delete backstops and owned cleanup remain un The retained continuation accepts whitespace between its exact marker and the exact `queue_r` output token; readiness and final grading use the same matcher. +Definition version 6 uses the existing warm reusable environment with a dedicated +configuration key. The prior version 5 per-turn/no-reuse measurement **failed** +retained native/provider identity and remains failed; it does not qualify this +warm fixture. Cache retirement moves the entire controller Runner directory, +including provider failover-backup bytes. Once a per-turn sandbox is disposed, +its DB checkpoint alone cannot replace those bytes. Per-turn application +qualification under this disk loss awaits the provider-backup owner's durable +storage contract; native controller recovery does not waive that dependency. + The explicit-only [GitHub review bot journey](GITHUB-REVIEW-JOURNEY.md) tests attended bot setup and an automatic PR review against an attached deployment. Private account bindings and setup receipts live in the private eval repository. diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 64628298d7f..f2e24e085d9 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -33,7 +33,7 @@ import { } from "./selectors.js"; describe("runner E2E catalog", () => { - it("keeps the Daytona controller outage inside the provider idle lifetime", () => { + it("keeps the Daytona controller outage on a reusable warm sandbox inside its idle lifetime", () => { const suite = runnerSuites.find(candidate => candidate.id === "daytona-restart-continuity")!; const secretRefs = { OPENAI_API_KEY: { type: "secret_ref" as const, secretId: "22222222-2222-4222-8222-222222222222", version: "latest" as const }, @@ -43,14 +43,21 @@ describe("runner E2E catalog", () => { const environmentInput = { executionId: "controller-outage", secretRefs, daytonaImage: `fixture@sha256:${"a".repeat(64)}` }; const base = runnerEnvironments.find(environment => environment.id === "daytona")!; const baseConfig = base.buildEnvironment(environmentInput).config as Record; + const warmConfig = daytonaWarmEnvironment.buildEnvironment(environmentInput).config as Record; const cells = runnerMatrix.filter(entry => entry.suite.id === suite.id); expect(cells).toHaveLength(2); + expect(cells.map(cell => cell.id)).toEqual([ + "daytona-restart-continuity.runner-codex.daytona.recover-controller", + "daytona-restart-continuity.runner-acpx-claude.daytona.recover-controller", + ]); for (const cell of cells) { const config = cell.environment.buildEnvironment(environmentInput).config as Record; + expect(config).toMatchObject({ reuseLease: true, runnerLifecycleMode: "warm" }); expect(config.autoStopInterval).toBe(15); expect(Number(config.autoStopInterval) * 60_000).toBeGreaterThan(420_000); - expect(config).toEqual({ ...baseConfig, autoStopInterval: 15 }); - expect(cell.environment.configurationKey).toBe("controller-outage-v1"); + expect(config).toEqual({ ...warmConfig, autoStopInterval: 15 }); + expect(cell.environment.configurationKey).toBe("warm-controller-outage-v1"); + expect(cell.environment.label).toContain("warm reusable"); const agentInput = { ...environmentInput, environmentId: "env-1", environmentFixtureId: "daytona" as const, workspacePath: "/workspace" }; const adapterConfig = cell.profile.buildAgent(agentInput).adapterConfig as Record; const original = runnerProfiles.find(profile => profile.id === cell.profile.id)!.buildAgent(agentInput).adapterConfig as Record; @@ -58,10 +65,13 @@ describe("runner E2E catalog", () => { expect(adapterConfig.lifecycleMode).toBe(original.lifecycleMode); expect(cell.suite.manualOnly).toBe(true); expect(cell.task.automaticRetryPolicy).toBe("single_attempt"); + expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(entry => entry.id)).toEqual([cell.id]); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(entry => entry.id === cell.id)).toBe(false); } - expect(baseConfig.autoStopInterval).toBe(5); - expect(daytonaWarmEnvironment.buildEnvironment(environmentInput).config).toMatchObject({ autoStopInterval: 5 }); - expect(suite.definitionMetadata).toMatchObject({ version: 5, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15 }); + expect(baseConfig).toMatchObject({ autoStopInterval: 5, reuseLease: false, runnerLifecycleMode: "per_turn" }); + expect(warmConfig).toMatchObject({ autoStopInterval: 5, reuseLease: true, runnerLifecycleMode: "warm" }); + expect(suite.definitionMetadata).toMatchObject({ version: 6, controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, + retainedControllerContinuation: true, maximumAttemptsPerCell: 1 }); }); it.each([ diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index ebaf9b291af..485324102de 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1159,21 +1159,21 @@ const repositoryModelProfiles = [ export const runnerSuites: readonly RunnerSuiteFixture[] = [ { - id: "daytona-restart-continuity", label: "Daytona controller outage continuity", manualOnly: true, - description: "Keep the original active native runner, task tools and saved files through five minutes without a controller, then continue the completed session after another replacement.", + id: "daytona-restart-continuity", label: "Daytona warm controller outage continuity", manualOnly: true, + description: "Keep the original active native runner, reusable warm sandbox, task tools and saved files through five minutes without a controller, then continue the completed session after another replacement.", groups: [], profiles: repositoryModelProfiles.map(({ native }) => productionStoryProfile(native)), - environments: runnerEnvironments.filter(environment => environment.id === "daytona").map(environment => ({ - ...environment, - configurationKey: "controller-outage-v1", + environments: [{ + ...daytonaWarmEnvironment, + configurationKey: "warm-controller-outage-v1", buildEnvironment(input: EnvironmentFixtureBuildInput) { - const payload = environment.buildEnvironment(input); + const payload = daytonaWarmEnvironment.buildEnvironment(input); // Keep provider idle stop beyond the outage and replacement budget. return { ...payload, config: { ...(payload.config as Record), autoStopInterval: 15 } }; }, - })), + }], tasks: everydayTasks.filter(task => task.id === "recover-controller") .map(task => ({ ...task, expectedRunCount: task.expectedRunCount + 1, automaticRetryPolicy: "single_attempt" as const })), expectedMatrixSize: 2, - definitionMetadata: { version: 5, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, + definitionMetadata: { version: 6, scheduling: "explicit-only", controllerDowntimeMs: 300_000, autoStopIntervalMinutes: 15, retainedControllerContinuation: true, maximumAttemptsPerCell: 1, budgetMonthlyCents: 1000, instructions: "production", grading: "same-active-process-lease-native-prefix-and-workspace-receipt", authentication: "api-key", stagingAuthentication: "subscription", stagingQualification: false }, },