From f5915fdc331110354f173f9236eaf2f40ae646ed Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 1 Nov 2026 08:05:02 -0600 Subject: [PATCH] test: isolate deployment probes from the business workflow Co-Authored-By: Paperclip --- .../2026-10-31-zero-downtime-web-deploys.md | 20 +++++++++++--- tests/runner-e2e/README.md | 4 ++- .../rolling-deployment-supervisor.test.ts | 27 ++++++++++++++++--- .../rolling-deployment-supervisor.ts | 4 ++- 4 files changed, 47 insertions(+), 8 deletions(-) diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index 322439952b0..a85a79356f5 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,6 +1,6 @@ # Zero-downtime web deployments -Date: 2026-10-31. Status (2026-11-01 13:54 UTC): existing-volume staging enrollment +Date: 2026-10-31. Status (2026-11-01 14:05 UTC): existing-volume staging enrollment is complete and its read/save/file-integrity checks pass. Active-agent recovery, natural database idling and the ten-deploy staging campaign remain unqualified. No production canary yet. @@ -21,7 +21,16 @@ validated target template. Fresh runs and ordinary later continuations still compose current instructions. The deleted-bundle reproduction failed before the fix, then passed with real backend session recovery and no new thread/turn start. All 59 backend factory tests and the focused checkpoint validation tests pass. -The next step is a clean rebuilt live retry and exact-head PR review. +At `cd690de4d3`, the real Codex/Daytona run survives removal of A's entire home, +reattaches the same runner/session under generation two, and succeeds. Its next +revision also succeeds. The handoff records 372 reads, 38 saves and 38 verified +saves with zero failures. Full qualification still fails: the supervisor created +an unrelated open probe task in the business company, so the unchanged all-tasks- +done oracle could never advance to the second replacement. The attempt was +interrupted and retained as incomplete. A regression reproduces this harness +scope defect; probes now use their own disposable company. The oracle is unchanged. +Exact-head review of the recovery fix is 5/5 with no unresolved threads; new +fixture checks and another bounded live attempt are next. Review also identified save quotas incorrectly applied during cache recovery. Recovery uses the existing inbound remote projection and preserves path/link @@ -46,7 +55,12 @@ not idle proof, and does not establish the source of provider activity. Automati idle sleep was correctly refused because this development fixture is not eligible for the free/demo policy. Manual sleep with force disabled passed: the registry reports sleeping, Railway has no active web deployment or attached volume, and -Neon is idle. Browser wake is being verified through the normal organization entry. +Neon is idle. The normal organization entry started wake, and Railway became healthy on the +same volume-free service. Post-wake proof passed 21 reads, three saves, a connected +WebSocket and all 34 file hashes. The browser displayed the retained task and saved +a new title, independently read back through the API. Automatic return from the +refreshing entry page was not verified; the browser navigated to the tenant after +readiness. Wake took several minutes while existing shared UI assets were checked. Private receipts remain under the task's live-test, staging enrollment and hosted probe directories; no credential or live target identity belongs here. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 30871455075..77d8deacc21 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -103,7 +103,9 @@ for `/api/ready`, moves ingress, sends SIGTERM to the predecessor, and deletes that predecessor's whole home and build. Cleanup makes read-only cache directories removable only after shutdown, without following dependency symlinks. Continuous ordinary task reads and comment saves must have zero failures, and every accepted save must be read back -exactly once. Forced process cleanup fails graceful qualification even if the +exactly once. Probe tasks live in a separate disposable company, so they cannot +block the business workflow's all-tasks-done oracle or consume its agent budget. +Forced process cleanup fails graceful qualification even if the successor recovers the execution. The read-only observer shares the controllers' disposable encryption key. It repeats replacement after completion before the retained-session follow-up. The five-minute outage suites are unchanged. diff --git a/tests/runner-e2e/rolling-deployment-supervisor.test.ts b/tests/runner-e2e/rolling-deployment-supervisor.test.ts index 13b7f9320a0..4ce8087c722 100644 --- a/tests/runner-e2e/rolling-deployment-supervisor.test.ts +++ b/tests/runner-e2e/rolling-deployment-supervisor.test.ts @@ -4,17 +4,17 @@ import { createLocalEncryptedCipher } from "../../server/src/secrets/local-encry const mocks = vi.hoisted(() => ({ start: vi.fn(), observer: vi.fn(), stop: vi.fn(), close: vi.fn(), writeFile: vi.fn(), })); -vi.mock("node:fs/promises", () => ({ mkdir: async () => {}, writeFile: mocks.writeFile, +vi.mock("node:fs/promises", () => ({ mkdir: async () => {}, rename: async () => {}, writeFile: mocks.writeFile, readFile: async () => JSON.stringify({ database: { mode: "postgres", connectionString: "postgres://fixture" }, secrets: { localEncrypted: { keyFilePath: "/fixture/secrets/master.key" } } }) })); vi.mock("../../packages/shared/src/config-schema.js", () => ({ paperclipConfigSchema: { parse: (value: unknown) => value } })); vi.mock("./rolling-deployment-process.js", () => ({ startRollingDeploymentProcess: mocks.start })); vi.mock("./rolling-deployment-proxy.js", () => ({ startRollingDeploymentProxy: async () => ({ origin: "http://127.0.0.1:1", routeTo() {}, close: mocks.close }) })); vi.mock("./controller-recovery-observer.js", () => ({ createControllerRecoveryObserver: mocks.observer })); -vi.mock("./server-restart-request.js", () => ({ readRestartRequest: async () => ({ requestId: "key-check", rollingDeployment: true, recoveryBinding: {} }) })); +vi.mock("./server-restart-request.js", () => ({ readRestartRequest: async () => ({ requestId: "key-check", rollingDeployment: true, recoveryBinding: { companyId: "business-company" } }) })); import { superviseRollingDeployment } from "./rolling-deployment-supervisor.js"; -afterEach(() => { vi.unstubAllEnvs(); vi.clearAllMocks(); }); +afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); vi.clearAllMocks(); }); it.each([undefined, "prior-disposable-key"])("shares the server key with checkpoint observation and restores %s on failure", async prior => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", prior); @@ -40,3 +40,24 @@ it.each([undefined, "prior-disposable-key"])("shares the server key with checkpo expect(mocks.close).toHaveBeenCalledOnce(); expect(process.env.PAPERCLIP_SECRETS_MASTER_KEY).toBe(prior); }); + + +it("keeps open deployment probe tasks out of the agent workflow's company", async () => { + mocks.start.mockResolvedValue({ child: { exitCode: null, signalCode: null }, configPath: "/fixture/config.json", + origin: "http://127.0.0.1:2", waitForReady: async () => {}, stop: mocks.stop }); + mocks.observer.mockResolvedValue(undefined); + const routes: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (url: string) => { + const route = new URL(url).pathname; + routes.push(route); + if (route === "/api/companies") return new Response(JSON.stringify({ id: "probe-company" })); + if (route === "/api/companies/probe-company/issues") throw new Error("probe fixture boundary reached"); + throw new Error(`Probe modified business workflow: ${route}`); + })); + await expect(superviseRollingDeployment({ temporaryRoot: "/fixture", repositoryRoot: "/fixture", + configPath: "/fixture/config.json", port: 1, executionIds: ["rolling"], cancelled: () => false, + environment: { PAPERCLIP_STORAGE_PROVIDER: "s3" }, + })).rejects.toThrow("probe fixture boundary reached"); + expect(routes).toEqual(["/api/companies", "/api/companies/probe-company/issues"]); + expect(mocks.stop).toHaveBeenCalledOnce(); +}); diff --git a/tests/runner-e2e/rolling-deployment-supervisor.ts b/tests/runner-e2e/rolling-deployment-supervisor.ts index 2b2597c9848..0db28f93792 100644 --- a/tests/runner-e2e/rolling-deployment-supervisor.ts +++ b/tests/runner-e2e/rolling-deployment-supervisor.ts @@ -77,7 +77,9 @@ export async function superviseRollingDeployment(input: { const writes: Array<{ id: string; body: string }> = []; try { if (!probeIssueId) { - const companyId = request.recoveryBinding?.companyId ?? (await api<{ id: string }>("/companies", "POST", { name: "Synthetic deployment probes" })).id; + // The business oracle observes every task in its company. Keep the + // unassigned read/save probe outside that workload and its agent budget. + const companyId = (await api<{ id: string }>("/companies", "POST", { name: "Synthetic deployment probes" })).id; const issue = await api<{ id: string }>(`/companies/${companyId}/issues`, "POST", { title: "Synthetic deployment read/save probe", description: "Unassigned fixture for ordinary user saves during deployment.", status: "todo", });