From 1d3f23723599be4f07709f1d164c534dfd72361b Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 30 Sep 2026 07:57:27 -0500 Subject: [PATCH] fix(e2e): bound remote bootstrap by the case deadline Wait through cold snapshot provisioning while rechecking exact task/run ownership and active lease admission. Fail promptly on terminal runs and retain bounded startup state. Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 9 +++ .../remote-native-bootstrap.test.ts | 78 ++++++++++++++++++- tests/runner-e2e/remote-native-bootstrap.ts | 71 +++++++++++------ 3 files changed, 131 insertions(+), 27 deletions(-) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c85f8aac051..652ea999b2c 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1349,6 +1349,15 @@ separate from the native operation under test; unknown or extra writes/commands still fail. SDK 0.203.0, immutable image and executable digests, public run/lease ownership, sandbox labels and workspace sentinel are verified before execution. +Bootstrap admission uses the existing authored case deadline, including cold +snapshot provisioning; it does not impose a separate 20-second lease deadline. +Every poll rechecks task/run ownership and run status, and requires exactly one +unambiguous active lease for that run and task. A stopped run fails immediately. +Startup failures save bounded, allowlisted run-stage and lease-count evidence; +this evidence does not infer a provider-side cause or change the machine grade. +The initial provider's instruction-file read window remains 20 seconds after +provider startup, and no action is published before the observer is armed. + The observer watches registered targets (including transient create/delete), user workspace changes and the exact runner process plus descendants. The single controller-owned `.paperclip-runtime/paperclip-runner` subtree is excluded from diff --git a/tests/runner-e2e/remote-native-bootstrap.test.ts b/tests/runner-e2e/remote-native-bootstrap.test.ts index ebca55f61b3..5bfed8401f3 100644 --- a/tests/runner-e2e/remote-native-bootstrap.test.ts +++ b/tests/runner-e2e/remote-native-bootstrap.test.ts @@ -1,11 +1,13 @@ -import { expect, it, vi } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js"; import type { RemoteFixtureApi, RemoteNativeFixture } from "./remote-native-fixtures.js"; -function harness() { +afterEach(() => vi.useRealTimers()); + +function harness(timeoutMs = 5000) { const order: string[] = []; const issue = { id: "issue", companyId: "company", assigneeAgentId: "agent" }; - const run = { id: "run", companyId: "company", agentId: "agent", status: "running" }; + const run = { id: "run", companyId: "company", agentId: "agent", status: "running", executionStage: "preparing" }; const leases = [{ id: "lease", heartbeatRunId: "run", issueId: "issue", status: "active", providerLeaseId: "sandbox" }]; const api = { get: vi.fn(async (path: string) => path === "/api/issues/issue" ? issue : path === "/api/heartbeat-runs/run" ? run : leases) }; const fixture = { @@ -18,7 +20,7 @@ function harness() { const input = { api: api as unknown as RemoteFixtureApi, daytona: { get: vi.fn() }, companyId: "company", environmentId: "env", agentId: "agent", image: `image@sha256:${"a".repeat(64)}`, nodeSha256: `sha256:${"b".repeat(64)}`, runnerdSha256: `sha256:${"c".repeat(64)}`, - deadlineAt: Date.now() + 5000, evidence: vi.fn(async () => { order.push("evidence"); }), + deadlineAt: Date.now() + timeoutMs, evidence: vi.fn(async (_name: string, _data: unknown) => { order.push("evidence"); }), }; const bootstrap = createRemoteNativeBootstrap(input, bind); const request = { issueId: "issue", runId: "run", targets: ["target.txt"], actionPrompt: async (actual: RemoteNativeFixture) => { @@ -95,3 +97,71 @@ it.each(["nodeSha256", "runnerdSha256", "image"])("requires immutable %s before expect(() => createRemoteNativeBootstrap({ ...h.input, [field]: "ambient-latest" }, h.bind)).toThrow("immutable"); expect(h.input.daytona.get).not.toHaveBeenCalled(); }); + + +it("admits a cold building_snapshot lease after 20 seconds within the unchanged case deadline", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const h = harness(60_000); h.bootstrap.prompt("cold"); + const original = h.api.get.getMockImplementation()!; + let sandboxState = "building_snapshot"; + h.api.get.mockImplementation(async path => path.endsWith("/leases") && sandboxState === "building_snapshot" ? [] : original(path)); + const delivery = h.bootstrap.bindAndRelease(h.request); + await vi.advanceTimersByTimeAsync(25_000); + expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled(); + sandboxState = "started"; + await vi.advanceTimersByTimeAsync(100); + await expect(delivery).resolves.toBe(h.fixture); + expect(h.bind).toHaveBeenCalledWith(expect.objectContaining({ deadlineAt: 60_000 })); + expect(h.input.daytona.get).not.toHaveBeenCalled(); +}); + +it.each(["failed", "cancelled", "succeeded"])("stops waiting immediately when provisioning run becomes %s", async status => { + vi.useFakeTimers(); vi.setSystemTime(0); + const h = harness(60_000); h.bootstrap.prompt("terminal"); h.leases.length = 0; + const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("stopped before observer setup"); + await vi.advanceTimersByTimeAsync(1000); h.run.status = status; + await vi.advanceTimersByTimeAsync(100); await delivery; + expect(Date.now()).toBeLessThan(h.input.deadlineAt); + expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled(); + expect(h.input.evidence).toHaveBeenCalledWith("remote-native-bootstrap-startup-run.json", expect.objectContaining({ runStatus: status, deadlineReached: false })); +}); + +it("rechecks run ownership while waiting for the lease", async () => { + vi.useFakeTimers(); vi.setSystemTime(0); + const h = harness(); h.bootstrap.prompt("owner"); h.leases.length = 0; + const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("ownership is unproven"); + await vi.advanceTimersByTimeAsync(100); h.run.companyId = "foreign"; + await vi.advanceTimersByTimeAsync(100); await delivery; + expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled(); +}); + +it.each(["foreign-run", "foreign-issue", "inactive", "missing-provider"])("never admits %s and stops at the authored deadline with bounded state", async variant => { + vi.useFakeTimers(); vi.setSystemTime(0); + const h = harness(1000); h.bootstrap.prompt("timeout"); + if (variant === "foreign-run") h.leases[0]!.heartbeatRunId = "foreign"; + if (variant === "foreign-issue") h.leases[0]!.issueId = "foreign"; + if (variant === "inactive") h.leases[0]!.status = "released"; + if (variant === "missing-provider") h.leases[0]!.providerLeaseId = ""; + h.run.executionStage = "PRIVATE".repeat(10_000); + const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Timed out waiting for owned native qualification lease"); + await vi.advanceTimersByTimeAsync(1000); await delivery; + expect(Date.now()).toBe(1000); expect(h.bind).not.toHaveBeenCalled(); + expect(h.fixture.publishAction).not.toHaveBeenCalled(); + const state = h.input.evidence.mock.calls[0]![1]; + expect(state).toMatchObject({ executionStage: "unknown", activeOwnedLeaseCount: 0, deadlineReached: true }); + expect(Buffer.byteLength(JSON.stringify(state))).toBeLessThan(512); + expect(JSON.stringify(state)).not.toContain("PRIVATE"); +}); + +it("rejects a second same-run lease even if only one lease is active", async () => { + const h = harness(); h.bootstrap.prompt("ambiguous"); + h.leases.push({ ...h.leases[0]!, id: "old", status: "released" }); + await expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Ambiguous native qualification lease"); + expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled(); +}); + +it("does not start observation after the case deadline", async () => { + const h = harness(0); h.bootstrap.prompt("expired"); + await expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Timed out waiting"); + expect(h.api.get).not.toHaveBeenCalled(); expect(h.bind).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/remote-native-bootstrap.ts b/tests/runner-e2e/remote-native-bootstrap.ts index 9aa0e7dbbd5..9c82eb844d8 100644 --- a/tests/runner-e2e/remote-native-bootstrap.ts +++ b/tests/runner-e2e/remote-native-bootstrap.ts @@ -61,29 +61,54 @@ export function createRemoteNativeBootstrap(input: { if (pending.length !== 1) throw new Error("Remote bootstrap requires one unconsumed instruction delivery"); const setup = pending[0]!; setup.consumed = true; if (![request.issueId, request.runId].every(id => /^[A-Za-z0-9_-]{1,128}$/u.test(id))) throw new Error("Invalid bootstrap task/run identity"); - await pollUntil({ - label: "owned native qualification run", deadlineAt: Math.min(input.deadlineAt, Date.now() + 20_000), intervalMs: 100, - load: async () => { - const [issue, run] = await Promise.all([ - input.api.get>(`/api/issues/${request.issueId}`), - input.api.get>(`/api/heartbeat-runs/${request.runId}`), - ]); - if (issue.id !== request.issueId || issue.companyId !== input.companyId || issue.assigneeAgentId !== input.agentId - || run.companyId !== input.companyId || run.agentId !== input.agentId || run.id !== request.runId) { - throw new Error("Remote bootstrap task/run ownership is unproven"); - } - return run; - }, - accept: run => run.status === "running", - reject: run => ["queued", "running"].includes(run.status) ? undefined : "Native qualification run stopped before observer setup", - }); - const leases = await pollUntil({ - label: "owned native qualification lease", deadlineAt: Math.min(input.deadlineAt, Date.now() + 20_000), intervalMs: 100, - load: () => input.api.get>>(`/api/environments/${input.environmentId}/leases`), - accept: rows => rows.filter(row => row.heartbeatRunId === request.runId && row.issueId === request.issueId && row.status === "active" && row.providerLeaseId).length === 1, - reject: rows => rows.filter(row => row.heartbeatRunId === request.runId).length > 1 ? "Ambiguous native qualification lease" : undefined, - }); - const lease = leases.find(row => row.heartbeatRunId === request.runId && row.issueId === request.issueId && row.status === "active")!; + // Provisioning consumes the authored case deadline. A cold snapshot need + // not produce a lease in 20 seconds; a terminal or foreign run never waits. + let lastState: Record = { observed: false }; + let lease: Record; + try { + const ready = await pollUntil({ + label: "owned native qualification lease", deadlineAt: input.deadlineAt, intervalMs: 100, + load: async () => { + const [issue, run, rows] = await Promise.all([ + input.api.get>(`/api/issues/${request.issueId}`), + input.api.get>(`/api/heartbeat-runs/${request.runId}`), + input.api.get>>(`/api/environments/${input.environmentId}/leases`), + ]); + const owned = issue.id === request.issueId && issue.companyId === input.companyId + && issue.assigneeAgentId === input.agentId && run.companyId === input.companyId + && run.agentId === input.agentId && run.id === request.runId; + const runLeases = rows.filter(row => row.heartbeatRunId === request.runId); + const active = runLeases.filter(row => row.issueId === request.issueId && row.status === "active" && row.providerLeaseId); + const rejection = !owned ? "Remote bootstrap task/run ownership is unproven" + : !["queued", "running"].includes(run.status) ? "Native qualification run stopped before observer setup" + : runLeases.length > 1 ? "Ambiguous native qualification lease" : undefined; + // Fixed keys and allowlisted enum values only: no provider IDs, + // errors, prompts or arbitrary API strings enter startup evidence. + lastState = { + observed: true, owned, + runStatus: ["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].includes(run.status) ? run.status : "unknown", + executionStage: ["queued", "preparing", "executing", "finalizing"].includes(run.executionStage) ? run.executionStage : "unknown", + runLeaseCount: runLeases.length, activeOwnedLeaseCount: active.length, + deadlineReached: Date.now() >= input.deadlineAt, + }; + return { run, lease: active[0], rejection }; + }, + accept: state => !state.rejection && state.run.status === "running" && Boolean(state.lease) + && Date.now() < input.deadlineAt, + reject: state => state.rejection, + timeoutDetail: () => JSON.stringify(lastState), + }); + lease = ready.lease!; + } catch (error) { + try { + await input.evidence(`remote-native-bootstrap-startup-${request.runId}.json`, { + ...lastState, deadlineReached: Date.now() >= input.deadlineAt, + }); + } catch (evidenceError) { + throw new AggregateError([error, evidenceError], "Remote bootstrap startup failed and state evidence could not be saved"); + } + throw error; + } const fixture = await bind({ api: input.api, daytona: input.daytona, sdkVersion: "0.203.0", nodeSha256: input.nodeSha256, runnerdSha256: input.runnerdSha256, authority: { companyId: input.companyId, environmentId: input.environmentId, runId: request.runId, leaseId: lease.id, sandboxId: lease.providerLeaseId, image: input.image }, targets: [...request.targets], crossRoot: request.crossRoot, actionFile: setup.path, deadlineAt: input.deadlineAt,