fix(e2e): bound remote bootstrap by the case deadline

Wait through cold snapshot provisioning while rechecking exact task/run ownership and active lease admission. Fail promptly on terminal runs and retain bounded startup state.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -1349,6 +1349,15 @@ separate from the native operation under test; unknown or extra writes/commands
 still fail. SDK 0.203.0, immutable image and executable digests, public run/lease
 ownership, sandbox labels and workspace sentinel are verified before execution.
 
+Bootstrap admission uses the existing authored case deadline, including cold
+snapshot provisioning; it does not impose a separate 20-second lease deadline.
+Every poll rechecks task/run ownership and run status, and requires exactly one
+unambiguous active lease for that run and task. A stopped run fails immediately.
+Startup failures save bounded, allowlisted run-stage and lease-count evidence;
+this evidence does not infer a provider-side cause or change the machine grade.
+The initial provider's instruction-file read window remains 20 seconds after
+provider startup, and no action is published before the observer is armed.
+
 The observer watches registered targets (including transient create/delete), user
 workspace changes and the exact runner process plus descendants. The single
 controller-owned `.paperclip-runtime/paperclip-runner` subtree is excluded from

tests/runner-e2e/remote-native-bootstrap.test.ts

@@ -1,11 +1,13 @@
-import { expect, it, vi } from "vitest";
+import { afterEach, expect, it, vi } from "vitest";
 import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js";
 import type { RemoteFixtureApi, RemoteNativeFixture } from "./remote-native-fixtures.js";
 
-function harness() {
+afterEach(() => vi.useRealTimers());
+
+function harness(timeoutMs = 5000) {
   const order: string[] = [];
   const issue = { id: "issue", companyId: "company", assigneeAgentId: "agent" };
-  const run = { id: "run", companyId: "company", agentId: "agent", status: "running" };
+  const run = { id: "run", companyId: "company", agentId: "agent", status: "running", executionStage: "preparing" };
   const leases = [{ id: "lease", heartbeatRunId: "run", issueId: "issue", status: "active", providerLeaseId: "sandbox" }];
   const api = { get: vi.fn(async (path: string) => path === "/api/issues/issue" ? issue : path === "/api/heartbeat-runs/run" ? run : leases) };
   const fixture = {
@@ -18,7 +20,7 @@ function harness() {
   const input = {
     api: api as unknown as RemoteFixtureApi, daytona: { get: vi.fn() }, companyId: "company", environmentId: "env", agentId: "agent",
     image: `image@sha256:${"a".repeat(64)}`, nodeSha256: `sha256:${"b".repeat(64)}`, runnerdSha256: `sha256:${"c".repeat(64)}`,
-    deadlineAt: Date.now() + 5000, evidence: vi.fn(async () => { order.push("evidence"); }),
+    deadlineAt: Date.now() + timeoutMs, evidence: vi.fn(async (_name: string, _data: unknown) => { order.push("evidence"); }),
   };
   const bootstrap = createRemoteNativeBootstrap(input, bind);
   const request = { issueId: "issue", runId: "run", targets: ["target.txt"], actionPrompt: async (actual: RemoteNativeFixture) => {
@@ -95,3 +97,71 @@ it.each(["nodeSha256", "runnerdSha256", "image"])("requires immutable %s before
   expect(() => createRemoteNativeBootstrap({ ...h.input, [field]: "ambient-latest" }, h.bind)).toThrow("immutable");
   expect(h.input.daytona.get).not.toHaveBeenCalled();
 });
+
+
+it("admits a cold building_snapshot lease after 20 seconds within the unchanged case deadline", async () => {
+  vi.useFakeTimers(); vi.setSystemTime(0);
+  const h = harness(60_000); h.bootstrap.prompt("cold");
+  const original = h.api.get.getMockImplementation()!;
+  let sandboxState = "building_snapshot";
+  h.api.get.mockImplementation(async path => path.endsWith("/leases") && sandboxState === "building_snapshot" ? [] : original(path));
+  const delivery = h.bootstrap.bindAndRelease(h.request);
+  await vi.advanceTimersByTimeAsync(25_000);
+  expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled();
+  sandboxState = "started";
+  await vi.advanceTimersByTimeAsync(100);
+  await expect(delivery).resolves.toBe(h.fixture);
+  expect(h.bind).toHaveBeenCalledWith(expect.objectContaining({ deadlineAt: 60_000 }));
+  expect(h.input.daytona.get).not.toHaveBeenCalled();
+});
+
+it.each(["failed", "cancelled", "succeeded"])("stops waiting immediately when provisioning run becomes %s", async status => {
+  vi.useFakeTimers(); vi.setSystemTime(0);
+  const h = harness(60_000); h.bootstrap.prompt("terminal"); h.leases.length = 0;
+  const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("stopped before observer setup");
+  await vi.advanceTimersByTimeAsync(1000); h.run.status = status;
+  await vi.advanceTimersByTimeAsync(100); await delivery;
+  expect(Date.now()).toBeLessThan(h.input.deadlineAt);
+  expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled();
+  expect(h.input.evidence).toHaveBeenCalledWith("remote-native-bootstrap-startup-run.json", expect.objectContaining({ runStatus: status, deadlineReached: false }));
+});
+
+it("rechecks run ownership while waiting for the lease", async () => {
+  vi.useFakeTimers(); vi.setSystemTime(0);
+  const h = harness(); h.bootstrap.prompt("owner"); h.leases.length = 0;
+  const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("ownership is unproven");
+  await vi.advanceTimersByTimeAsync(100); h.run.companyId = "foreign";
+  await vi.advanceTimersByTimeAsync(100); await delivery;
+  expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled();
+});
+
+it.each(["foreign-run", "foreign-issue", "inactive", "missing-provider"])("never admits %s and stops at the authored deadline with bounded state", async variant => {
+  vi.useFakeTimers(); vi.setSystemTime(0);
+  const h = harness(1000); h.bootstrap.prompt("timeout");
+  if (variant === "foreign-run") h.leases[0]!.heartbeatRunId = "foreign";
+  if (variant === "foreign-issue") h.leases[0]!.issueId = "foreign";
+  if (variant === "inactive") h.leases[0]!.status = "released";
+  if (variant === "missing-provider") h.leases[0]!.providerLeaseId = "";
+  h.run.executionStage = "PRIVATE".repeat(10_000);
+  const delivery = expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Timed out waiting for owned native qualification lease");
+  await vi.advanceTimersByTimeAsync(1000); await delivery;
+  expect(Date.now()).toBe(1000); expect(h.bind).not.toHaveBeenCalled();
+  expect(h.fixture.publishAction).not.toHaveBeenCalled();
+  const state = h.input.evidence.mock.calls[0]![1];
+  expect(state).toMatchObject({ executionStage: "unknown", activeOwnedLeaseCount: 0, deadlineReached: true });
+  expect(Buffer.byteLength(JSON.stringify(state))).toBeLessThan(512);
+  expect(JSON.stringify(state)).not.toContain("PRIVATE");
+});
+
+it("rejects a second same-run lease even if only one lease is active", async () => {
+  const h = harness(); h.bootstrap.prompt("ambiguous");
+  h.leases.push({ ...h.leases[0]!, id: "old", status: "released" });
+  await expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Ambiguous native qualification lease");
+  expect(h.bind).not.toHaveBeenCalled(); expect(h.fixture.publishAction).not.toHaveBeenCalled();
+});
+
+it("does not start observation after the case deadline", async () => {
+  const h = harness(0); h.bootstrap.prompt("expired");
+  await expect(h.bootstrap.bindAndRelease(h.request)).rejects.toThrow("Timed out waiting");
+  expect(h.api.get).not.toHaveBeenCalled(); expect(h.bind).not.toHaveBeenCalled();
+});

tests/runner-e2e/remote-native-bootstrap.ts

@@ -61,29 +61,54 @@ export function createRemoteNativeBootstrap(input: {
       if (pending.length !== 1) throw new Error("Remote bootstrap requires one unconsumed instruction delivery");
       const setup = pending[0]!; setup.consumed = true;
       if (![request.issueId, request.runId].every(id => /^[A-Za-z0-9_-]{1,128}$/u.test(id))) throw new Error("Invalid bootstrap task/run identity");
-      await pollUntil({
-        label: "owned native qualification run", deadlineAt: Math.min(input.deadlineAt, Date.now() + 20_000), intervalMs: 100,
-        load: async () => {
-          const [issue, run] = await Promise.all([
-            input.api.get<Record<string, any>>(`/api/issues/${request.issueId}`),
-            input.api.get<Record<string, any>>(`/api/heartbeat-runs/${request.runId}`),
-          ]);
-          if (issue.id !== request.issueId || issue.companyId !== input.companyId || issue.assigneeAgentId !== input.agentId
-            || run.companyId !== input.companyId || run.agentId !== input.agentId || run.id !== request.runId) {
-            throw new Error("Remote bootstrap task/run ownership is unproven");
-          }
-          return run;
-        },
-        accept: run => run.status === "running",
-        reject: run => ["queued", "running"].includes(run.status) ? undefined : "Native qualification run stopped before observer setup",
-      });
-      const leases = await pollUntil({
-        label: "owned native qualification lease", deadlineAt: Math.min(input.deadlineAt, Date.now() + 20_000), intervalMs: 100,
-        load: () => input.api.get<Array<Record<string, any>>>(`/api/environments/${input.environmentId}/leases`),
-        accept: rows => rows.filter(row => row.heartbeatRunId === request.runId && row.issueId === request.issueId && row.status === "active" && row.providerLeaseId).length === 1,
-        reject: rows => rows.filter(row => row.heartbeatRunId === request.runId).length > 1 ? "Ambiguous native qualification lease" : undefined,
-      });
-      const lease = leases.find(row => row.heartbeatRunId === request.runId && row.issueId === request.issueId && row.status === "active")!;
+      // Provisioning consumes the authored case deadline. A cold snapshot need
+      // not produce a lease in 20 seconds; a terminal or foreign run never waits.
+      let lastState: Record<string, string | number | boolean> = { observed: false };
+      let lease: Record<string, any>;
+      try {
+        const ready = await pollUntil({
+          label: "owned native qualification lease", deadlineAt: input.deadlineAt, intervalMs: 100,
+          load: async () => {
+            const [issue, run, rows] = await Promise.all([
+              input.api.get<Record<string, any>>(`/api/issues/${request.issueId}`),
+              input.api.get<Record<string, any>>(`/api/heartbeat-runs/${request.runId}`),
+              input.api.get<Array<Record<string, any>>>(`/api/environments/${input.environmentId}/leases`),
+            ]);
+            const owned = issue.id === request.issueId && issue.companyId === input.companyId
+              && issue.assigneeAgentId === input.agentId && run.companyId === input.companyId
+              && run.agentId === input.agentId && run.id === request.runId;
+            const runLeases = rows.filter(row => row.heartbeatRunId === request.runId);
+            const active = runLeases.filter(row => row.issueId === request.issueId && row.status === "active" && row.providerLeaseId);
+            const rejection = !owned ? "Remote bootstrap task/run ownership is unproven"
+              : !["queued", "running"].includes(run.status) ? "Native qualification run stopped before observer setup"
+              : runLeases.length > 1 ? "Ambiguous native qualification lease" : undefined;
+            // Fixed keys and allowlisted enum values only: no provider IDs,
+            // errors, prompts or arbitrary API strings enter startup evidence.
+            lastState = {
+              observed: true, owned,
+              runStatus: ["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].includes(run.status) ? run.status : "unknown",
+              executionStage: ["queued", "preparing", "executing", "finalizing"].includes(run.executionStage) ? run.executionStage : "unknown",
+              runLeaseCount: runLeases.length, activeOwnedLeaseCount: active.length,
+              deadlineReached: Date.now() >= input.deadlineAt,
+            };
+            return { run, lease: active[0], rejection };
+          },
+          accept: state => !state.rejection && state.run.status === "running" && Boolean(state.lease)
+            && Date.now() < input.deadlineAt,
+          reject: state => state.rejection,
+          timeoutDetail: () => JSON.stringify(lastState),
+        });
+        lease = ready.lease!;
+      } catch (error) {
+        try {
+          await input.evidence(`remote-native-bootstrap-startup-${request.runId}.json`, {
+            ...lastState, deadlineReached: Date.now() >= input.deadlineAt,
+          });
+        } catch (evidenceError) {
+          throw new AggregateError([error, evidenceError], "Remote bootstrap startup failed and state evidence could not be saved");
+        }
+        throw error;
+      }
       const fixture = await bind({ api: input.api, daytona: input.daytona, sdkVersion: "0.203.0", nodeSha256: input.nodeSha256, runnerdSha256: input.runnerdSha256,
         authority: { companyId: input.companyId, environmentId: input.environmentId, runId: request.runId, leaseId: lease.id, sandboxId: lease.providerLeaseId, image: input.image },
         targets: [...request.targets], crossRoot: request.crossRoot, actionFile: setup.path, deadlineAt: input.deadlineAt,