From 4c6adcadcbeb95ab3f756d88d16faadc86e22390 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 1 Oct 2026 19:26:25 -0500 Subject: [PATCH] test(runner-e2e): verify pending Pi input after provider loss Add one explicit Daytona Product cell that admits the exact Pi child before faulting it, then requires production expiry of the original native question, failed-run Blocked disposition, stale-answer rejection, distinct unanswered fallback, and no replay through owned retirement. Retain the existing local scope and pending qualification. Calibrate public lifecycle evidence, generated observer one-shot dispatch, and candidate failure classification. Runtime, provider profile, dependency closure, deadlines, and lockfiles are unchanged. Co-Authored-By: Paperclip --- doc/evals.md | 7 +- tests/runner-e2e/FIXTURES.md | 37 +++-- tests/runner-e2e/README.md | 21 ++- tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 6 +- tests/runner-e2e/failure-classifier.ts | 3 + tests/runner-e2e/pi-controls-evidence.test.ts | 12 +- tests/runner-e2e/pi-native-cases.test.ts | 10 +- tests/runner-e2e/pi-native-cases.ts | 8 +- tests/runner-e2e/pi-native-flow.ts | 17 ++- .../pi-native-provider-death-flow.test.ts | 116 +++++++++++++++ .../pi-native-provider-death-flow.ts | 135 ++++++++++++++++++ tests/runner-e2e/pi-native-restart-flow.ts | 12 +- .../runner-e2e/remote-native-fixtures.test.ts | 66 ++++++++- tests/runner-e2e/remote-native-fixtures.ts | 51 ++++++- 15 files changed, 462 insertions(+), 45 deletions(-) create mode 100644 tests/runner-e2e/pi-native-provider-death-flow.test.ts create mode 100644 tests/runner-e2e/pi-native-provider-death-flow.ts diff --git a/doc/evals.md b/doc/evals.md index 87410e8e1f7..194d99abf31 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -35,8 +35,11 @@ Pi's explicit-only `pi-controls` Product suite tests Stop while native permissio is unanswered and browser-originated same-turn steering, locally and on Daytona. Its [fixture contract](../tests/runner-e2e/FIXTURES.md#pi-active-controls) separates control acknowledgment, actual message consumption and owned process retirement. -The current Pi matrix has 25 explicit cells (13 local and 12 Daytona), including -these four controls and pending-native-question controller restart. File editing +The current Pi matrix has 26 explicit cells (13 local and 13 Daytona), including +these four controls, pending-native-question controller restart, and a Daytona-only +exact-Pi-child provider-death journey. Provider death must expire the original +unanswered card and reject stale replies without a replacement run; any durable +fallback is distinct from restoration. File editing also requires native edit/validation events and a registered artifact download. All remain unqualified until measured on the exact candidate runtime and harness. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index f360da075fc..3f1867bdca2 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -350,21 +350,40 @@ a candidate failure; transport/infrastructure failures retain the existing harness classification. Positive and plausible-wrong/missing-evidence unit calibrations do not count as paid qualification. All automatic retries stay zero. -Provider-death coverage remains bounded: the existing patched-ACP package test -`provider exit fails promptly and missing owned extension fails admission` runs -an actual wrapper child against a deterministic fake Pi child. The bridge test -`expires pending requests and discards a late answer exactly once` and driver -abort/event-stream tests separately cover pending cleanup with mocked transport. -They do not establish a single actual native-provider-death to durable browser -request-expiry chain. No local or Daytona paid death case is claimed, and no -arbitrary process-kill API is introduced by these fixtures. +`native-pending-provider-death` adds one real Daytona-only Product journey. +After the native input is publicly durable and still unanswered, the existing +owned remote observer admits the exact Pi child through its verified wrapper +parent, source-pinned closure files, executable inode, workspace, run/lease/session +ancestry and fresh PID/start-time checks. Pi overwrites Linux argv via +`process.title`, so the private receipt explicitly uses pinned-parent entrypoint +attribution and never claims original child argv. A pidfd targets only that child; +worker death, broad process-name matching and controller Stop cannot substitute. + +The runtime itself must emit `runtime_request.expired` for the original callback +with `provider_process_lost` and `replayAllowed:false`, followed by native turn +failure. The permanent failed-terminal recovery projection must put the owned +issue in Blocked, proved through API and browser and retained through cleanup. +The original durable card must expire, retain zero answers and lose its browser +answer controls; any supersession must name the separate fallback card. +Both public stale-response APIs must reject the old answer. Any production-created +`wake_assignee` fallback is separately identified and remains unanswered; it is not +a restored native callback. The sealed observer proves no continuation marker was +created and all owned processes retired before public lease deletion. The company +must retain exactly the original run through cleanup. + +Local provider-death remains excluded. The Python admission tests include a +real pidfd calibration against a synthetic title-changing child on native Linux; +the normal E2E unit wrapper invokes it with no provider credentials or network. +macOS runs metadata negatives and explicitly skips this Linux-only calibration. +That calibration and the earlier fake-Pi wrapper/bridge tests do not count as the +real paid Product lifecycle proof. This new candidate cell remains unqualified. ## Pi active controls The explicit-only `pi-controls` suite adds `pending-permission-stop` and `same-turn-steering` on local and Daytona, each with one provider run, a 120-second active-turn timeout and a 300-second attempt budget. These four -control cases retain their behavior; the current matrix totals 25 Pi cells. +control cases retain their behavior; the current matrix totals 26 Pi cells. Pi 1/profile 11 and coverage revisions intentionally change the affected suite fingerprints, so older qualification receipts cannot be reused. Catalog presence and deterministic calibration do not constitute paid qualification. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 6f41d46ea95..14fa6fadb72 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1437,15 +1437,32 @@ mock control plane. Neither suite substitutes for the other. ### Pi native Product fixtures -The separate `pi-native` suite defines nine explicit Pi cells: native questions, +The separate `pi-native` suite defines ten explicit Pi cells: native questions, agent-file persistence, browser permission denial and controller restart with an unanswered native question on local and Daytona, plus -a local-only automatic `restrictive-denial` case. The latter cannot use the remote +a local-only automatic `restrictive-denial` case and one Daytona-only +`native-pending-provider-death` case. The latter cannot use the remote setup read under `deny-all`, so it is explicitly excluded there. It uses the pinned candidate profile and exact OpenRouter model. It is excluded from `--all`, never automatically retries, and retains qualification as pending until live proof. Reserve and reconcile each paid cell just as for `extended-harnesses`. +Provider death is a real, one-run paid Product journey, distinct from controller +restart and Stop. The owned Linux observer pins the verified wrapper ancestry, +closure and executable inode before signalling exactly its unique Pi child through +a pidfd. Pi overwrites its argv with `process.title`; entrypoint attribution is +explicitly parent-attested, not an original-child-argv claim. The fixture requires +production-generated non-replayable expiry, an expired original browser card, +rejection of stale answers by both public APIs, no continuation file effect, and +no second run. Any durable `wake_assignee` fallback stays separately identified +and unanswered. Full process/run/lease evidence remains private. Local provider +death is excluded because this exact ownership mechanism requires Linux pidfd. +The Python helper calibration uses a synthetic transport and a real title-changing +Node child; it proves fault ownership only, never paid Pi lifecycle behavior. Run +it on native Linux with pinned Node on PATH: `python3 tests/runner-e2e/pi-provider-fault.test.py`. +This adds one cell to the pending Pi Product matrix (26: 13 local, 13 Daytona); +no qualification or live success is implied by discovery or oracle tests. + ```sh pnpm test:e2e:runner -- --list --suite pi-native pnpm test:e2e:runner -- --id pi-native.runner-acpx-pi.local.native-questions diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 02d8f0c4b5a..93e6a4cfbbe 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -136,10 +136,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 4, 8, 9, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, + 4, 8, 10, 4, 4, 6, 6, 30, 3, 16, 16, 2, 6, 8, 46, 23, 47, 20, 52, 28, 18, 6, 6, 12, 10, 48, 16, 10, 2, 1, 1, ]); - expect(validateRunnerCatalog()).toHaveLength(468); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(468); + expect(validateRunnerCatalog()).toHaveLength(469); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(469); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0cf9d6ec813..4ddf6d50b07 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1103,9 +1103,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: "pi-native", label: "Pi native boundaries", manualOnly: true, description: "Pi native forms, registered agent files and human permission denial on local and Daytona execution; automatic deny-all remains local-only.", groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), - environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 9, - excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial"], - definitionMetadata: { version: 3, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" }, + environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 10, + excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial", "pi-native.runner-acpx-pi.local.native-pending-provider-death"], + definitionMetadata: { version: 4, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" }, }, { id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, diff --git a/tests/runner-e2e/failure-classifier.ts b/tests/runner-e2e/failure-classifier.ts index 1f8a82cc7e8..6d2a152fd37 100644 --- a/tests/runner-e2e/failure-classifier.ts +++ b/tests/runner-e2e/failure-classifier.ts @@ -34,6 +34,9 @@ export function classifyFailure(error: unknown): FailureClass { : "cleanup_failure"; } if (PERMANENT.test(message)) return "permanent_infrastructure"; + // A deliberately killed Pi child is the fixture stimulus. Missing lifecycle + // proof must not become a retryable transport failure because of its wording. + if (/pi_provider_death_proof:/.test(message)) return "candidate_failure"; if ( /chat_idle_state_invariant/.test(message) || NON_RETRYABLE_SESSION_CLOSE.test(message) || diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index dcf2238ec1c..557f076002e 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -124,24 +124,24 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(468); + expect(validateRunnerCatalog()).toHaveLength(469); for (const cell of cells) expect(JSON.parse(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION!)).toEqual([{ agent: "pi", model: cell.profile.model }]); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); it("pins the Pi 1 profile and versioned coverage while retaining active Stop identity", () => { - // Pi 1/profile 11 changes profile-bearing definitions. Coverage v3/v2 adds - // pending restart and the strict file oracle; no runtime admission is promoted. + // Pi 1/profile 11 changes profile-bearing definitions. Coverage v4/v2 adds + // provider death, pending restart and the strict file oracle; no runtime admission is promoted. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 3, profileVersion: 11 }); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 4, profileVersion: 11 }); expect(runnerSuites.find(s => s.id === "extended-harnesses")!.definitionMetadata).toMatchObject({ version: 2 }); const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "00d4edfd0d61772977434389cb601bbdc08173db4a33fd1e153d2e18f488f597", + "pi-native": "8c4d4f1cba234223ee54b5a19d5fe3f815424e7335cf15f43bd3f77d342aec74", "native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d", "rich-acp-warm-continuity": "25e69f031696aeb459e6722a056ae0802379f9137799648fdd1945881f32f41a", "extended-harnesses": "e50f79cf604bffb2cdee3ad4e8bc1d106045a4f44ef07e877032ff5b8960c8ab", }); - expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(21); + expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(22); }); }); diff --git a/tests/runner-e2e/pi-native-cases.test.ts b/tests/runner-e2e/pi-native-cases.test.ts index 1df4265fa6f..f99a6b78870 100644 --- a/tests/runner-e2e/pi-native-cases.test.ts +++ b/tests/runner-e2e/pi-native-cases.test.ts @@ -5,11 +5,11 @@ import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; describe("Pi native Product qualification", () => { - it("selects five local and four remote Pi cases without changing the basic extended matrix", () => { + it("selects five local and five remote Pi cases without changing the basic extended matrix", () => { const suite = runnerSuites.find(row => row.id === "pi-native")!; - expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(9); + expect(suite.manualOnly).toBe(true); expect(suite.expectedMatrixSize).toBe(10); const cells = runnerMatrix.filter(row => row.suite.id === suite.id); - expect(cells).toHaveLength(9); + expect(cells).toHaveLength(10); expect(cells.every(row => row.profile.qualificationCandidate === "pi")).toBe(true); expect(cells.filter(row => row.environment.id === "local").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([ ["native-questions", 1], ["agent-files-fresh-run", 2], ["restrictive-denial", 1], ["human-permission-denial", 1], @@ -17,9 +17,9 @@ describe("Pi native Product qualification", () => { ]); expect(cells.filter(row => row.environment.id === "daytona").map(row => [row.task.id, row.task.expectedRunCount])).toEqual([ ["native-questions", 1], ["agent-files-fresh-run", 2], ["human-permission-denial", 1], - ["native-pending-controller-restart", 1], + ["native-pending-controller-restart", 1], ["native-pending-provider-death", 1], ]); - expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(11); + expect(cells.reduce((sum, row) => sum + row.task.expectedRunCount, 0)).toBe(12); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.suite.id === suite.id)).toBe(false); expect(runnerMatrix.filter(row => row.suite.id === "extended-harnesses")).toHaveLength(30); expect(piNativeTasks[0]!.buildPrompt("fixture")).toContain("paperclip_native_question"); diff --git a/tests/runner-e2e/pi-native-cases.ts b/tests/runner-e2e/pi-native-cases.ts index 8daadeae7c7..a7011f0aaf5 100644 --- a/tests/runner-e2e/pi-native-cases.ts +++ b/tests/runner-e2e/pi-native-cases.ts @@ -7,10 +7,11 @@ export const piNativeTasks: readonly RunnerTaskFixture[] = [ ["restrictive-denial", "Restrictive native write denial prevents a file effect", 1], ["human-permission-denial", "Browser denial prevents a native write through process retirement", 1], ["native-pending-controller-restart", "Pending native question survives controller restart in the same live run", 1], + ["native-pending-provider-death", "Lost Pi provider expires its original unanswered native question", 1], ].map(([id, label, turns]) => ({ id: String(id), label: String(label), groups: [], workMode: "standard", flow: "pi_native", expectedRunCount: Number(turns), attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 }, - expectedTerminalState: { issue: "done", run: "succeeded" }, + expectedTerminalState: id === "native-pending-provider-death" ? { issue: "blocked", run: "failed" } : { issue: "done", run: "succeeded" }, buildTitle: nonce => `Pi ${id} ${nonce}`, buildVisibleMarker: nonce => `PI-NATIVE-${id}-${nonce}`, buildPrompt: nonce => piNativePrompt(String(id), nonce), @@ -22,6 +23,11 @@ export function piNativeFinish(marker: string): string { } export function piNativePrompt(id: string, nonce: string): string { + if (id === "native-pending-provider-death") return [ + 'Call paperclip_native_question exactly once with {method:"input",title:"Pi native provider death",placeholder:"Answer"}. Wait for the real answer; never guess it or substitute semantic request_human_input.', + "Only after receiving the actual answer, use native write to save its exact tool-result object as JSON to pi-provider-death-answer.json in the task workspace. Never write that file while the question is unanswered. This is an internal verification file, not a deliverable.", + piNativeFinish(`PI-NATIVE-${id}-${nonce}`), + ].join("\n"); if (id === "native-pending-controller-restart") return [ 'Call paperclip_native_question exactly once with {method:"input",title:"Pi native restart",placeholder:"Answer"}. Wait for the real answer; never guess it or substitute semantic request_human_input.', "Use native write to save the exact returned tool-result object as JSON to pi-native-restart-answer.json in the task workspace. Do not change or infer its fields. This is an internal verification file, not a deliverable.", diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index 7bd86e2ffa2..2b771161cdb 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -12,6 +12,7 @@ import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; import type { RemoteNativeFixture, RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { runPiPendingProviderDeath, PI_DEATH_MARKER } from "./pi-native-provider-death-flow.js"; import { runPiPendingControllerRestart } from "./pi-native-restart-flow.js"; export interface PiRemoteBootstrap { @@ -93,7 +94,21 @@ export async function runPiNativeFlow(input: { return result; } try { - if (execution.task.id === "native-pending-controller-restart") { + if (execution.task.id === "native-pending-provider-death") { + if (!remote) throw new Error("Exact Pi-child fault is available only on owned Daytona Linux"); + await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: [PI_DEATH_MARKER] }); + checks.push(...await runPiPendingProviderDeath({ page, api, companyId: fixtures.company.id, deadlineAt: input.deadlineAt, + fixture: currentRemote!, load, events, capture: input.capture, evidence: input.evidence })); + const originalRunId = runs[0]!.id; + input.registerCleanupAssertion!(async () => { + const state = await load(); + const passed = state.issue.status === "blocked" && state.runs.length === 1 && state.runs[0]?.id === originalRunId && state.runs[0]?.status === "failed" + && state.interactions.every(card => card.status === "expired" || (card.status === "pending" && card.continuationPolicy === "wake_assignee")); + await input.evidence("pi-provider-death-final-no-replay.json", { ...state, passed }); + if (!passed) throw new Error("Provider loss replayed a run or consumed the unanswered fallback during cleanup"); + return [{ id: "death-no-replay-through-cleanup", passed, detail: "Original failed run remains the only run through cleanup" }]; + }); + } else if (execution.task.id === "native-pending-controller-restart") { await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce), { targets: ["pi-native-restart-answer.json"] }); checks.push(...await runPiPendingControllerRestart({ page, companyId: fixtures.company.id, deadlineAt: input.deadlineAt, load, events, diff --git a/tests/runner-e2e/pi-native-provider-death-flow.test.ts b/tests/runner-e2e/pi-native-provider-death-flow.test.ts new file mode 100644 index 00000000000..487e60f15af --- /dev/null +++ b/tests/runner-e2e/pi-native-provider-death-flow.test.ts @@ -0,0 +1,116 @@ +import { expect, it, vi } from "vitest"; +import { gradePiProviderDeath, runPiPendingProviderDeath, PI_DEATH_MARKER } from "./pi-native-provider-death-flow.js"; +import { classifyFailure } from "./failure-classifier.js"; +import { observePiPendingNativeInput } from "./pi-native-restart-flow.js"; + +type Row = Record; +vi.mock("@playwright/test", () => ({ expect: (locator: { count(): number }) => ({ + toHaveCount: async (expected: number) => expect(locator.count()).toBe(expected), + toBeVisible: async () => expect(locator.count()).toBe(1), +}) })); +function fixture(fallback = true) { + const question = { schema: "paperclip.question_set.v1", questions: [{ id: "answer", header: "Pi native provider death", prompt: "Answer", answerMode: "text", required: true }] }; + const request = { schema: "paperclip.runtime_request.v2", requestKind: "runtime", type: "input", status: "pending", requestId: "request", turnId: "turn", itemId: "request", input: question, + origin: { adapter: "acpx-runtime-sidecar", provider: "pi", method: "elicitation/create" } }; + const wrap = (eventType: string, sourceSeq: number, payload: Row): Row => ({ companyId: "company", runId: "run", seq: sourceSeq, protocolSchemaVersion: 1, eventType, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", eventType, runId: "run", turnId: "turn", itemId: "request", + normalizedSessionId: "session", sourceInstanceId: "runner", sourceSeq, sourceEventId: `runner:run:${sourceSeq}`, payload } } }); + const state = { issue: { id: "issue", companyId: "company", status: "in_progress" }, runs: [{ id: "run", companyId: "company", nativeIssueId: "issue", runtimeMode: "native", status: "running", nativeSessionId: "session", runnerInstanceId: "runner" } as Row], + interactions: [{ id: "original", companyId: "company", issueId: "issue", kind: "ask_user_questions", sourceRunId: "run", status: "pending", result: null, + continuationPolicy: "none", resolverPolicy: "human_only", idempotencyKey: "paperclip-runner-question:run:request", payload: { runtimeRequestId: "request", questionSet: question } } as Row] }; + const events = [wrap("turn.started", 1, {}), wrap("runtime_request.created", 2, { request })]; + const pending = observePiPendingNativeInput(state, events, "company", "Pi native provider death"); + const before = structuredClone({ state, events }); + state.runs[0]!.status = "failed"; state.issue.status = "blocked"; state.interactions[0]!.status = "expired"; + events.push(wrap("runtime_request.expired", 3, { requestId: "request", requestKind: "runtime", requestType: "input", turnId: "turn", itemId: "request", reason: "provider_process_lost", replayAllowed: false, request }), + wrap("turn.failed", 4, { code: "provider_transport_lost" })); + if (fallback) state.interactions.push({ ...structuredClone(state.interactions[0]!), id: "fallback", status: "pending", continuationPolicy: "wake_assignee", idempotencyKey: "runtime-input-durable:v1:run:request" }); + state.interactions[0]!.result = { version: 1, answers: [], ...(fallback ? { expirationReason: "superseded_by_newer_interaction", supersededByInteractionId: "fallback" } : {}) }; + const grade = () => gradePiProviderDeath(state, events, pending, question); + return { state, events, pending, question, wrap, grade, before }; +} +it.each([false, true])("accepts production loss with distinct fallback=%s without claiming restoration", fallback => { + expect(fixture(fallback).grade()).toEqual({ originalCardExpired: true, replayAllowed: false, originalProviderRestored: false, fallbackInteractionId: fallback ? "fallback" : null }); +}); +it.each([ + ["wrong issue disposition", (f: ReturnType) => { f.state.issue.status = "in_progress"; }], + ["completed issue", (f: ReturnType) => { f.state.issue.status = "done"; }], + ["answered expired card", (f: ReturnType) => { f.state.interactions[0]!.result.answers = [{ questionId: "answer", otherText: "stale" }]; }], + ["missing expired result", (f: ReturnType) => { f.state.interactions[0]!.result = null; }], + ["foreign supersession", (f: ReturnType) => { f.state.interactions[0]!.result.supersededByInteractionId = "other"; }], + ["missing expiry", (f: ReturnType) => { f.events.splice(2, 1); }], + ["model-only claim", f => { f.events[2] = { eventType: "assistant.message", payload: { text: "The callback expired" } }; }], + ["controller injected expiry", f => { f.events[2]!.payload.prpEvent.sourceKind = "control_plane"; }], + ["cancelled instead of lost", f => { f.events[2]!.eventType = f.events[2]!.payload.prpEvent.eventType = "runtime_request.cancelled"; }], + ["resolved instead of lost", f => { f.events[2]!.eventType = f.events[2]!.payload.prpEvent.eventType = "runtime_request.resolved"; }], + ["normal durable handoff", f => { f.events[2]!.payload.prpEvent.payload.reason = "durable_handoff"; }], + ["replay allowed", f => { f.events[2]!.payload.prpEvent.payload.replayAllowed = true; }], + ["foreign request", f => { f.events[2]!.payload.prpEvent.payload.requestId = "other"; }], + ["foreign item", f => { f.events[2]!.payload.prpEvent.payload.itemId = "other"; }], + ["missing canonical request", f => { delete f.events[2]!.payload.prpEvent.payload.request; }], + ["replaced canonical question", f => { f.events[2]!.payload.prpEvent.payload.request = { ...f.events[2]!.payload.prpEvent.payload.request, input: {} }; }], + ["duplicate expiry", f => { f.events.push(f.wrap("runtime_request.expired", 5, f.events[2]!.payload.prpEvent.payload)); }], + ["new native turn", f => { f.events.push(f.wrap("turn.started", 5, {})); }], + ["successful provider", f => { f.state.runs[0]!.status = "succeeded"; }], + ["Stop cancellation", f => { f.state.runs[0]!.resultJson = { nativeCancellation: {} }; }], + ["worker failure without turn failure", f => { f.events.pop(); }], + ["completed turn", f => { f.events[3]!.eventType = f.events[3]!.payload.prpEvent.eventType = "turn.completed"; }], + ["terminal before expiry", f => { f.events[3]!.seq = 0; }], + ["second run", f => { f.state.runs.push({ ...f.state.runs[0], id: "other" }); }], + ["restored session", f => { f.state.runs[0]!.nativeSessionId = "restored"; }], + ["foreign company", f => { f.events[2]!.companyId = "foreign"; }], + ["foreign source", f => { f.events[2]!.payload.prpEvent.sourceInstanceId = "foreign"; }], + ["original still pending", f => { f.state.interactions[0]!.status = "pending"; }], + ["answered original", f => { f.state.interactions[0]!.status = "answered"; }], + ["rebound original card", f => { f.state.interactions[0]!.sourceRunId = "other"; }], + ["duplicate fallback", f => { f.state.interactions.push(structuredClone(f.state.interactions[1]!)); }], + ["fallback masquerading as callback", f => { f.state.interactions[1]!.continuationPolicy = "none"; }], + ["answered fallback", f => { f.state.interactions[1]!.status = "answered"; }], + ["foreign fallback", f => { f.state.interactions[1]!.idempotencyKey = "foreign"; }], +] satisfies Array<[string, (f: ReturnType) => void]>)("rejects %s", (_name, mutate) => { + const f = fixture(); mutate(f); expect(f.grade).toThrow(); +}); + +async function journey(staleStatus = 409, markerEffect = false) { + const f = fixture(), evidence = new Map(), posted: string[] = []; + let lost = false, signals = 0, retired = false; + Object.assign(f.before.state.runs[0]!, { contextSnapshot: { executionWorkspaceId: "workspace", paperclipEnvironment: { leaseId: "lease" } }, + runnerProfileJson: { nativeExecutionInput: { binding: { executionWorkspaceId: "workspace" } } } }); + const root = { pid: 21, startTicks: "100", bootId: "boot" }; + const composer: any = { count: () => 1, filter: () => composer }; + const page: any = { getByTestId: (id: string) => id === "issue-detail-header" ? { getByRole: (_role: string, options: Row) => { + expect(options).toEqual({ name: "Change status (current: Blocked", exact: false }); return { count: () => 1 }; + } } : composer, reload: async () => { expect(retired).toBe(true); }, + locator: (selector: string) => { expect(selector).toBe("#interaction-original"); return { count: () => 1, locator: () => ({ count: () => 0 }) }; } }; + const remoteFixture: any = { binding: { companyId: "company", runId: "run", leaseId: "lease" }, baseline: { scope: { observedPrpEnvironmentLeaseId: "workspace" } }, + terminatePiProvider: async (lease: string) => { expect(lease).toBe("workspace"); expect(lost).toBe(false); signals++; lost = true; return { targetKind: "pi_native_child", signalled: true }; }, + finish: async () => { expect(lost).toBe(true); retired = true; return { complete: true, observedAtMs: 1, watcher: { complete: true }, + processes: { captured: true, root, journal: [root], live: [] }, targets: { [PI_DEATH_MARKER]: { absent: !markerEffect, complete: true, mutationCount: markerEffect ? 1 : 0 } } }; } }; + const result = runPiPendingProviderDeath({ page, api: { request: { post: async (path: string) => { expect(retired).toBe(true); posted.push(path); return { status: () => staleStatus }; } } } as any, + companyId: "company", deadlineAt: Date.now() + 1000, fixture: remoteFixture, + load: async () => structuredClone(lost ? f.state : f.before.state), events: async () => structuredClone(lost ? f.events : f.before.events), + capture: async () => {}, evidence: async (name, value) => { evidence.set(name, value); } }); + return { result, evidence, posted, signalCount: () => signals }; +} +it("drives one admitted fault, retirement, original-card UI and both stale public APIs in order", async () => { + const j = await journey(); expect((await j.result).every(check => check.passed)).toBe(true); + expect(j.signalCount()).toBe(1); expect(j.posted).toEqual(["/api/issues/issue/interactions/original/respond", "/api/heartbeat-runs/run/runtime-requests/request/resolve"]); + expect(j.evidence.get("pi-provider-death-result.json")).toMatchObject({ originalProviderRestored: false, fallbackInteractionId: "fallback", staleCardStatus: 409, staleRuntimeStatus: 409 }); +}); +it("does not accept a successful stale answer despite otherwise valid expiry receipts", async () => { + const j = await journey(200); await expect(j.result).rejects.toThrow("Both original durable-card"); expect(j.signalCount()).toBe(1); +}); +it("rejects a continuation file effect before sending either stale answer", async () => { + const j = await journey(409, true); await expect(j.result).rejects.toThrow("Exact owned process tree"); expect(j.posted).toEqual([]); +}); + +it("classifies deliberate-loss proof failures as candidate failures, preserving real API transport errors", () => { + const f = fixture(); f.state.runs[0]!.status = "succeeded"; + let observed: unknown; + try { f.grade(); } catch (error) { observed = error; } + expect(String(observed)).toContain("pi_provider_death_proof:"); + expect(classifyFailure(observed)).toBe("candidate_failure"); + expect(classifyFailure(new Error("GET controller returned 503 service unavailable"))).toBe("transient_infrastructure"); + expect(classifyFailure(new Error("pi_provider_death_proof: secret leak"))).toBe("secret_leak"); + expect(classifyFailure(new Error("pi_provider_death_proof: cleanup incomplete"))).toBe("cleanup_failure"); +}); diff --git a/tests/runner-e2e/pi-native-provider-death-flow.ts b/tests/runner-e2e/pi-native-provider-death-flow.ts new file mode 100644 index 00000000000..1189e0863ae --- /dev/null +++ b/tests/runner-e2e/pi-native-provider-death-flow.ts @@ -0,0 +1,135 @@ +import { createHash, randomBytes } from "node:crypto"; +import { expect, type Page } from "@playwright/test"; +import { canonicalJson } from "../../packages/shared/src/portability-hash.js"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { isValidNativePrpEnvelope } from "./native-event-envelope.js"; +import { observePiPendingNativeInput, type PiRestartPending } from "./pi-native-restart-flow.js"; +import { hasPiRemoteRetirement } from "./pi-native-evidence.js"; +import type { RemoteNativeFixture, PiProviderDeathReceipt } from "./remote-native-fixtures.js"; + +type Row = Record; +type State = { issue: Row; runs: Row[]; interactions: Row[] }; +type Check = { id: string; passed: boolean; detail: string }; +const hash = (value: unknown) => createHash("sha256").update(canonicalJson(value)).digest("hex"); +const demand = (value: unknown, reason: string): void => { if (!value) throw new Error(`pi_provider_death_proof: ${reason}`); }; +export const PI_DEATH_MARKER = "pi-provider-death-answer.json"; + +/** All events are public durable production receipts, never a fault fixture's events. */ +export function gradePiProviderDeath(state: State, events: readonly Row[], pending: PiRestartPending, originalQuestion: unknown) { + demand(state.issue.id === pending.issueId && state.issue.companyId === pending.companyId && state.issue.status === "blocked" && state.runs.length === 1, "foreign issue, wrong blocked disposition or second run"); + const run = state.runs[0]!; + demand(run.id === pending.runId && run.companyId === pending.companyId && run.nativeIssueId === pending.issueId && run.runtimeMode === "native" + && run.nativeSessionId === pending.nativeSessionId && run.runnerInstanceId === pending.sourceInstanceId + && run.status === "failed" && run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null, + "original provider did not fail independently of Stop"); + demand(Array.isArray(events) && events.length > 0 && events.length <= 20_000, "missing event history"); + const rows = events.filter(row => row.payload?.prpEvent).sort((a, b) => a.seq - b.seq); + const ids = new Set(), seqs = new Set(), cursors = new Map(); + for (const row of rows) { + const e = row.payload.prpEvent; + demand(row.companyId === pending.companyId && row.runId === pending.runId && row.eventType === e.eventType + && Number.isSafeInteger(row.seq) && row.seq > 0 && !seqs.has(row.seq) && isValidNativePrpEnvelope(e, row.protocolSchemaVersion) + && e.runId === pending.runId && e.normalizedSessionId === pending.nativeSessionId + && ((e.sourceKind === "runner" && e.sourceInstanceId === pending.sourceInstanceId) + || (e.sourceKind === "control_plane" && e.sourceInstanceId === `${pending.sourceInstanceId}:control`)) + && e.sourceEventId === `${e.sourceInstanceId}:${pending.runId}:${e.sourceSeq}` && !ids.has(e.sourceEventId) + && Number.isSafeInteger(e.sourceSeq) && e.sourceSeq > (cursors.get(e.sourceInstanceId) ?? 0), "foreign, duplicate or reordered event"); + ids.add(e.sourceEventId); seqs.add(row.seq); cursors.set(e.sourceInstanceId, e.sourceSeq); + if (e.turnId != null) demand(e.turnId === pending.turnId, "replacement native turn"); + } + const of = (kind: string) => rows.filter(row => row.eventType === kind); + const created = of("runtime_request.created"); + demand(created.length === 1 && hash(created[0]) === pending.createdRowSha256, "original request changed or replayed"); + const closures = rows.filter(row => ["runtime_request.expired", "runtime_request.resolved", "runtime_request.cancelled"].includes(row.eventType)); + demand(closures.length === 1 && closures[0]!.eventType === "runtime_request.expired", "missing unique production expiry"); + const expired = closures[0]!.payload.prpEvent, p = expired.payload; + demand(expired.sourceKind === "runner" && expired.sourceSeq > pending.createdSourceSeq && expired.turnId === pending.turnId + && p.requestId === pending.requestId && p.requestKind === "runtime" && p.requestType === "input" && p.turnId === pending.turnId && p.itemId === pending.itemId + && p.reason === "provider_process_lost" && p.replayAllowed === false && hash(p.request) === hash(created[0]!.payload.prpEvent.payload.request), "expiry is not the original lost Pi callback"); + const failed = of("turn.failed"); + demand(failed.length === 1 && failed[0]!.seq > closures[0]!.seq && failed[0]!.payload.prpEvent.sourceKind === "runner" + && failed[0]!.payload.prpEvent.turnId === pending.turnId && !of("turn.completed").length && !of("turn.cancelled").length + && of("turn.started").length === 1 && !of("run.result.proposed").length, "wrong terminal or replacement turn"); + const card = state.interactions.find(card => card.id === pending.interactionId); + demand(card?.companyId === pending.companyId && card.issueId === pending.issueId && card.sourceRunId === pending.runId + && card.idempotencyKey === `paperclip-runner-question:${pending.runId}:${pending.requestId}` && card.continuationPolicy === "none" + && card.status === "expired" && card.result?.version === 1 && Array.isArray(card.result.answers) && card.result.answers.length === 0 + && card.kind === "ask_user_questions" && card.resolverPolicy === "human_only" && card.payload?.runtimeRequestId === pending.requestId && hash(card.payload.questionSet) === hash(originalQuestion), "original durable card did not expire"); + const fallback = state.interactions.filter(card => card.id !== pending.interactionId); + demand(fallback.length <= 1, "duplicate fallback interaction"); + if (fallback.length) { + const next = fallback[0]!; + demand(next.id !== pending.interactionId && next.companyId === pending.companyId && next.issueId === pending.issueId && next.sourceRunId === pending.runId + && next.kind === "ask_user_questions" && next.status === "pending" && next.result == null && next.continuationPolicy === "wake_assignee" + && next.idempotencyKey === `runtime-input-durable:v1:${pending.runId}:${pending.requestId}` + && card!.result.expirationReason === "superseded_by_newer_interaction" && card!.result.supersededByInteractionId === next.id + && next.payload?.runtimeRequestId === pending.requestId && hash(next.payload.questionSet) === hash(originalQuestion), "fallback was mistaken for restored input"); + } + return { originalCardExpired: true, replayAllowed: false, originalProviderRestored: false, fallbackInteractionId: fallback[0]?.id ?? null }; +} + +export async function runPiPendingProviderDeath(input: { + page: Page; api: RunnerApi; companyId: string; deadlineAt: number; fixture: RemoteNativeFixture; + load(): Promise; events(runId: string): Promise; + capture(id: string, label: string, file: string): Promise; evidence(name: string, value: unknown): Promise; +}): Promise { + const checks: Check[] = []; + const check = (id: string, passed: boolean, detail: string) => { checks.push({ id, passed, detail }); demand(passed, detail); }; + let fault: PiProviderDeathReceipt | undefined; + const snapshot = async (name: string) => { + const state = await input.load(), events = state.runs.length === 1 ? await input.events(state.runs[0]!.id) : []; + await input.evidence(name, { ...state, events, fault }); return { state, events }; + }; + try { + await pollUntil({ label: "Pi native input before provider death", deadlineAt: input.deadlineAt, load: input.load, + accept: s => s.interactions.some(c => c.status === "pending"), reject: s => s.runs.length > 1 || s.runs.some(r => ["succeeded", "failed", "cancelled", "timed_out"].includes(r.status)) ? "Provider ended before fault admission" : undefined }); + const before = await snapshot("pi-provider-death-before.json"); + const pending = observePiPendingNativeInput(before.state, before.events, input.companyId, "Pi native provider death"); + const run = before.state.runs[0]!, question = before.state.interactions[0]!.payload.questionSet; + demand(input.fixture.binding.runId === pending.runId && input.fixture.binding.companyId === pending.companyId && input.fixture.terminatePiProvider, + "missing exact owned Daytona fault capability"); + // createRunnerdBackend's default PRP lease is the execution workspace ID; + // reject alternate/recovered authority instead of guessing that identity. + const workspaceId = run.contextSnapshot?.executionWorkspaceId; + demand(typeof workspaceId === "string" && workspaceId.length > 0 + && run.runnerProfileJson?.nativeExecutionInput?.binding?.executionWorkspaceId === workspaceId + && run.contextSnapshot?.paperclipEnvironment?.leaseId === input.fixture.binding.leaseId + && input.fixture.baseline.scope.observedPrpEnvironmentLeaseId === workspaceId, "native run/lease/workspace binding unavailable"); + await expect(input.page.getByTestId("question-text-answer-composer").filter({ visible: true })).toHaveCount(1); + await input.capture("pi-provider-death-pending", "Unanswered native Pi input before provider loss", "pi-provider-death-pending.png"); + // Refresh immediately before the one-shot irreversible fault. + const fresh = await snapshot("pi-provider-death-admission.json"); + check("death-original-pending-input", hash(observePiPendingNativeInput(fresh.state, fresh.events, input.companyId, "Pi native provider death")) === hash(pending), "Fault admits only the unchanged original native input"); + fault = await input.fixture.terminatePiProvider!(workspaceId); + await input.evidence("pi-provider-death-fault.json", fault); + await pollUntil({ label: "production expiry after Pi provider loss", deadlineAt: input.deadlineAt, load: input.load, + accept: s => s.issue.status === "blocked" && s.runs[0]?.status === "failed" && s.interactions.some(c => c.id === pending.interactionId && c.status === "expired"), + reject: s => s.runs.length !== 1 || s.runs.some(r => ["succeeded", "cancelled", "timed_out"].includes(r.status)) ? "Unexpected run after provider fault" : undefined }); + const retired = await input.fixture.finish(); + check("death-owned-retirement-no-effect", hasPiRemoteRetirement(retired) && retired.targets[PI_DEATH_MARKER]?.absent === true + && retired.targets[PI_DEATH_MARKER]?.complete === true && retired.targets[PI_DEATH_MARKER]?.mutationCount === 0, + "Exact owned process tree retired while the unanswered continuation marker remained absent"); + await input.evidence("pi-provider-death-retirement.json", retired); + await input.page.reload(); + const final = await snapshot("pi-provider-death-expired.json"), result = gradePiProviderDeath(final.state, final.events, pending, question); + // Failed native terminals are permanent on the first attempt; the + // controller's nativeSessionRecoveryProjection projects Blocked while this + // run still owns the task (native-session-executor.ts). + await expect(input.page.getByTestId("issue-detail-header").getByRole("button", { name: "Change status (current: Blocked", exact: false })).toBeVisible(); + const oldCard = input.page.locator(`#interaction-${pending.interactionId}`); + await expect(oldCard).toHaveCount(1); + await expect(oldCard.locator('textarea,[contenteditable="true"],button[type="submit"]')).toHaveCount(0); + await input.capture("pi-provider-death-expired", "Expired original input and any separate durable fallback", "pi-provider-death-expired.png"); + const answer = `STALE-${randomBytes(16).toString("hex")}`; + const original = await input.api.request.post(`/api/issues/${pending.issueId}/interactions/${pending.interactionId}/respond`, + { data: { answers: [{ questionId: pending.questionId, optionIds: [], otherText: answer }] } }); + const runtime = await input.api.request.post(`/api/heartbeat-runs/${pending.runId}/runtime-requests/${encodeURIComponent(pending.requestId)}/resolve`, + { data: { resolution: { action: "submit", response: { schema: "paperclip.question_response.v1", answers: { [pending.questionId]: { text: answer } } } } } }); + check("death-stale-original-responses-rejected", original.status() === 409 && runtime.status() === 409, "Both original durable-card and native-runtime response APIs reject the stale answer"); + const after = await snapshot("pi-provider-death-after-stale.json"); + check("death-expiry-not-restoration", hash(gradePiProviderDeath(after.state, after.events, pending, question)) === hash(result) + && hash(after.state.interactions) === hash(final.state.interactions), "Stale answers neither resolve/replay the original request nor start a new run; any fallback remains a separate unanswered interaction"); + await input.evidence("pi-provider-death-result.json", { ...result, staleCardStatus: original.status(), staleRuntimeStatus: runtime.status(), paidProviderDeath: true }); + return checks; + } finally { await input.evidence("pi-provider-death-checks.json", checks); } +} diff --git a/tests/runner-e2e/pi-native-restart-flow.ts b/tests/runner-e2e/pi-native-restart-flow.ts index a590549e3c4..ff8b2eb3c60 100644 --- a/tests/runner-e2e/pi-native-restart-flow.ts +++ b/tests/runner-e2e/pi-native-restart-flow.ts @@ -28,7 +28,7 @@ function interactionBinding(card: Row) { continuationPolicy: card.continuationPolicy, resolverPolicy: card.resolverPolicy, payload: card.payload }; } -function inspect(state: State, events: readonly Row[], companyId: string) { +function inspect(state: State, events: readonly Row[], companyId: string, header = "Pi native restart") { requireProof(id(companyId) && id(state.issue.id) && state.issue.companyId === companyId && state.runs.length === 1 && state.interactions.length === 1, "one company-scoped task, run and question required"); const run = state.runs[0]!, card = state.interactions[0]!; @@ -63,7 +63,7 @@ function inspect(state: State, events: readonly Row[], companyId: string) { requireProof(request.schema === "paperclip.runtime_request.v2" && request.requestKind === "runtime" && request.type === "input" && request.status === "pending" && id(request.requestId) && id(request.itemId) && id(request.turnId) && request.turnId === opening.event.turnId && request.itemId === opening.event.itemId && hasAcpxNativeOrigin(request.origin, "pi", "elicitation/create") && questionSet.schema === "paperclip.question_set.v1" - && id(question.id) && question.answerMode === "text" && question.header === "Pi native restart" + && id(question.id) && question.answerMode === "text" && question.header === header && card.payload?.runtimeRequestId === request.requestId && digest(card.payload.questionSet) === digest(questionSet) && card.idempotencyKey === `paperclip-runner-question:${run.id}:${request.requestId}`, "canonical Pi input request identity missing"); requireProof(rows.filter(x => x.event.turnId != null || x.event.eventType.startsWith("turn.") || x.event.eventType.startsWith("runtime_request.")).every(({ event }) => event.turnId === request.turnId @@ -77,14 +77,18 @@ function inspect(state: State, events: readonly Row[], companyId: string) { } /** Grade public state and PRP receipts; assistant prose is never restart proof. */ -export function observePiRestartPending(state: State, events: readonly Row[], companyId: string): PiRestartPending { - const { run, card, rows, binding } = inspect(state, events, companyId); +export function observePiPendingNativeInput(state: State, events: readonly Row[], companyId: string, header: string): PiRestartPending { + const { run, card, rows, binding } = inspect(state, events, companyId, header); requireProof(state.issue.status === "in_progress" && run.status === "running" && card.status === "pending" && card.result == null && run.resultJson?.nativeCancellation == null && run.resultJson?.startupCancellation == null && !rows.some(x => closures.has(x.event.eventType) || terminals.has(x.event.eventType) || x.event.sourceKind === "control_plane"), "question is no longer unanswered in a live run"); return binding; } +export function observePiRestartPending(state: State, events: readonly Row[], companyId: string): PiRestartPending { + return observePiPendingNativeInput(state, events, companyId, "Pi native restart"); +} + export function gradePiRestartCompletion(state: State, events: readonly Row[], pending: PiRestartPending, answer: string, proof: unknown): boolean { const { run, card, rows, binding } = inspect(state, events, pending.companyId); requireProof(digest(binding) === digest(pending), "original request/run/source identity changed"); diff --git a/tests/runner-e2e/remote-native-fixtures.test.ts b/tests/runner-e2e/remote-native-fixtures.test.ts index b9733b4c3d4..8654f50fa5c 100644 --- a/tests/runner-e2e/remote-native-fixtures.test.ts +++ b/tests/runner-e2e/remote-native-fixtures.test.ts @@ -7,8 +7,9 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { Script } from "node:vm"; import { describe, expect, it, vi } from "vitest"; -import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, validatePiProviderDeathReceipt, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { PI_DISTRIBUTION_CLOSURE_SHA256 } from "../../packages/paperclip-runner/src/drivers/acpx/pi-closure-pins.js"; import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js"; const hash = (s: string) => `sha256:${createHash("sha256").update(s).digest("hex")}`; @@ -418,8 +419,9 @@ describe("actual generated observer state machine", () => { close: vi.fn(() => listeners.clear()), }; const net = { createServer(fn: (socket: any) => void) { handlers.push(fn); return server; } }; + const faultSpawn = vi.fn(() => ({ status: 0, stdout: JSON.stringify(piFaultReceipt()) })); const context = { - require(name: string) { if (name === "node:fs") return fs; if (name === "node:net") return net; if (name === "node:child_process") return { spawn: vi.fn(() => { const child = Object.assign(new EventEmitter(), { pid: 88, exitCode: null, signalCode: null, kill: vi.fn() }); children.push(child); return child; }) }; if (name === "node:path") return { join: (...paths: string[]) => paths.join("/"), dirname: (path: string) => path.slice(0, path.lastIndexOf("/")), basename: (path: string) => path.slice(path.lastIndexOf("/") + 1) }; if (name === "node:crypto") return { createHash }; throw new Error("unexpected module"); }, + require(name: string) { if (name === "node:fs") return fs; if (name === "node:net") return net; if (name === "node:child_process") return { spawnSync: faultSpawn, spawn: vi.fn(() => { const child = Object.assign(new EventEmitter(), { pid: 88, exitCode: null, signalCode: null, kill: vi.fn() }); children.push(child); return child; }) }; if (name === "node:path") return { join: (...paths: string[]) => paths.join("/"), dirname: (path: string) => path.slice(0, path.lastIndexOf("/")), basename: (path: string) => path.slice(path.lastIndexOf("/") + 1) }; if (name === "node:crypto") return { createHash }; throw new Error("unexpected module"); }, process: { argv: ["node", `${config.root}/observer.cjs`, Buffer.from(JSON.stringify(config)).toString("base64")], execPath: "/node", hrtime: { bigint: () => 12345n }, exit: vi.fn() }, Buffer, __filename: `${config.root}/observer.cjs`, setInterval(fn: () => void) { intervals.push(fn); return 1; }, clearInterval: vi.fn(), @@ -431,8 +433,28 @@ describe("actual generated observer state machine", () => { const replies: any[] = [], socket = Object.assign(new EventEmitter(), { end: (value: string) => replies.push(JSON.parse(value)), destroy: vi.fn() }); handlers[0]!(socket); socket.emit("data", Buffer.from(JSON.stringify({ op, nonce: config.nonce, ...args }) + "\n")); return replies; } - return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, symbolicLinks, context, server, directories, listeners, install: h.calls.find(c => c.request.op === "install")!, replaceRuntimeRoot() { runtimeInode = 999n; } }; + return { request, proc, files, watches, fs, intervals, timers, config, handlers, children, faultSpawn, symbolicLinks, context, server, directories, listeners, install: h.calls.find(c => c.request.op === "install")!, replaceRuntimeRoot() { runtimeInode = 999n; } }; } + it("executes the actual observer fault phase once with a closed environment and exact identity", async () => { + for (const published of [false, true]) { + const rejected = await observerHarness(); + if (published) rejected.request("publish", { path: "action.txt", text: "ask native input" }); + expect(rejected.request("pi-provider-death", { runtimeEnvironmentLeaseId: published ? "foreign" : "workspace-id" })[0].ok).toBe(false); + expect(rejected.faultSpawn).not.toHaveBeenCalled(); + } + const o = await observerHarness(); + expect(o.request("snapshot")[0].ok).toBe(true); + expect(o.request("publish", { path: "action.txt", text: "ask native input" })[0].ok).toBe(true); + expect(o.request("pi-provider-death", { runtimeEnvironmentLeaseId: "workspace-id" })[0]).toEqual({ ok: true, result: piFaultReceipt() }); + expect(o.faultSpawn).toHaveBeenCalledTimes(1); + const [program, args, options] = o.faultSpawn.mock.calls[0] as unknown as [string, string[], Record]; + expect(program).toBe("/usr/bin/python3"); expect(args.slice(0, 2)).toEqual(["-I", "-c"]); + expect(args[2]).toContain("signal.pidfd_send_signal"); + expect(JSON.parse(Buffer.from(args[3]!, "base64").toString())).toEqual({ root, binding, runtimeEnvironmentLeaseId: "workspace-id", runnerdSha256: hash("runnerd"), closureSha256: PI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"] }); + expect(options).toMatchObject({ env: { PATH: "/usr/bin:/bin" }, timeout: 8000, maxBuffer: 32768 }); + expect(o.request("pi-provider-death", { runtimeEnvironmentLeaseId: "workspace-id" })[0].ok).toBe(false); + expect(o.faultSpawn).toHaveBeenCalledTimes(1); + }); async function generatedRpc(o: Awaited>, request: Record, mutateSource = (source: string) => source) { const quoted = o.install.command.match(/ -e (.+) '[A-Za-z0-9+/=]+'$/su)![1]!; const source = quoted.slice(1, -1).replaceAll("'\\''", "'"); @@ -692,3 +714,41 @@ describe("actual generated observer state machine", () => { expect(other.request("snapshot")[0].ok).toBe(false); }); }); + +function piFaultReceipt() { + const parent = { pid: 22, ppid: 21, startTicks: "101", bootId }, target = { pid: 23, ppid: 22, startTicks: "102", bootId }; + return { schema: "paperclip.e2e.pi-provider-death.v1", binding, runtimeEnvironmentLeaseId: "workspace-id", target, ancestry: [target, parent, root], + nodeSha256: hash("node"), entrypointSha256: hash("entry"), closureSha256: PI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"], + entrypointAttribution: "pinned_wrapper_parent", originalChildArgvAvailable: false, observedChildTitle: "pi", signalled: true, signal: "SIGKILL", targetKind: "pi_native_child", workerSignalled: false }; +} +describe("exact Pi-child fault capability", () => { + it("validates ancestry and honest parent-attested entrypoint identity", () => { + const receipt = piFaultReceipt(); + expect(validatePiProviderDeathReceipt(receipt, binding, root, "workspace-id")).toEqual(receipt); + for (const patch of [{ workerSignalled: true }, { target: root }, { originalChildArgvAvailable: true }, { entrypointAttribution: "process_title" }, + { binding: { ...binding, runId: "foreign" } }, { runtimeEnvironmentLeaseId: "foreign" }, { closureSha256: "0".repeat(64) }, + { ancestry: [receipt.target, { ...receipt.ancestry[1], startTicks: "" }, root] }, { ancestry: [receipt.target, root] }, + { ancestry: [receipt.target, { ...receipt.ancestry[1], ppid: 999 }, root] }, { ancestry: [receipt.target, receipt.ancestry[1], { ...root, startTicks: "999" }] }]) { + expect(() => validatePiProviderDeathReceipt({ ...receipt, ...patch }, binding, root, "workspace-id")).toThrow(); + } + }); + it("requires published action and consumes the one-shot capability even on RPC failure", async () => { + const h = harness(), fixture = await bindRemoteNativeFixture(h.options); + await expect(fixture.terminatePiProvider!("workspace-id")).rejects.toThrow(); + await fixture.publishAction("action.txt", "Ask the native question"); + h.override(request => request.op === "pi-provider-death" ? { exitCode: 0, result: JSON.stringify({ ok: false, error: "pi_provider_identity_or_signal_failed" }) } : undefined); + await expect(fixture.terminatePiProvider!("workspace-id")).rejects.toThrow(); + await expect(fixture.terminatePiProvider!("workspace-id")).rejects.toThrow(); + expect(h.calls.filter(call => call.request.op === "pi-provider-death")).toHaveLength(1); + await fixture.close(); + }); + it("returns only the exact reviewed fault receipt and never signals the worker", async () => { + const h = harness(), fixture = await bindRemoteNativeFixture(h.options); + await fixture.publishAction("action.txt", "Ask the native question"); + h.override(request => request.op === "pi-provider-death" ? { exitCode: 0, result: JSON.stringify({ ok: true, result: piFaultReceipt() }) } : undefined); + expect(await fixture.terminatePiProvider!("workspace-id")).toEqual(piFaultReceipt()); + await expect(fixture.terminatePiProvider!("workspace-id")).rejects.toThrow(); + expect(h.calls.filter(call => call.request.op === "pi-provider-death")).toHaveLength(1); + await fixture.close(); + }); +}); diff --git a/tests/runner-e2e/remote-native-fixtures.ts b/tests/runner-e2e/remote-native-fixtures.ts index 924e7a9682a..b0e5f3db617 100644 --- a/tests/runner-e2e/remote-native-fixtures.ts +++ b/tests/runner-e2e/remote-native-fixtures.ts @@ -1,7 +1,10 @@ import { createHash, randomBytes } from "node:crypto"; -import { watch, lstatSync } from "node:fs"; +import { watch, lstatSync, readFileSync } from "node:fs"; import { posix } from "node:path"; +import { PI_DISTRIBUTION_CLOSURE_SHA256 } from "../../packages/paperclip-runner/src/drivers/acpx/pi-closure-pins.js"; +const PI_FAULT_SOURCE = readFileSync(new URL("./pi-provider-fault.py", import.meta.url), "utf8"); + export const REMOTE_FIXTURE_DAYTONA_SDK_VERSION = "0.203.0"; const NODE = "/opt/paperclip-runner/provider-pack/node_modules/node/bin/node"; const MAX_OUTPUT = 256 * 1024; @@ -30,8 +33,8 @@ function relative(value: string): string { } // Only closed diagnostic enums cross the remote boundary; SDK errors and output // are never retained. These diagnostics explain failed evidence, not qualification. -const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "read"] as const; -const RPC_CODES = ["node_identity", "sentinel_type", "sentinel", "cwd", "runtime_root_identity", "runtime_binary_identity", "proc_bound", "ambiguous_run_root", "runtime_not_ready", "runtime_identity_changed", "invalid_proc_identity", "invalid_proc_fields", "socket_error", "socket_timeout", "output_bound", "rpc_deadline", "remote_unknown", "transport_failure", "invalid_response", "readiness_deadline"] as const; +const RPC_PHASES = ["runtime-ready", "install", "wait", "close", "arm", "publish", "snapshot", "attached", "read", "pi-provider-death"] as const; +const RPC_CODES = ["node_identity", "sentinel_type", "sentinel", "cwd", "runtime_root_identity", "runtime_binary_identity", "proc_bound", "ambiguous_run_root", "runtime_not_ready", "runtime_identity_changed", "invalid_proc_identity", "invalid_proc_fields", "socket_error", "socket_timeout", "output_bound", "rpc_deadline", "remote_unknown", "transport_failure", "invalid_response", "readiness_deadline", "pi_provider_identity_or_signal_failed"] as const; type RpcPhase = typeof RPC_PHASES[number]; type RpcCode = typeof RPC_CODES[number]; interface RpcDiagnostic { phase: RpcPhase; code: RpcCode } @@ -121,13 +124,13 @@ export function createRemoteTargetWatch(directory: string, name: string, io = { const OBSERVER = String.raw` const fs=require('node:fs'),path=require('node:path'),crypto=require('node:crypto'),net=require('node:net'),cp=require('node:child_process'); const config=JSON.parse(Buffer.from(process.argv[2],'base64').toString()); -const parseStat=PARSE_STAT;const runRoot=RUN_ROOT;const watchTarget=WATCH_TARGET; +const parseStat=PARSE_STAT;const runRoot=RUN_ROOT;const watchTarget=WATCH_TARGET;const PI_FAULT_SOURCE=PI_FAULT_PROGRAM,PI_CLOSURE_PIN=PI_CLOSURE_DIGEST; const hash=x=>'sha256:'+crypto.createHash('sha256').update(x).digest('hex'); const cwdStat=fs.lstatSync(config.binding.remoteCwd,{bigint:true}),rootStat=fs.lstatSync(config.root,{bigint:true}),scriptStat=fs.lstatSync(__filename,{bigint:true}),scriptHash=hash(fs.readFileSync(__filename)); const runtimeRoot=path.join(config.binding.remoteCwd,config.runtimeRelative),runtimeStat=fs.lstatSync(runtimeRoot,{bigint:true});if(!runtimeStat.isDirectory()||runtimeStat.isSymbolicLink()||fs.realpathSync(runtimeRoot)!==runtimeRoot)throw Error('runtime_root_identity');let observedPrpEnvironmentLeaseId=null; const boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim(); const targets=new Map();let complete=true,sealed=false,root=null,attached=null,child=null,client=null,childTimer=null; -const journal=new Map(),sockets=new Set(),waiters=new Set(),armWaiters=new Set();let observedRootCount=0,publishedHash=null,finalReceipt=null,retiring=false; +const journal=new Map(),sockets=new Set(),waiters=new Set(),armWaiters=new Set();let observedRootCount=0,publishedHash=null,finalReceipt=null,retiring=false,piFaultAttempted=false; function identity(pid){return parseStat(pid,fs.readFileSync('/proc/'+pid+'/stat','utf8'),boot)} function table(){const ids=fs.readdirSync('/proc').filter(x=>/^\d+$/.test(x)&&Number(x)>1);if(ids.length>4096)throw Error('process_bound');return ids.flatMap(x=>{try{return [identity(Number(x))]}catch(e){if(e.code==='ENOENT'||e.code==='ESRCH')return [];throw e}})} function sample(){ @@ -166,6 +169,13 @@ const server=net.createServer(socket=>{sockets.add(socket);socket.on('close',()= else if(r.op==='arm'){if(waiters.size)result={armed:true,sealed};else{armWaiters.add(socket);socket.on('close',()=>armWaiters.delete(socket));return}} else if(r.op==='publish'){if(sealed||publishedHash||r.path!==config.actionFile||typeof r.text!=='string'||Buffer.byteLength(r.text)>16384)throw Error('publish_bound');const p=path.join(config.binding.remoteCwd,config.actionFile);if(fs.realpathSync(path.dirname(p))!==path.dirname(p))throw Error('publish_parent');publishedHash=hash(r.text);try{fs.writeFileSync(p,r.text,{flag:'wx',mode:0o600})}catch(e){publishedHash=null;throw e}result={path:r.path,sha256:publishedHash,published:true};} else if(r.op==='read'){const p=r.path==='@cross-root'&&config.crossRoot?path.join(config.root,'cross-root-target'):path.join(config.binding.remoteCwd,r.path);if(r.path!=='@cross-root'&&!config.targets.includes(r.path))throw Error('unregistered_read');const status=file(p);if(status.absent)throw Error('file_missing');result={...status,base64:readSafe(p).toString('base64')};} + else if(r.op==='pi-provider-death'){ + if(piFaultAttempted||sealed||!publishedHash||!root||!complete||typeof r.runtimeEnvironmentLeaseId!=='string'||r.runtimeEnvironmentLeaseId!==observedPrpEnvironmentLeaseId)throw Error('pi_provider_identity_or_signal_failed'); + piFaultAttempted=true;const before=sample();if(!before.captured||!before.live.includes(root.pid))throw Error('pi_provider_identity_or_signal_failed'); + const request={root:{pid:root.pid,ppid:root.ppid,startTicks:root.startTicks,bootId:root.bootId},binding:config.binding,runtimeEnvironmentLeaseId:r.runtimeEnvironmentLeaseId,runnerdSha256:config.runnerdSha256,closureSha256:PI_CLOSURE_PIN}; + const childResult=cp.spawnSync('/usr/bin/python3',['-I','-c',PI_FAULT_SOURCE,Buffer.from(JSON.stringify(request)).toString('base64')],{env:{PATH:'/usr/bin:/bin'},timeout:8000,maxBuffer:32768,encoding:'utf8'}); + if(childResult.status!==0||childResult.error)throw Error('pi_provider_identity_or_signal_failed');result=JSON.parse(childResult.stdout); + } else if(r.op==='attached'){if(attached||sealed)throw Error('attached_already_configured');if(!config.targets.includes(r.marker)||!Number.isInteger(r.delayMs)||r.delayMs<100||r.delayMs>8000)throw Error('attached_bounds'); attached={connections:0,failure:null,commandExit:null,markerWrittenAtMs:null,markerWrittenMonotonicNs:null,clientExitedAtMs:null,clientExitedMonotonicNs:null}; const clientScript=path.join(config.root,'client.cjs'),clientSocket=path.join(config.root,'attached.sock'); @@ -186,7 +196,7 @@ server.listen(path.join(config.root,'control.sock'));setTimeout(()=>{complete=fa `; const ATTACHED_CLIENT = String.raw`const net=require('node:net');const s=net.connect(process.argv[2]);let b='';s.setTimeout(15000,()=>process.exit(3));s.on('error',()=>process.exit(4));s.on('connect',()=>s.write(JSON.stringify({nonce:process.argv[3],pid:process.pid})+'\n'));s.on('data',x=>{b+=x;if(b.length>1024)process.exit(6);if(b.includes('\n')){const r=JSON.parse(b);s.end();process.exit(r.code===0?0:5)}});`; function observerSource() { - return OBSERVER.replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString()) + return OBSERVER.replace("PI_FAULT_PROGRAM", () => JSON.stringify(PI_FAULT_SOURCE)).replace("PI_CLOSURE_DIGEST", () => JSON.stringify(PI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"])).replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString()) .replace("WATCH_TARGET", () => createRemoteTargetWatch.toString()).replace("ATTACHED_CLIENT", () => JSON.stringify(ATTACHED_CLIENT)); } const RPC = String.raw`const fs=require('node:fs'),net=require('node:net'),cp=require('node:child_process'),crypto=require('node:crypto');const r=JSON.parse(Buffer.from(process.argv[1],'base64').toString());const hash=x=>'sha256:'+crypto.createHash('sha256').update(x).digest('hex'); @@ -204,6 +214,26 @@ function rpcSource() { return RPC.replace("RPC_CODES", () => JSON.stringify(RPC_CODES)).replace("RPC_PHASES", () => JSON.stringify(RPC_PHASES)).replace("PARSE_STAT", () => parseRemoteProcStat.toString()).replace("RUN_ROOT", () => isRemoteRunRoot.toString()); } +export interface PiProviderDeathReceipt { + schema: "paperclip.e2e.pi-provider-death.v1"; binding: RemoteNativeBinding; runtimeEnvironmentLeaseId: string; + target: RemoteProcessIdentity; ancestry: RemoteProcessIdentity[]; nodeSha256: string; entrypointSha256: string; closureSha256: string; + entrypointAttribution: "pinned_wrapper_parent"; originalChildArgvAvailable: false; observedChildTitle: "pi"; + signalled: true; signal: "SIGKILL"; targetKind: "pi_native_child"; workerSignalled: false; +} +export function validatePiProviderDeathReceipt(value: unknown, binding: RemoteNativeBinding, root: RemoteProcessIdentity, lease: string): PiProviderDeathReceipt { + const r = record(value), target = record(r.target), ancestry = Array.isArray(r.ancestry) ? r.ancestry.map(record) : []; + fail(r.schema === "paperclip.e2e.pi-provider-death.v1" && JSON.stringify(r.binding) === JSON.stringify(binding) + && r.entrypointAttribution === "pinned_wrapper_parent" && r.originalChildArgvAvailable === false && r.observedChildTitle === "pi" + && r.runtimeEnvironmentLeaseId === lease && r.signalled === true && r.signal === "SIGKILL" && r.targetKind === "pi_native_child" && r.workerSignalled === false + && sha(r.nodeSha256) && sha(r.entrypointSha256) && r.closureSha256 === PI_DISTRIBUTION_CLOSURE_SHA256["linux-x64"], "pi_fault_receipt"); + fail(ancestry.length >= 3 && ancestry.length <= 64 && new Set(ancestry.map(p => p.pid)).size === ancestry.length + && ancestry.every(p => Number.isSafeInteger(p.pid) && Number(p.pid) > 1 && Number.isSafeInteger(p.ppid) && Number(p.ppid) > 0 + && typeof p.startTicks === "string" && /^\d+$/u.test(p.startTicks) && p.bootId === root.bootId) + && ["pid", "ppid", "startTicks", "bootId"].every(k => ancestry[0]?.[k] === target[k] && ancestry.at(-1)?.[k] === record(root)[k]) + && ancestry.slice(0, -1).every((p, i) => p.ppid === ancestry[i + 1]?.pid) && target.pid !== root.pid, "pi_fault_ancestry"); + return value as PiProviderDeathReceipt; +} + export interface RemoteNativeFixture { readonly binding: RemoteNativeBinding; readonly remoteCwd: string; @@ -212,6 +242,8 @@ export interface RemoteNativeFixture { /** Watchers and exact run-root identity are already armed at return from bind. */ readonly baseline: RemoteNativeSnapshot; snapshot(label: string): Promise; + /** One-shot fault against the exact admitted Pi child; never a worker kill. */ + terminatePiProvider?(runtimeEnvironmentLeaseId: string): Promise; readFile(path: string): Promise; publishAction(path: string, text: string): Promise; setupAttachedCommand(input: { marker: string; markerText: string; delayMs: number }): Promise<{ command: string; commandSha256: string }>; @@ -424,6 +456,7 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption stopReceipt(); throw error; } + let piFaultAttempted = false; let closed = false, published = false, finished: RemoteNativeSnapshot | undefined; const retainedFiles = new Map(); return { @@ -433,6 +466,12 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption fail(!closed && !finished, "fixture_closed"); return readSnapshot(await rpc({ op: "snapshot" }), binding!, names, actionFile, options.runnerdSha256); }, + async terminatePiProvider(runtimeEnvironmentLeaseId) { + fail(!closed && !finished && published && !piFaultAttempted && /^[A-Za-z0-9._:-]{1,240}$/u.test(runtimeEnvironmentLeaseId), "pi_fault_admission"); + piFaultAttempted = true; // Uncertain delivery never retries the signal. + const receipt = await rpc({ op: "pi-provider-death", runtimeEnvironmentLeaseId }); + return validatePiProviderDeathReceipt(receipt, binding!, baseline.processes.root!, runtimeEnvironmentLeaseId); + }, async readFile(path) { fail(!closed && names.includes(path), "unregistered_read"); if (finished) { const bytes = retainedFiles.get(path); fail(bytes, "terminal_file_missing"); return Buffer.from(bytes!); }