From 8ce546da0eb936b39e40bd2cab4e83b9194ae8da Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 5 Oct 2026 21:14:26 -0500 Subject: [PATCH] Fix Pi memory parent setup and retired observer cleanup --- .../2026-10-02-pi-production-readiness.md | 66 ++++++++++++++++++- tests/runner-e2e/README.md | 12 ++++ tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/pi-controls-evidence.test.ts | 4 +- tests/runner-e2e/pi-native-cases.ts | 2 + tests/runner-e2e/pi-native-flow.ts | 12 +++- .../runner-e2e/pi-native-remote-flow.test.ts | 14 ++-- .../runner-e2e/remote-native-fixtures.test.ts | 42 ++++++++++++ tests/runner-e2e/remote-native-fixtures.ts | 36 +++++++--- 9 files changed, 169 insertions(+), 21 deletions(-) diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index aeb0a9fcbe2..bbaabe5135f 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -13,7 +13,67 @@ Runner prompts pass at private definitions revision `7b112ffe3` with unchanged graders, one attempt per correction and zero automatic retries. Their original failures remain retained. The Mac file-edit correction also passes. -### Latest boundary fixes — 2026-10-05 +### Current fixture corrections and remaining failures — 2026-10-05 + +Shipping runtime remains `f5024863e`; these changes affect only fixtures and +documentation. Pi native definition v14 is +`8a93306b6df0008ff10b5398049fad7b4bfa7a7cbf4f4ee0dedfe76eed6f0111`. +The model, profile 15, low thinking, byte grades, deadlines and automatic retry +count remain unchanged. All 691 Pi fixture tests across 20 files and the final +harness typecheck pass. + +The two explicit v13 attempts at harness `b3318d6be` remain failed. Restart +retains the original question, run, turn, session and producer and accepts the +exact browser answer. Its provider run succeeds, but the observer is incomplete +and canonical cleanup fails after lease release. The memory attempt also +completes its provider turn. Its retained public managed-file response has +32 bytes without LF, so the exact 33-byte assertion fails before the fresh +task. Its observer and canonical cleanup also fail. Both evidence manifests +have no missing files or reported leaks. Independent cleanup verifies both +owned child sandboxes absent. Original grades and receipt hashes are unchanged. + +The observer now retains bounded, closed failure reasons. A valid terminal +receipt can close the observer without a second RPC after lease release only +when it proves captured processes retired and has complete evidence or known +filesystem-watch failures. Unknown causes, reused process identities and live +attached processes still require cleanup proof. An incomplete filesystem +receipt still fails qualification and cannot supply qualified file bytes. +The released-lease regression fails before this correction and passes afterward. + +Before memory-task admission, the fixture uses the public managed-file API to +create `memory/.pi-e2e-parent.txt`. The actual memory target remains absent. +Both turns must preserve the setup file. This avoids a new directory racing +strict watcher installation. Actual Linux observer calibration reproduces the +failure with an absent parent and passes with the seeded parent. Installed Pi +native write preserves 33 bytes in both controls. Generic transfer scratch +creation also reproduces an unwatched-directory failure; native sync avoids it. +This supports a restart-path hypothesis, but the v13 receipt does not identify +the actual restart cause. No speculative shipping correction is made. + +Normal installed Mac startup, the public parent API and actual browser prompt +submission now pass with zero model calls and cleanup passing. The submitted +JSON content contains all 33 bytes, including LF. The initial Mac startup +failure remains recorded: shared-memory capacity was exhausted. Capacity later +freed without host setting changes or stopping unrelated services. The Linux +parent API check also passes; its combined auxiliary browser probe remains +failed because it selected an unavailable browser cache. Earlier installed +Linux browser proof remains a separate passing receipt. + +The live v13 memory tool input is unavailable after owned temporary-root +cleanup. The saved 32-byte file does not prove whether the model omitted LF or +a product boundary removed it. Parent setup and cleanup corrections do not +address that byte failure and do not permit an unchanged paid memory retry. +Keep both live cases held until a concrete correction addresses the observed +failure. Then run one explicit attempt per correction, with zero automatic +retries. Fresh full Product 26/26 and Runner 7/7 proof, Intel installation, +complete workspace tests, latest-head CI/review and prerequisite disposition +remain required. Current-runtime qualified counts remain zero. + +Final read-only key accounting observes $0.324966594 used, $4.675033406 remaining, +zero BYOK usage and the unchanged $5 lifetime cap without reset. Billing remains +provisional. The $100 campaign ceiling and no-merge/no-release instruction remain. + +### Prior v13 boundary corrections — 2026-10-05 The next fixture correction preserves the same `f5024863e` shipping inputs. The observer now accepts a directory notification only for the same device/inode @@ -264,11 +324,11 @@ local tests and release review are still required. | --- | --- | --- | | Runtime identity and admission | Exact runtime/profile/model; verified effective thinking; startup below the 60-second admission limit; unsupported configuration fails before a prompt | Shipping runtime `f5024863e` passes normal profile-15 admission on ARM Mac and native Linux. Completed live runs confirm the frozen model and native low. Fresh Intel admission for these shipping bytes remains required. Profile-14 and earlier profile-15 results below are historical. | | Live lifecycle | Pending native question survives controller restart in the original process; one creation and resolution; stale answer after provider death rejected; Stop retires owned processes; three warm turns keep process identity | Current Linux restart retains the same native question/run/turn/session/producer and the browser answer; the run succeeds, but its incomplete independent terminal evidence and failed canonical cleanup keep the case failed. Earlier `b012b3aeb` provider-death and native-question passes do not qualify `f5024863e`. | -| Product workflows | All 26 explicit Pi Product E2E cells (13 local, 13 Daytona); screenshots, public state, independent artifacts, terminal and cleanup evidence | Not qualified at `f5024863e`. Current Mac memory fails exact saved bytes with cleanup passing. Current Linux restart fails terminal evidence and canonical cleanup; its owned resources are independently absent. Both original grades remain failed. Historical `0bd040093` has 21/26 passes, kept only as history. Fresh complete current-runtime coverage is required. | +| Product workflows | All 26 explicit Pi Product E2E cells (13 local, 13 Daytona); screenshots, public state, independent artifacts, terminal and cleanup evidence | Not qualified at `f5024863e`. Mac and latest Linux memory attempts fail exact saved bytes. Latest Linux restart fails independent terminal evidence. Both v13 Linux attempts fail canonical cleanup; owned children are independently absent. V14 fixture calibration and 691 tests pass, but do not regrade those cases. Historical `0bd040093` has 21/26 passes, kept only as history. Fresh complete current-runtime coverage is required. | | Runner protocol | Pi roster passes through the native packaged runner and authenticated mock control plane; lifecycle/denial/control cases remain distinct from Product tests | Current-runtime Runner qualification remains required. Historical `0bd040093` has 7/7 passes at private definitions `7b112ffe3`, with unchanged graders and zero automatic retries. Those passes retain their original source labels. | | Installed distribution | Public CLI/server tars on ARM Mac, Intel Mac and Linux; normal Pi setup; exact Linux companion and immutable Daytona image imported without binary override | Normal ARM public install verifies 722 packages and 79,373 files at `f5024863e`. Normal native Linux install, companion import, Pi setup and admission pass at immutable candidate image `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:20c7fffddeee4298830ead1ea3f0b70819f61e549279588b605440d1155f7260`. No binary override is used. This image is not release-qualified. Fresh Intel proof remains required; earlier three-platform proofs stay historical. | | Governance and spend | Company isolation, human-only permission, duplicate/stale answers, Stop and budget hard stop; pricing estimates never become claimed bills | Historical 58 focused invariants and live pending-permission Stop keep their source labels. Fresh shipping qualification remains required. The dedicated key retains a $5 lifetime cap, zero BYOK usage and the $100 campaign ceiling. Last read-only accounting observes $0.319832563 used and $4.680167437 remaining, with delayed settlement still possible. The key cap is not proof of Paperclip budget enforcement. | -| Integration and rollout | Review each prerequisite; final-head typecheck, tests, build and CI pass; no unresolved review; exact artifacts; rollback recorded | Full build and recursive typecheck pass at shipping runtime `f5024863e`. At harness head `df748c997`, 53 CI checks pass with two skips. The evidence-only follow-up passes 207 affected tests and harness typecheck; its final head still needs CI and review. The full local workspace test command remains failed. Prerequisite #14921's valid production-admission finding remains open. Complete current-runtime qualification and prerequisite disposition remain required. No merge or release is authorized. | +| Integration and rollout | Review each prerequisite; final-head typecheck, tests, build and CI pass; no unresolved review; exact artifacts; rollback recorded | Full build and recursive typecheck pass at shipping runtime `f5024863e`. V14 fixture changes pass 691 tests and harness typecheck; latest-head CI and review must be recorded separately. The full local workspace test command remains failed. Prerequisite #14921's valid production-admission finding remains open. Complete current-runtime qualification and prerequisite disposition remain required. No merge or release is authorized. | ## Bounded execution diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 87e387e7c4f..d422a6853e3 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1602,6 +1602,18 @@ and unknown notifications remain incomplete. Pi native definition v13 also gives memory content a single JSON representation, including the required final LF; the exact managed bytes and fresh-task readback assertions remain unchanged. +Pi native definition v14 seeds `memory/.pi-e2e-parent.txt` through the public +managed-file API before task admission. This creates the watched parent while +leaving `memory/pi-native.txt` absent; the native write still has to supply all +33 bytes and both turns must preserve the setup file. New or replaced watched +directories still fail the oracle. Closed incompleteness reasons identify watch +gaps or process ambiguity without retaining paths or raw errors. An incomplete +receipt remains failed. A validated terminal receipt with a captured, retired +process tree closes its observer without another RPC to a publicly deleted lease +only when evidence is complete or its failures are known filesystem-watch gaps. +Unknown causes, reused identities and live attached processes still need cleanup +proof. + `--suite rich-acp-warm-continuity` adds six explicit cells: all three providers on local and Daytona. Three browser-driven turns must preserve native session, provider session, runner instance, PID/start identity and project workspace. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index fd57d701f0c..d4785e2b5bc 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1114,7 +1114,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 10, excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial", "pi-native.runner-acpx-pi.local.native-pending-provider-death"], - definitionMetadata: { version: 13, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-memory-single-json-content-including-final-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" }, + definitionMetadata: { version: 14, agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-memory-single-json-content-including-final-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" }, }, { id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index aaf04f0495c..b92a05e79ae 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -189,7 +189,7 @@ describe("Pi controls catalog admission", () => { // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 13, profileVersion: 15, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-memory-single-json-content-including-final-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 14, agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 15, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-memory-single-json-content-including-final-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 8, profileVersion: 15, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); @@ -200,7 +200,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "8ac5fa3e8dc75486c56a92364b1c9e141011ae1bd04a431d6c3ae2f237f99b58", + "pi-native": "8a93306b6df0008ff10b5398049fad7b4bfa7a7cbf4f4ee0dedfe76eed6f0111", "native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d", "rich-acp-warm-continuity": "036c0faebc2f6eee5cd22ea38887c9c22650a83561fd08ee83473a565b11bb00", "extended-harnesses": "121f4cf75267bcdb003e5ed59e165755b2ac0c3d31c82dc217a62c8abe3d1b4d", diff --git a/tests/runner-e2e/pi-native-cases.ts b/tests/runner-e2e/pi-native-cases.ts index baa98f65bd6..61455e29d09 100644 --- a/tests/runner-e2e/pi-native-cases.ts +++ b/tests/runner-e2e/pi-native-cases.ts @@ -1,6 +1,8 @@ import type { RunnerTaskFixture } from "./types.js"; export const PI_NATIVE_MEMORY_PATH = "memory/pi-native.txt"; +export const PI_NATIVE_MEMORY_PARENT_SEED_PATH = "memory/.pi-e2e-parent.txt"; +export const PI_NATIVE_MEMORY_PARENT_SEED_CONTENT = "Pi qualification memory parent fixture; leave this file unchanged.\n"; export const piNativeTasks: readonly RunnerTaskFixture[] = [ ["native-questions", "Four native questions survive browser reconnect", 1], ["agent-files-fresh-run", "Agent files save and survive a fresh task", 2], diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index c19c76038c8..91177bb0d9b 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -8,7 +8,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; -import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js"; +import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -162,6 +162,14 @@ export async function runPiNativeFlow(input: { await input.evidence("pi-native-typed-proof.json", { proof, interactions: final.interactions, events: await events(runs[0]!.id) }); } else if (execution.task.id === "agent-files-fresh-run") { const retained = randomBytes(16).toString("hex"); const outside = resolve(input.workspacePath, "..", `pi-unassigned-${nonce}.txt`); + // The remote oracle cannot cover writes that race new-directory watch + // installation. Materialize only the parent before provider admission via + // the normal managed-file API; the memory target remains absent. + const seed = await api.request.put(`/api/agents/${fixtures.agent.id}/instructions-bundle/file`, { + data: { path: PI_NATIVE_MEMORY_PARENT_SEED_PATH, content: PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, baseHash: null }, + }); + if (!seed.ok()) throw new Error(`Pi memory parent setup failed (${seed.status()})`); + check("memory-parent-seeded", (await seed.json()).content === PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, "Public managed-file setup created the watched memory parent before task admission"); if (!remote) check("unassigned-initially-absent", await absent(outside), "The isolated cross-root marker did not already exist"); const prompt = (outsidePath: string) => piNativeMemoryPrompt(retained, outsidePath); await create(execution.task.buildTitle(nonce), remote ? fixture => { if (!fixture.outsideTarget) throw new Error("Remote cross-root target is absent"); return prompt(fixture.outsideTarget); } : prompt(outside), { crossRoot: { initialText: `unchanged-${randomBytes(16).toString("hex")}` } }); @@ -184,6 +192,8 @@ export async function runPiNativeFlow(input: { check("fresh-run-readback", await readWorkspace("pi-agent-memory-proof.txt") === `${retained}\n`, "A new issue after server restart copied the undisclosed saved agent-file bytes"); const current = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`); check("persistent-bytes-unchanged", current.content === personal.content, "Fresh-run readback preserved the saved managed bytes"); + const parentSeed = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PARENT_SEED_PATH)}`); + check("memory-parent-seed-unchanged", parentSeed.content === PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, "Both native turns preserved the parent setup file"); await input.evidence("pi-agent-files-fresh-read.json", { current, runs, events: await events(runs[1]!.id) }); } else if (execution.task.id === "human-permission-denial") { if (!input.registerCleanupAssertion) throw new Error("Pi human denial requires post-retirement cleanup assertions"); diff --git a/tests/runner-e2e/pi-native-remote-flow.test.ts b/tests/runner-e2e/pi-native-remote-flow.test.ts index ead29865ed8..e83d0556966 100644 --- a/tests/runner-e2e/pi-native-remote-flow.test.ts +++ b/tests/runner-e2e/pi-native-remote-flow.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { expect, it, vi } from "vitest"; -import { piNativeTasks } from "./pi-native-cases.js"; +import { piNativeTasks, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT } from "./pi-native-cases.js"; import { runPiNativeFlow } from "./pi-native-flow.js"; const browser = vi.hoisted(() => ({ create: (_value: any) => {} })); vi.mock("./user-actions.js", () => ({ createTaskThroughUi: async (value: unknown) => browser.create(value) })); @@ -13,18 +13,24 @@ it.each([false, true])("retains managed bytes before grading and rebinds success const root = await mkdtemp(join(tmpdir(), "pi-remote-memory-")); const task = piNativeTasks.find(row => row.id === "agent-files-fresh-run")!; const issues: any[] = [], runs: any[] = [], cleanup: Array<() => Promise> = []; const captures: any[] = [], evidence = new Map(); - let personal = "", restarted = false, readAfterFinish = false; + let personal = "", restarted = false, readAfterFinish = false, parentSeeded = false; browser.create = value => { + expect(parentSeeded).toBe(true); expect(value.prompt).toMatch(/^Read only bootstrap-/); expect(value.prompt).not.toContain("forbidden"); const n = issues.length + 1; issues.push({ id: `issue-${n}`, title: value.title, status: "in_progress" }); runs.push({ id: `run-${n}`, status: "running", runtimeMode: "native", createdAt: `2026-09-29T00:00:0${n}Z` }); }; - const api = { post: async () => ({ name: "Pi remote project" }), get: async (path: string) => { + const api = { request: { put: async (path: string, input: any) => { + expect(path).toBe("/api/agents/agent/instructions-bundle/file"); + expect(input.data).toEqual({ path: PI_NATIVE_MEMORY_PARENT_SEED_PATH, content: PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, baseHash: null }); + expect(issues).toHaveLength(0); expect(personal).toBe(""); parentSeeded = true; + return { ok: () => true, json: async () => ({ content: PI_NATIVE_MEMORY_PARENT_SEED_CONTENT }) }; + } }, post: async () => ({ name: "Pi remote project" }), get: async (path: string) => { if (path.endsWith("/issues?limit=100")) return issues; if (path.endsWith("/heartbeat-runs?limit=100")) return runs; if (/\/api\/issues\/issue-[12]$/.test(path)) return issues.find(row => path.endsWith(row.id)); if (/\/api\/heartbeat-runs\/run-[12]$/.test(path)) return runs.find(row => path.endsWith(row.id)); if (path.endsWith("/interactions") || path.endsWith("/comments")) return []; - if (path.includes("instructions-bundle/file?")) return { content: personal }; + if (path.includes("instructions-bundle/file?")) return { content: decodeURIComponent(path.split("?path=")[1]!) === PI_NATIVE_MEMORY_PARENT_SEED_PATH ? PI_NATIVE_MEMORY_PARENT_SEED_CONTENT : personal }; if (path.includes("/events?")) return path.includes("run-1/") ? [{ eventType: "instruction_save", seq: 1, payload: { state: "saved" } }, { eventType: "tool.execution.completed", seq: 2, payload: { prpEvent: { payload: { schema: "paperclip.tool.execution.v1", transport: "builtin", operation: "edit", name: "write", status: "failed", target: null, executionId: "tool-1", output: "Pi tool path is outside its assigned workspace and agent files" } } } }] : []; throw new Error(`Unexpected fixture path ${path}`); } }; diff --git a/tests/runner-e2e/remote-native-fixtures.test.ts b/tests/runner-e2e/remote-native-fixtures.test.ts index 6757b5b20b9..896b7070d6a 100644 --- a/tests/runner-e2e/remote-native-fixtures.test.ts +++ b/tests/runner-e2e/remote-native-fixtures.test.ts @@ -244,11 +244,53 @@ describe("remote native lease admission", () => { const retained = remoteNativeIncompleteTerminalEvidence(error)!; expect(retained.complete).toBe(false); expect(retained.watcher.complete).toBe(false); expect(retained.processes.liveCount).toBe(0); + expect(retained.incompleteReasons).toEqual([]); expect(JSON.stringify(retained)).not.toContain("PRIVATE RPC CONTENT"); retained.watcher.complete = true; expect(remoteNativeIncompleteTerminalEvidence(error)!.watcher.complete).toBe(false); expect(remoteNativeIncompleteTerminalEvidence(new Error("PRIVATE RPC CONTENT"))).toBeUndefined(); }); + it("retains only closed incompleteness reasons and rejects raw diagnostics", async () => { + const h = harness(), f = await bindRemoteNativeFixture(h.options); await f.publishAction("action.txt", "task"); + h.resolveTerminal({ ...structuredClone(h.current), complete: false, incompleteReasons: ["unwatched_directory"], + processes: { ...h.current.processes, live: [] }, files: {} }); + const error = await f.finish().catch(error => error); + expect(remoteNativeIncompleteTerminalEvidence(error)?.incompleteReasons).toEqual(["unwatched_directory"]); + await f.close(); + const bad = harness(), other = await bindRemoteNativeFixture(bad.options); await other.publishAction("action.txt", "task"); + bad.resolveTerminal({ ...structuredClone(bad.current), complete: false, incompleteReasons: ["PRIVATE RAW PATH"], + processes: { ...bad.current.processes, live: [] }, files: {} }); + const rejected = await other.finish().catch(error => error); + expect(rejected.message).toContain("incomplete_reasons_shape"); + expect(remoteNativeIncompleteTerminalEvidence(rejected)).toBeUndefined(); + await other.close(); + }); + it("keeps incomplete evidence failed while closing a retired observer after public lease deletion", async () => { + const h = harness(), f = await bindRemoteNativeFixture(h.options); + await f.publishAction("action.txt", "task"); + h.resolveTerminal({ ...structuredClone(h.current), complete: false, incompleteReasons: ["unwatched_directory"], + watcher: { ...h.current.watcher, complete: false }, processes: { ...h.current.processes, live: [] }, files: {} }); + await expect(f.finish()).rejects.toThrow("terminal_evidence_incomplete"); + h.setLease({ ...h.lease(), status: "released", releasedAt: "2026-10-06T01:00:00Z" }); + const before = h.calls.length; + await expect(f.close()).resolves.toBeUndefined(); + expect(h.calls).toHaveLength(before); + await expect(f.finish()).rejects.toThrow("terminal_evidence_incomplete"); + await expect(f.readFile("result.txt")).rejects.toThrow("unregistered_read"); + }); + it.each(["live", "missing-root", "invalid-shape", "unknown-cause", "reused-process", "attached-live"])("does not treat %s evidence as retired cleanup", async kind => { + const h = harness(), f = await bindRemoteNativeFixture(h.options); + await f.publishAction("action.txt", "task"); + const end: any = { ...structuredClone(h.current), complete: false, processes: { ...h.current.processes, live: [] }, files: {} }; + if (kind === "live") end.processes.live = [21]; + if (kind === "missing-root") end.processes = { captured: false, root: null, journal: [], live: [] }; + if (kind === "invalid-shape") end.observedAtMs = "invalid"; + if (kind === "reused-process") end.incompleteReasons = ["process_identity_reused"]; + if (kind === "attached-live") end.incompleteReasons = ["attached_process_live"]; + h.resolveTerminal(end); await expect(f.finish()).rejects.toThrow(); + h.setLease({ ...h.lease(), status: "released", releasedAt: "2026-10-06T01:00:00Z" }); + await expect(f.close()).rejects.toThrow("lease_scope"); + }); it("keeps a fixture-owned cross-root sentinel distinct from workspace targets", async () => { const h = harness(); h.options.crossRoot = { initialText: "outside sentinel" }; h.current.targets["@cross-root"] = { absent: false, sha256: hash("outside sentinel"), parent: { dev: "1", ino: "outside" }, mutationCount: 0, complete: true }; diff --git a/tests/runner-e2e/remote-native-fixtures.ts b/tests/runner-e2e/remote-native-fixtures.ts index 7c87669ddce..588e786f020 100644 --- a/tests/runner-e2e/remote-native-fixtures.ts +++ b/tests/runner-e2e/remote-native-fixtures.ts @@ -51,12 +51,16 @@ export function remoteNativeFixtureDiagnostics(error: unknown): RpcDiagnostic[] export interface RemoteNativeAuthority { companyId: string; environmentId: string; runId: string; leaseId: string; sandboxId: string; image: string } export interface RemoteNativeBinding extends RemoteNativeAuthority { remoteCwd: string } export interface RemoteProcessIdentity { pid: number; ppid: number; startTicks: string; bootId: string } +const INCOMPLETE_REASONS = ["multiple_run_roots", "run_root_changed", "process_identity_reused", "process_sample_failed", "unknown_watch_name", "setup_file_changed", "workspace_event_bound", "workspace_symlink", "unwatched_directory", "directory_identity_changed", "watch_stat_failed", "workspace_watch_error", "target_watch_incomplete", "receipt_byte_bound", "attached_process_live", "control_request_bound", "observer_ttl"] as const; +type IncompleteReason = typeof INCOMPLETE_REASONS[number]; +const FILESYSTEM_INCOMPLETE_REASONS: readonly IncompleteReason[] = ["unknown_watch_name", "setup_file_changed", "workspace_event_bound", "workspace_symlink", "unwatched_directory", "directory_identity_changed", "watch_stat_failed", "workspace_watch_error", "target_watch_incomplete"]; export interface RemoteNativeSnapshot { binding: RemoteNativeBinding; observedAtMs: number; observedMonotonicNs: string; receivedAtMs: number; complete: boolean; + incompleteReasons?: IncompleteReason[]; workspace: Record; targets: Record; watcher: { complete: boolean; targetMutationCount: number; workspaceMutationCount: number }; @@ -81,6 +85,7 @@ export function remoteNativeIncompleteTerminalEvidence(error: unknown) { const row = error.snapshot, root = row.processes.root; return { observedAtMs: row.observedAtMs, receivedAtMs: row.receivedAtMs, complete: row.complete, + incompleteReasons: [...(row.incompleteReasons ?? [])], setupPublished: row.setup.published, watcher: { complete: row.watcher.complete, targetMutationCount: row.watcher.targetMutationCount, workspaceMutationCount: row.watcher.workspaceMutationCount }, processes: { captured: row.processes.captured, liveCount: row.processes.live.length, journalCount: row.processes.journal.length, @@ -149,6 +154,7 @@ const cwdStat=fs.lstatSync(config.binding.remoteCwd,{bigint:true}),rootStat=fs.l const runtimeRoot=path.join(config.binding.remoteCwd,config.runtimeRelative),runtimeStat=fs.lstatSync(runtimeRoot,{bigint:true});if(!runtimeStat.isDirectory()||runtimeStat.isSymbolicLink()||fs.realpathSync(runtimeRoot)!==runtimeRoot)throw Error('runtime_root_identity');let observedPrpEnvironmentLeaseId=null; const boot=fs.readFileSync('/proc/sys/kernel/random/boot_id','utf8').trim(); const targets=new Map();let complete=true,sealed=false,root=null,attached=null,child=null,client=null,childTimer=null; +const incompleteReasons=new Set();function incomplete(reason){complete=false;incompleteReasons.add(reason)} const journal=new Map(),sockets=new Set(),waiters=new Set(),armWaiters=new Set();let observedRootCount=0,publishedHash=null,finalReceipt=null,retiring=false,piFaultAttempted=false; function identity(pid){return parseStat(pid,fs.readFileSync('/proc/'+pid+'/stat','utf8'),boot)} function table(){const ids=fs.readdirSync('/proc').filter(x=>/^\d+$/.test(x)&&Number(x)>1);if(ids.length>4096)throw Error('process_bound');return ids.flatMap(x=>{try{return [identity(Number(x))]}catch(e){if(e.code==='ENOENT'||e.code==='ESRCH')return [];throw e}})} @@ -156,11 +162,11 @@ function sample(){ const all=table();const candidates=[]; for(const p of all){if(p.state==='Z')continue;let argv;try{argv=fs.readFileSync('/proc/'+p.pid+'/cmdline').toString().split('\0').filter(Boolean)}catch(e){if(e.code==='ENOENT'||e.code==='ESRCH')continue;throw e} if(runRoot(argv,config.binding.runId,p)){const at=argv.indexOf('--environment-lease-id'),stateAt=argv.indexOf('--state-dir');if(at<1||argv.lastIndexOf('--environment-lease-id')!==at||!/^[-a-zA-Z0-9._:]{1,256}$/.test(argv[at+1]??''))throw Error('process_prp_identity_shape');if(argv[0]!==path.join(runtimeRoot,'bin','paperclip-runnerd')||stateAt<1||argv.lastIndexOf('--state-dir')!==stateAt||!argv[stateAt+1]?.startsWith(runtimeRoot+'/sessions/')||!/^([a-f0-9]{64})\/runner$/.test(argv[stateAt+1].slice((runtimeRoot+'/sessions/').length)))throw Error('process_runtime_binding');if(observedPrpEnvironmentLeaseId!==null&&observedPrpEnvironmentLeaseId!==argv[at+1])throw Error('process_prp_identity_changed');observedPrpEnvironmentLeaseId=argv[at+1];candidates.push(p);}} - if(candidates.length>1)complete=false; + if(candidates.length>1)incomplete('multiple_run_roots'); if(!root&&candidates.length===1){if(hash(fs.readFileSync('/proc/'+candidates[0].pid+'/exe'))!==config.runnerdSha256)throw Error('runner_binary_identity');root=candidates[0];journal.set(root.pid,root);observedRootCount++;} - if(root&&candidates.some(p=>p.pid!==root.pid||p.startTicks!==root.startTicks))complete=false; + if(root&&candidates.some(p=>p.pid!==root.pid||p.startTicks!==root.startTicks))incomplete('run_root_changed'); let changed=true;while(changed){changed=false;for(const p of all){const parent=journal.get(p.ppid);if(!journal.has(p.pid)&&((parent&&all.some(q=>q.pid===parent.pid&&q.startTicks===parent.startTicks))||(root&&p.group===root.pid))){if(journal.size>=512)throw Error('journal_bound');journal.set(p.pid,p);changed=true;}}} - if(all.some(p=>journal.has(p.pid)&&journal.get(p.pid).startTicks!==p.startTicks))complete=false; + if(all.some(p=>journal.has(p.pid)&&journal.get(p.pid).startTicks!==p.startTicks))incomplete('process_identity_reused'); const live=all.filter(p=>journal.get(p.pid)?.startTicks===p.startTicks&&p.state!=='Z').map(p=>p.pid); if(client&&!all.some(p=>p.pid===client.pid&&p.startTicks===client.startTicks&&p.state!=='Z')&&attached&&!attached.clientExitedAtMs){attached.clientExitedAtMs=Date.now();attached.clientExitedMonotonicNs=process.hrtime.bigint().toString();} return {captured:root!==null,root,journal:[...journal.values()],live}; @@ -171,7 +177,7 @@ function guard(){const rs=fs.lstatSync(runtimeRoot,{bigint:true});if(!rs.isDirec function readSafe(p){const fd=fs.openSync(p,fs.constants.O_RDONLY|fs.constants.O_NOFOLLOW);try{const before=fs.fstatSync(fd,{bigint:true});if(!before.isFile()||before.size>65536n)throw Error('file_bound_or_type');const bytes=fs.readFileSync(fd),after=fs.fstatSync(fd,{bigint:true}),named=fs.lstatSync(p,{bigint:true});if(before.dev!==named.dev||before.ino!==named.ino||named.isSymbolicLink()||before.size!==after.size||before.mtimeNs!==after.mtimeNs||BigInt(bytes.length)!==before.size)throw Error('file_changed');return bytes}finally{fs.closeSync(fd)}} function file(p){try{const s=fs.lstatSync(p);if(s.isSymbolicLink()||!s.isFile()||s.size>65536)throw Error('file_bound_or_type');return {absent:false,sha256:hash(readSafe(p))}}catch(e){if(e.code==='ENOENT')return {absent:true,sha256:null};throw e}} function workspace(){const result={};let count=0,bytes=0;function visit(dir,prefix){for(const name of fs.readdirSync(dir).sort()){if(++count>512)throw Error('workspace_entry_bound');const full=path.join(dir,name),rel=prefix+name,s=fs.lstatSync(full);if(rel===config.runtimeRelative)continue;if(rel===config.actionFile){if(!publishedHash||file(full).sha256!==publishedHash)throw Error('setup_file_changed');continue;}if(s.isSymbolicLink())throw Error('workspace_symlink');if(s.isDirectory()){result[rel]='directory';visit(full,rel+'/')}else if(s.isFile()){bytes+=s.size;if(s.size>65536||bytes>4194304)throw Error('workspace_byte_bound');result[rel]=hash(readSafe(full))}else throw Error('workspace_special_file')}}visit(config.binding.remoteCwd,'');return result} -function snapshot(){guard();verifyWorkspaceWatchRoots();const processes=sample(),out={};let watchComplete=complete,total=0;for(const [name,t] of targets){const status=t.watch.snapshot();out[name]={...file(t.path),...status};watchComplete&&=status.complete;total+=status.mutationCount}return {binding:config.binding,observedAtMs:Date.now(),observedMonotonicNs:process.hrtime.bigint().toString(),complete:complete&&watchComplete,workspace:workspace(),targets:out,watcher:{complete:watchComplete,targetMutationCount:total,workspaceMutationCount},processes,scope:{kind:'user_workspace',excludedRuntime:{relativePath:config.runtimeRelative,absolutePath:runtimeRoot,dev:String(runtimeStat.dev),ino:String(runtimeStat.ino),runnerExecutableSha256:config.runnerdSha256},observedPrpEnvironmentLeaseId,prpEnvironmentLeaseIdVerified:false},setup:{path:config.actionFile,sha256:publishedHash,published:publishedHash!==null},attached}} +function snapshot(){guard();verifyWorkspaceWatchRoots();const processes=sample(),out={},reasons=new Set(incompleteReasons);let watchComplete=complete,total=0;for(const [name,t] of targets){const status=t.watch.snapshot();out[name]={...file(t.path),...status};watchComplete&&=status.complete;if(!status.complete)reasons.add('target_watch_incomplete');total+=status.mutationCount}return {binding:config.binding,observedAtMs:Date.now(),observedMonotonicNs:process.hrtime.bigint().toString(),complete:complete&&watchComplete,incompleteReasons:[...reasons].sort(),workspace:workspace(),targets:out,watcher:{complete:watchComplete,targetMutationCount:total,workspaceMutationCount},processes,scope:{kind:'user_workspace',excludedRuntime:{relativePath:config.runtimeRelative,absolutePath:runtimeRoot,dev:String(runtimeStat.dev),ino:String(runtimeStat.ino),runnerExecutableSha256:config.runnerdSha256},observedPrpEnvironmentLeaseId,prpEnvironmentLeaseIdVerified:false},setup:{path:config.actionFile,sha256:publishedHash,published:publishedHash!==null},attached}} for(const name of config.targets){const p=path.join(config.binding.remoteCwd,name);if(fs.realpathSync(path.dirname(p))!==path.dirname(p))throw Error('target_parent_symlink');targets.set(name,{path:p,watch:watchTarget(path.dirname(p),path.basename(p),{watch:fs.watch,lstatSync:fs.lstatSync})})} if(config.crossRoot){const p=path.join(config.root,'cross-root-target');fs.writeFileSync(p,config.crossRoot.initialText,{flag:'wx',mode:0o600});targets.set('@cross-root',{path:p,watch:watchTarget(config.root,'cross-root-target',{watch:fs.watch,lstatSync:fs.lstatSync})})} let workspaceMutationCount=0;const directoryWatches=[]; @@ -179,14 +185,14 @@ let workspaceMutationCount=0;const directoryWatches=[]; // accept only the same inode with an already registered recursive watch. New // directories and replacements remain incomplete because their writes can race // watch installation. Every notification still counts as a workspace mutation. -function watchDirectory(directory,prefix=''){if(directoryWatches.length>=512)throw Error('watch_directory_bound');const before=fs.lstatSync(directory,{bigint:true});if(!before.isDirectory()||before.isSymbolicLink())throw Error('watch_directory_identity');const handle=fs.watch(directory,(_kind,name)=>{if(name===null){complete=false;return}const relative=prefix+String(name);if(relative===config.runtimeRelative)return;if(relative===config.actionFile){try{if(!publishedHash||file(path.join(config.binding.remoteCwd,config.actionFile)).sha256!==publishedHash)complete=false}catch{complete=false}return}workspaceMutationCount++;if(workspaceMutationCount>4096)complete=false;try{const changed=path.join(directory,String(name)),st=fs.lstatSync(changed,{bigint:true});if(st.isSymbolicLink())complete=false;else if(st.isDirectory()){const watched=directoryWatches.find(item=>item.directory===changed);if(!watched||st.dev!==watched.before.dev||st.ino!==watched.before.ino)complete=false}}catch(e){if(e.code!=='ENOENT')complete=false}});handle.on('error',()=>{complete=false});directoryWatches.push({directory,before,handle});for(const name of fs.readdirSync(directory)){const rel=prefix+name;if(rel===config.runtimeRelative)continue;const full=path.join(directory,name),s=fs.lstatSync(full);if(s.isSymbolicLink())throw Error('workspace_symlink');if(s.isDirectory())watchDirectory(full,rel+'/')}} +function watchDirectory(directory,prefix=''){if(directoryWatches.length>=512)throw Error('watch_directory_bound');const before=fs.lstatSync(directory,{bigint:true});if(!before.isDirectory()||before.isSymbolicLink())throw Error('watch_directory_identity');const handle=fs.watch(directory,(_kind,name)=>{if(name===null){incomplete('unknown_watch_name');return}const relative=prefix+String(name);if(relative===config.runtimeRelative)return;if(relative===config.actionFile){try{if(!publishedHash||file(path.join(config.binding.remoteCwd,config.actionFile)).sha256!==publishedHash)incomplete('setup_file_changed')}catch{incomplete('setup_file_changed')}return}workspaceMutationCount++;if(workspaceMutationCount>4096)incomplete('workspace_event_bound');try{const changed=path.join(directory,String(name)),st=fs.lstatSync(changed,{bigint:true});if(st.isSymbolicLink())incomplete('workspace_symlink');else if(st.isDirectory()){const watched=directoryWatches.find(item=>item.directory===changed);if(!watched)incomplete('unwatched_directory');else if(st.dev!==watched.before.dev||st.ino!==watched.before.ino)incomplete('directory_identity_changed')}}catch(e){if(e.code!=='ENOENT')incomplete('watch_stat_failed')}});handle.on('error',()=>{incomplete('workspace_watch_error')});directoryWatches.push({directory,before,handle});for(const name of fs.readdirSync(directory)){const rel=prefix+name;if(rel===config.runtimeRelative)continue;const full=path.join(directory,name),s=fs.lstatSync(full);if(s.isSymbolicLink())throw Error('workspace_symlink');if(s.isDirectory())watchDirectory(full,rel+'/')}} watchDirectory(config.binding.remoteCwd); const workspaceWatch={close(){for(const item of directoryWatches)item.handle.close()}}; function verifyWorkspaceWatchRoots(){for(const item of directoryWatches){const after=fs.lstatSync(item.directory,{bigint:true});if(after.dev!==item.before.dev||after.ino!==item.before.ino||!after.isDirectory()||after.isSymbolicLink())throw Error('workspace_watch_root_replaced')}} function publicSnapshot(){const result=snapshot();if(result.attached)result.attached={connections:attached.connections,failure:attached.failure,commandExit:attached.commandExit,markerWrittenAtMs:attached.markerWrittenAtMs,markerWrittenMonotonicNs:attached.markerWrittenMonotonicNs,clientExitedAtMs:attached.clientExitedAtMs,clientExitedMonotonicNs:attached.clientExitedMonotonicNs};return result} -function seal(){if(sealed)return;sealed=true;workspaceWatch.close();for(const t of targets.values())t.watch.close();clearInterval(observer);finalReceipt=publicSnapshot();finalReceipt.files={};for(const [name,t] of targets){if(!file(t.path).absent)finalReceipt.files[name]=readSafe(t.path).toString('base64');}if(Buffer.byteLength(JSON.stringify(finalReceipt))>250000){finalReceipt.complete=false;finalReceipt.files={};}if(child&&child.exitCode===null&&child.signalCode===null)finalReceipt.complete=false;for(const socket of waiters)socket.end(JSON.stringify({ok:true,result:finalReceipt})+'\n');waiters.clear();setTimeout(()=>shutdown(null),250);} -const observer=setInterval(()=>{try{const p=sample();if(p.captured&&p.live.length===0&&!retiring){retiring=true;setTimeout(()=>{try{const end=sample();if(end.live.length===0)seal();else retiring=false}catch{complete=false}},100)}}catch{complete=false}},25); -const server=net.createServer(socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));socket.on('error',()=>{});let buffer='';socket.on('data',data=>{buffer+=data;if(Buffer.byteLength(buffer)>8192){socket.destroy();complete=false;return}if(!buffer.includes('\n'))return;socket.removeAllListeners('data');try{const r=JSON.parse(buffer);if(r.nonce!==config.nonce)throw Error('control_identity');guard();let result; +function seal(){if(sealed)return;sealed=true;workspaceWatch.close();for(const t of targets.values())t.watch.close();clearInterval(observer);finalReceipt=publicSnapshot();finalReceipt.files={};for(const [name,t] of targets){if(!file(t.path).absent)finalReceipt.files[name]=readSafe(t.path).toString('base64');}if(Buffer.byteLength(JSON.stringify(finalReceipt))>250000){finalReceipt.complete=false;finalReceipt.incompleteReasons.push('receipt_byte_bound');finalReceipt.files={};}if(child&&child.exitCode===null&&child.signalCode===null){finalReceipt.complete=false;finalReceipt.incompleteReasons.push('attached_process_live')};for(const socket of waiters)socket.end(JSON.stringify({ok:true,result:finalReceipt})+'\n');waiters.clear();setTimeout(()=>shutdown(null),250);} +const observer=setInterval(()=>{try{const p=sample();if(p.captured&&p.live.length===0&&!retiring){retiring=true;setTimeout(()=>{try{const end=sample();if(end.live.length===0)seal();else retiring=false}catch{incomplete('process_sample_failed')}},100)}}catch{incomplete('process_sample_failed')}},25); +const server=net.createServer(socket=>{sockets.add(socket);socket.on('close',()=>sockets.delete(socket));socket.on('error',()=>{});let buffer='';socket.on('data',data=>{buffer+=data;if(Buffer.byteLength(buffer)>8192){socket.destroy();incomplete('control_request_bound');return}if(!buffer.includes('\n'))return;socket.removeAllListeners('data');try{const r=JSON.parse(buffer);if(r.nonce!==config.nonce)throw Error('control_identity');guard();let result; if(r.op==='snapshot')result=finalReceipt??publicSnapshot(); else if(r.op==='wait'){if(finalReceipt)result=finalReceipt;else{if(waiters.size)throw Error('duplicate_receipt_channel');waiters.add(socket);for(const arm of armWaiters)arm.end(JSON.stringify({ok:true,result:{armed:true,sealed:false}})+'\n');armWaiters.clear();socket.on('close',()=>waiters.delete(socket));return}} else if(r.op==='arm'){if(waiters.size)result={armed:true,sealed};else{armWaiters.add(socket);socket.on('close',()=>armWaiters.delete(socket));return}} @@ -215,7 +221,7 @@ const server=net.createServer(socket=>{sockets.add(socket);socket.on('close',()= socket.end(JSON.stringify({ok:true,result})+'\n'); }catch{socket.end(JSON.stringify({ok:false,error:'observer_evidence_incomplete'})+'\n')}})}); let closing=false;async function shutdown(reply){if(closing){reply?.end(JSON.stringify({ok:false,error:'closing'})+'\n');return}closing=true;sealed=true;workspaceWatch.close();for(const t of targets.values())t.watch.close();clearInterval(observer);let settled=true;try{if(child&&child.exitCode===null&&child.signalCode===null){child.kill('SIGTERM');const exited=new Promise(resolve=>child.once('exit',resolve));await Promise.race([exited,new Promise(resolve=>setTimeout(resolve,2000))]);if(child.exitCode===null&&child.signalCode===null){child.kill('SIGKILL');await Promise.race([exited,new Promise(resolve=>setTimeout(resolve,2000))]);}settled=child.exitCode!==null||child.signalCode!==null;}if(reply)reply.end(JSON.stringify({ok:settled,result:{closed:settled}})+'\n');}finally{for(const s of sockets)if(s!==reply)s.destroy();server.close();if(attached?.server)attached.server.close();setTimeout(()=>{reply?.destroy();if(settled){const current=fs.lstatSync(config.root,{bigint:true});if(current.dev===rootStat.dev&¤t.ino===rootStat.ino&&!current.isSymbolicLink())fs.rmSync(config.root,{recursive:true,force:true});else settled=false;}process.exit(settled?0:2)},100)}} -server.listen(path.join(config.root,'control.sock'));setTimeout(()=>{complete=false;shutdown(null)},config.observerTtlMs).unref(); +server.listen(path.join(config.root,'control.sock'));setTimeout(()=>{incomplete('observer_ttl');shutdown(null)},config.observerTtlMs).unref(); `; const ATTACHED_CLIENT = String.raw`const net=require('node:net');const s=net.connect(process.argv[2]);let b='';s.setTimeout(15000,()=>process.exit(3));s.on('error',()=>process.exit(4));s.on('connect',()=>s.write(JSON.stringify({nonce:process.argv[3],pid:process.pid})+'\n'));s.on('data',x=>{b+=x;if(b.length>1024)process.exit(6);if(b.includes('\n')){const r=JSON.parse(b);s.end();process.exit(r.code===0?0:5)}});`; function observerSource() { @@ -304,6 +310,9 @@ function readSnapshot(value: unknown, binding: RemoteNativeBinding, names: strin && /^\d+$/u.test(String(parent.dev)) && /^\d+$/u.test(String(parent.ino)), "target_shape"); } fail(typeof watcher.complete === "boolean" && [watcher.targetMutationCount, watcher.workspaceMutationCount].every(n => Number.isSafeInteger(n) && (n as number) >= 0), "watcher_shape"); + fail(row.incompleteReasons === undefined || (Array.isArray(row.incompleteReasons) && row.incompleteReasons.length <= INCOMPLETE_REASONS.length + && new Set(row.incompleteReasons).size === row.incompleteReasons.length + && row.incompleteReasons.every(reason => INCOMPLETE_REASONS.includes(reason as IncompleteReason))), "incomplete_reasons_shape"); const validProcess = (p: unknown) => { const i = record(p); return Number.isSafeInteger(i.pid) && (i.pid as number) >= 2 && Number.isSafeInteger(i.ppid) && (i.ppid as number) >= 0 && typeof i.startTicks === "string" && /^\d+$/u.test(i.startTicks) && typeof i.bootId === "string" && /^[a-f0-9-]{36}$/u.test(i.bootId); @@ -481,6 +490,7 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption } let piFaultAttempted = false; let closed = false, published = false, finished: RemoteNativeSnapshot | undefined; + let retiredTerminal: RemoteNativeSnapshot | undefined; const retainedFiles = new Map(); return { binding: binding!, remoteCwd: binding!.remoteCwd, actionFile, outsideTarget: options.crossRoot ? `${root}/cross-root-target` : null, baseline, @@ -526,6 +536,12 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption if ("error" in receipt) throw receipt.error; fail(receipt.receivedAtMs <= receiptDeadlineAt, "receipt_deadline"); const result = readSnapshot(receipt.value, binding!, names, actionFile, options.runnerdSha256); + // The observer only seals its receipt after captured descendants retire. + // A lost filesystem watch still fails qualification, but must not require + // a second RPC to an already deleted lease just to close that observer. + const knownFilesystemGap = result.incompleteReasons !== undefined && result.incompleteReasons.length > 0 + && result.incompleteReasons.every(reason => FILESYSTEM_INCOMPLETE_REASONS.includes(reason)); + if (result.processes.captured && result.processes.live.length === 0 && (result.complete || knownFilesystemGap)) retiredTerminal = result; if (!(published && result.setup.published && result.complete && result.watcher.complete && result.processes.captured && result.processes.live.length === 0)) { throw new IncompleteRemoteTerminalEvidenceError(result); } @@ -546,7 +562,7 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption closed = true; // A finalized receipt survives ordinary public lease deletion. If the // lease still exists, clean our opaque observer; never discover by name. - if (!finished) { stopReceipt(); await rpc({ op: "close" }); } + if (!retiredTerminal) { stopReceipt(); await rpc({ op: "close" }); } }, }; }