diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index b446525c15b..859bfa1fc03 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -15,6 +15,60 @@ failures remain retained. The Mac file-edit correction also passes. ### Latest boundary fixes — 2026-10-05 +Current shipping artifacts are bound to `f5024863e9ed014e2661e6d7f23fc0e6b32e0911` +and profile 15. The normal public ARM graph verifies 722 packages and 79,373 +files with zero mismatches. Full build and recursive typecheck pass. Normal +Linux public install, companion import, native admission and browser startup +pass against immutable image +`ghcr.io/paperclipai/paperclip-daytona-runner@sha256:20c7fffddeee4298830ead1ea3f0b70819f61e549279588b605440d1155f7260`. +Its controller passes 342 fixture checks, 21 native fault checks and collection +of all 26 Product cells with zero model calls. An installed-UI intercepted POST +proves one fenced prompt with valid JSON, all 33 content bytes and no duplicate. + +The canonical-root Mac attempt uses shipping runtime `f5024863e` and harness +`df748c997`; the recorded diff contains only fixture infrastructure and the plan. +The provider now completes its first turn and writes to the correct native +agent directory. Its native read returns the 32-character nonce without LF. +The exact managed-byte assertion fails; canonical cleanup passes. The public +description independently contains one valid JSON content value of 33 bytes. +The original managed-file response was not retained before that assertion, so +its exact returned bytes cannot be claimed from the saved evidence. A free +check of the frozen Pi JSON parser, argument validator, native write and native +read preserves all 33 bytes. There is no proven byte-trimming product defect. +Do not weaken the byte grade, add a newline in the product, or repeat this paid +attempt without a concrete correction. + +The next Linux restart attempt retains the same native request, turn, run, +session and producer across controller restart and accepts the exact browser +answer. The original run succeeds. Its independent observer returns incomplete +terminal evidence; canonical cleanup also fails after public lease release. +Both owned child and controller are separately verified absent, and the active +campaign claim is closed. The canonical result stays failed. The missing +incomplete receipt prevents a precise observer-cause attribution; do not claim +that public run success proves the final independent file or cleanup grades. +The initial controller preparation also fails before provider calls because +consumer and plugin shared-package archives collide by basename. Separating +their staging namespaces fixes preparation with every archive digest unchanged. + +The fixture now records the managed-file response before grading and retains a +closed, validated summary of an incomplete terminal receipt before rethrowing. +It omits raw RPC content and file bodies. A missing-LF regression fails before +the capture-order correction and still fails the byte grade afterward while +retaining the response. These are evidence-only changes; no runtime input, +model, timeout, assertion or original result changes. They do not justify a +paid rerun solely to collect diagnostics. +All 207 affected fixture tests pass with filesystem-watch access, and the +harness typecheck passes. The initial sandbox run retains three failed watch +observations; it is not reported as a passing run. + +At `df748c997`, 53 CI checks pass with two skips. This is source-bound CI, not +proof for a later evidence-only commit. Current-source live qualification is +still incomplete: neither failed attempt qualifies its case, and historical +profile-14 or earlier profile-15 passes cannot certify this runtime. Review and +the complete workspace unit command remain held. Last observed dedicated-key +usage is $0.318304276 against its $5 lifetime cap; the immediate restart delta +of $0.000785504 is provisional. No merge or release is authorized. + The next explicit Mac attempt at `f5024863e` confirms that the description is no longer duplicated, but still times out with cleanup passing. Its runtime context advertises `/tmp/.../live`, while the native Pi grant binds the physical diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 0cd186165c1..5f318966d40 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1483,6 +1483,11 @@ the durable browser form bridge. The existing five extended-harness journeys continue to prove semantic Paperclip questions and planning separately. Personal file persistence uses two fresh task runs and independent byte checks; restrictive denial uses one run and requires both a failed tool receipt and no file effect. +The first managed-file API response is saved before its byte assertion, including +when the final LF is missing. An incomplete remote terminal receipt retains only +validated completion flags, watcher counts, process counts and target hashes. +Raw RPC fields and file bodies are omitted. These diagnostic records do not +change a failed byte or retirement grade and do not justify an unchanged retry. `human-permission-denial` additionally requires the exact browser Decline response, delivered native denial and independent file/process observation through retirement. The restrictive Daytona fixtures approve only the exact operator-published diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index dd7171935ec..c19c76038c8 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -12,7 +12,7 @@ import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; -import type { RemoteNativeFixture, RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { remoteNativeIncompleteTerminalEvidence, type RemoteNativeFixture, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; import { approvePiBootstrapRead, withoutApprovedPiBootstrapRequests, type PiBootstrapApproval } from "./pi-bootstrap-permission.js"; import { runPiPendingProviderDeath, PI_DEATH_MARKER } from "./pi-native-provider-death-flow.js"; import { runPiPendingControllerRestart } from "./pi-native-restart-flow.js"; @@ -42,9 +42,18 @@ export async function runPiNativeFlow(input: { let bootstrapApproval: PiBootstrapApproval | undefined; let currentRemote: RemoteNativeFixture | undefined, currentBaseline: RemoteNativeSnapshot | undefined; let remoteSequence = 0; + const retainedIncompleteTerminals = new Set(); + async function retainIncompleteTerminal(fixture: RemoteNativeFixture, ordinal: number, error: unknown) { + const snapshot = remoteNativeIncompleteTerminalEvidence(error); + if (!snapshot || retainedIncompleteTerminals.has(ordinal)) return; + await input.evidence(`pi-remote-${ordinal}-incomplete-terminal.json`, { binding: fixture.binding, snapshot, passed: false }); + retainedIncompleteTerminals.add(ordinal); + } async function finishRemote(label: string) { if (!currentRemote) throw new Error("Missing exact owned remote fixture"); - const snapshot = await currentRemote.finish(); + let snapshot: RemoteNativeSnapshot; + try { snapshot = await currentRemote.finish(); } + catch (error) { await retainIncompleteTerminal(currentRemote, remoteSequence, error); throw error; } await input.evidence(`pi-remote-${remoteSequence}-${label}.json`, { binding: currentRemote.binding, baseline: currentBaseline, snapshot }); if (!hasPiRemoteRetirement(snapshot)) throw new Error("Pi remote provider retirement is unproven; sandbox deletion is not proof"); return snapshot; @@ -84,6 +93,7 @@ export async function runPiNativeFlow(input: { const fixture = currentRemote, ordinal = remoteSequence; input.registerCleanupAssertion!(async () => { try { const snapshot = await fixture.finish(); const passed = hasPiRemoteRetirement(snapshot); await input.evidence(`pi-remote-${ordinal}-retirement.json`, { binding: fixture.binding, snapshot, passed }); if (!passed) throw new Error("Pi remote retirement proof incomplete"); return [{ id: `remote-retirement-${ordinal}`, passed, detail: "Independent remote observer sealed exact run retirement before public lease deletion" }]; } + catch (error) { await retainIncompleteTerminal(fixture, ordinal, error); throw error; } finally { await fixture.close(); } }); } @@ -158,11 +168,11 @@ export async function runPiNativeFlow(input: { await settle(1); const personal = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`); const firstEvents = await events(runs[0]!.id); + await input.evidence("pi-agent-files-first-save.json", { personal, run: runs[0], events: firstEvents }); check("registered-file-saved", personal.content === `${retained}\n`, "Public managed-file API contains the exact native-write bytes"); check("stopped-save-receipt", firstEvents.some(row => row.eventType === "instruction_save" && row.payload?.state === "saved"), "Provider stop produced a durable file-save receipt"); const crossRootIntact = remote ? hasUnchangedPiRemoteTarget(currentBaseline, await finishRemote("cross-root-final"), "@cross-root") : await absent(outside); check("cross-root-denied", crossRootIntact && hasPiCrossRootDenial(firstEvents), "A single native write recorded the exact cross-root denial reason and the isolated target remains unchanged"); - await input.evidence("pi-agent-files-first-save.json", { personal, run: runs[0], events: firstEvents }); await input.restart(); await create(`Pi read persisted memory ${nonce}`, [ `Use native read to read ${PI_NATIVE_MEMORY_PATH} under the fresh registered AGENT_HOME. Read its exact current bytes; do not infer them from another task, conversation, or history.`, diff --git a/tests/runner-e2e/pi-native-remote-flow.test.ts b/tests/runner-e2e/pi-native-remote-flow.test.ts index 577089fe486..ead29865ed8 100644 --- a/tests/runner-e2e/pi-native-remote-flow.test.ts +++ b/tests/runner-e2e/pi-native-remote-flow.test.ts @@ -9,7 +9,7 @@ const browser = vi.hoisted(() => ({ create: (_value: any) => {} })); vi.mock("./user-actions.js", () => ({ createTaskThroughUi: async (value: unknown) => browser.create(value) })); vi.mock("@playwright/test", () => ({ expect: (value: unknown, message?: string) => ({ toBe: (expected: unknown) => expect(value, message).toBe(expected), toBeVisible: async () => {} }) })); -it("rebinds both remote runs, saves managed bytes, and reads sealed sandbox bytes after restart", async () => { +it.each([false, true])("retains managed bytes before grading and rebinds successful runs (missing LF=%s)", async missingLF => { const root = await mkdtemp(join(tmpdir(), "pi-remote-memory-")); const task = piNativeTasks.find(row => row.id === "agent-files-fresh-run")!; const issues: any[] = [], runs: any[] = [], cleanup: Array<() => Promise> = []; const captures: any[] = [], evidence = new Map(); @@ -46,13 +46,22 @@ it("rebinds both remote runs, saves managed bytes, and reads sealed sandbox byte personal = JSON.parse(content!).content; expect(personal).toMatch(/^[a-f0-9]{32}\n$/); expect(Buffer.byteLength(personal, "utf8")).toBe(33); + if (missingLF) personal = personal.slice(0, -1); } else { expect(action).not.toContain(personal.trim()); expect(input.targets).toEqual(["pi-agent-memory-proof.txt"]); } issues.at(-1).status = "done"; runs.at(-1).status = "succeeded"; captures.push(fixture.binding); return fixture; } }; try { await writeFile(join(root, "pi-agent-memory-proof.txt"), "WRONG HOST COPYBACK"); - const result = await runPiNativeFlow({ page: { goto: async () => {}, reload: async () => {}, getByTestId: () => ({ getByRole: () => ({}) }) }, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "daytona-env" } }, execution: { task, environment: { id: "daytona" }, profile: { qualificationCandidate: "pi" } }, nonce: "fixture", workspacePath: root, deadlineAt: Date.now() + 2000, + const call = runPiNativeFlow({ page: { goto: async () => {}, reload: async () => {}, getByTestId: () => ({ getByRole: () => ({}) }) }, api, fixtures: { company: { id: "company", issuePrefix: "PI" }, agent: { id: "agent", name: "Pi" }, environment: { id: "daytona-env" } }, execution: { task, environment: { id: "daytona" }, profile: { qualificationCandidate: "pi" } }, nonce: "fixture", workspacePath: root, deadlineAt: Date.now() + 2000, restart: async () => { expect(evidence.has("pi-remote-1-cross-root-final.json")).toBe(true); restarted = true; }, observe: () => {}, capture: async () => {}, evidence: async (name: string, value: unknown) => { evidence.set(name, value); }, remoteBootstrap, registerCleanupAssertion: (fn: () => Promise) => cleanup.push(fn) } as any); + if (missingLF) { + await expect(call).rejects.toThrow("Public managed-file API contains the exact native-write bytes"); + expect(evidence.get("pi-agent-files-first-save.json").personal.content).toBe(personal); + expect(Buffer.byteLength(personal)).toBe(32); + expect(restarted).toBe(false); expect(runs).toHaveLength(1); + return; + } + const result = await call; expect(result.checks.every(check => check.passed)).toBe(true); expect(captures).toEqual([{ runId: "run-1" }, { runId: "run-2" }]); expect(readAfterFinish).toBe(true); expect(cleanup).toHaveLength(2); for (const fn of cleanup) await fn(); } finally { await rm(root, { recursive: true, force: true }); } }); diff --git a/tests/runner-e2e/remote-native-fixtures.test.ts b/tests/runner-e2e/remote-native-fixtures.test.ts index 8654f50fa5c..35dd55b33f6 100644 --- a/tests/runner-e2e/remote-native-fixtures.test.ts +++ b/tests/runner-e2e/remote-native-fixtures.test.ts @@ -7,7 +7,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { Script } from "node:vm"; import { describe, expect, it, vi } from "vitest"; -import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, validatePiProviderDeathReceipt, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; +import { REMOTE_FIXTURE_MIN_SETUP_BUDGET_MS, bindRemoteNativeFixture, validatePiProviderDeathReceipt, createRemoteTargetWatch, isRemoteRunRoot, parseRemoteProcStat, remoteNativeFixtureDiagnostics, remoteNativeIncompleteTerminalEvidence, type RemoteNativeFixtureOptions, type RemoteNativeSnapshot } from "./remote-native-fixtures.js"; import { PI_DISTRIBUTION_CLOSURE_SHA256 } from "../../packages/paperclip-runner/src/drivers/acpx/pi-closure-pins.js"; import { createRemoteNativeBootstrap } from "./remote-native-bootstrap.js"; @@ -233,6 +233,22 @@ describe("remote native lease admission", () => { if (type === "bad-file") end.targets["result.txt"] = { ...end.targets["result.txt"], absent: false, sha256: hash("missing") }; h.resolveTerminal(end); await expect(f.finish()).rejects.toThrow(); }); + it("retains a validated incomplete terminal receipt without raw file or RPC content", async () => { + const h = harness(), f = await bindRemoteNativeFixture(h.options); + await f.publishAction("action.txt", "task"); + const end = { ...structuredClone(h.current), complete: false, watcher: { ...h.current.watcher, complete: false }, + processes: { ...h.current.processes, live: [] }, files: { "private.txt": "PRIVATE RPC CONTENT" }, untrustedDump: "PRIVATE RPC CONTENT" }; + h.resolveTerminal(end); + const error = await f.finish().catch(error => error); + expect(error.message).toBe("remote_native_fixture:terminal_evidence_incomplete"); + const retained = remoteNativeIncompleteTerminalEvidence(error)!; + expect(retained.complete).toBe(false); expect(retained.watcher.complete).toBe(false); + expect(retained.processes.liveCount).toBe(0); + expect(JSON.stringify(retained)).not.toContain("PRIVATE RPC CONTENT"); + retained.watcher.complete = true; + expect(remoteNativeIncompleteTerminalEvidence(error)!.watcher.complete).toBe(false); + expect(remoteNativeIncompleteTerminalEvidence(new Error("PRIVATE RPC CONTENT"))).toBeUndefined(); + }); it("keeps a fixture-owned cross-root sentinel distinct from workspace targets", async () => { const h = harness(); h.options.crossRoot = { initialText: "outside sentinel" }; h.current.targets["@cross-root"] = { absent: false, sha256: hash("outside sentinel"), parent: { dev: "1", ino: "outside" }, mutationCount: 0, complete: true }; diff --git a/tests/runner-e2e/remote-native-fixtures.ts b/tests/runner-e2e/remote-native-fixtures.ts index b0e5f3db617..ae8eba2c5fa 100644 --- a/tests/runner-e2e/remote-native-fixtures.ts +++ b/tests/runner-e2e/remote-native-fixtures.ts @@ -70,6 +70,25 @@ export interface RemoteNativeSnapshot { setup: { path: string; sha256: string | null; published: boolean }; attached: { connections: number; failure: string | null; commandExit: { code: number; observedAtMs: number; observedMonotonicNs: string } | null; markerWrittenAtMs: number | null; markerWrittenMonotonicNs: string | null; clientExitedAtMs: number | null; clientExitedMonotonicNs: string | null } | null; } +class IncompleteRemoteTerminalEvidenceError extends Error { + constructor(readonly snapshot: RemoteNativeSnapshot) { + super("remote_native_fixture:terminal_evidence_incomplete"); + } +} +/** Retain only a shape-validated public receipt, without raw RPC text or files. */ +export function remoteNativeIncompleteTerminalEvidence(error: unknown) { + if (!(error instanceof IncompleteRemoteTerminalEvidenceError)) return undefined; + const row = error.snapshot, root = row.processes.root; + return { + observedAtMs: row.observedAtMs, receivedAtMs: row.receivedAtMs, complete: row.complete, + setupPublished: row.setup.published, + watcher: { complete: row.watcher.complete, targetMutationCount: row.watcher.targetMutationCount, workspaceMutationCount: row.watcher.workspaceMutationCount }, + processes: { captured: row.processes.captured, liveCount: row.processes.live.length, journalCount: row.processes.journal.length, + root: root ? { pid: root.pid, ppid: root.ppid, startTicks: root.startTicks, bootId: root.bootId } : null }, + targets: Object.fromEntries(Object.entries(row.targets).map(([name, target]) => [name, + { absent: target.absent, sha256: target.sha256, mutationCount: target.mutationCount, complete: target.complete }])), + }; +} /** Structural subset of pinned SDK0.203.0; caller supplies its authenticated * client. No create/list/delete or arbitrary remote command API is exposed. */ export interface RemoteFixtureDaytona { @@ -503,7 +522,9 @@ export async function bindRemoteNativeFixture(options: RemoteNativeFixtureOption if ("error" in receipt) throw receipt.error; fail(receipt.receivedAtMs <= receiptDeadlineAt, "receipt_deadline"); const result = readSnapshot(receipt.value, binding!, names, actionFile, options.runnerdSha256); - fail(published && result.setup.published && result.complete && result.watcher.complete && result.processes.captured && result.processes.live.length === 0, "terminal_evidence_incomplete"); + if (!(published && result.setup.published && result.complete && result.watcher.complete && result.processes.captured && result.processes.live.length === 0)) { + throw new IncompleteRemoteTerminalEvidenceError(result); + } const files = record(record(receipt.value).files); for (const name of names) { const target = result.targets[name]!;