diff --git a/doc/plans/2026-10-02-pi-production-readiness.md b/doc/plans/2026-10-02-pi-production-readiness.md index c081a84c5cb..1a63ccd8bba 100644 --- a/doc/plans/2026-10-02-pi-production-readiness.md +++ b/doc/plans/2026-10-02-pi-production-readiness.md @@ -226,7 +226,36 @@ recorded coverage. Its process scan omitted executables in Pi snapshot roots. A supplemental scan at 22:53:48 UTC covers those paths too and finds no owned runtime processes or temporary roots; launcher 7869 and observed sidecar 8466 are absent. This proves current quiescence and does not retroactively broaden -the original receipt. No subsequent paid case or automatic retry ran. +the original receipt. No subsequent paid case or automatic retry ran in that +qualification sequence. + +After the user's request to get qualification working, one separately named +input-boundary diagnostic ran on a fresh cloud host against those same packages, +harness, model, low thinking and graders. The original host had been automatically +deleted. The restored consumer's 391 packages and 80,151 files match the frozen +runtime inventory; four declared native-build configuration files regenerated. +Registry drift was corrected using the original dependency versions and retained +integrities before the diagnostic. No shipping inputs changed. + +That diagnostic retained actual Pi session tool inputs: all eight writes supplied +32-byte content without LF. Observed memory-file hashes matched the supplied +content hashes. This rules out newline removal by native write/storage for those +inspected calls; upstream raw provider SSE was not captured. The unchanged +120-second deadline still failed, and its canonical `transient_infrastructure` +grade remains intact. Cleanup passed, with no owned runtime processes or temporary +roots in the independent scan. All 42 canonical files, totaling 130,691,700 bytes, +were retained and hash-verified locally. Its provisional key-usage delta is +$0.004511326; cumulative key usage is $0.585187764 against the unchanged $5 lifetime +cap inside the approved $100 campaign ceiling. There were no automatic retries. + +On 2026-10-08 the user approved switching qualification to +`openrouter/anthropic/claude-sonnet-4.6` and requested the memory-test repair. +Pi native definition 18 uses one ordinary JSON write and exact complete read, +with all byte, persistence, permission and restart assertions retained. Native +low thinking and zero automatic retries remain required. Sonnet coverage starts +at 0/26 Product cells and 0/7 Runner cells; previous DeepSeek measurements remain +historical. Production model selection remains caller-controlled. The frozen +shipping source, packages and cloud image do not change for these test inputs. The owned Daytona host is now stopped normally after a fresh broad process/root scan and renewed local verification of all six canonical attempts: 120 files, @@ -242,7 +271,7 @@ variant also verifies the exact LF-bearing text in the subsequent provider request. Network connections outside the owned loopback fixture are denied before connect, owned children close and shipping sources remain unchanged. These checks identify no supported production correction and do not count as -live qualification passes. Keep the accepted DeepSeek model and native low +live qualification passes. The approved Sonnet qualification keeps native low thinking. Do not substitute a passing surrogate or weaken exact-byte grading. Native hosted ARM installation on the same shipping source passes normal npm diff --git a/tests/acpx-qualification-models.ts b/tests/acpx-qualification-models.ts index 44452c47e85..ef4a978009c 100644 --- a/tests/acpx-qualification-models.ts +++ b/tests/acpx-qualification-models.ts @@ -3,5 +3,5 @@ export const ACPX_QUALIFICATION_MODELS = { claude: "claude-sonnet-5", codex: "gpt-5.6-sol", grok: "grok-4.7", - pi: "openrouter/deepseek/deepseek-v4-flash-0731", + pi: "openrouter/anthropic/claude-sonnet-4.6", } as const; diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 4b22b09f51b..26205a70b05 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -31,16 +31,16 @@ profile, model qualification, environment, task, or ranking-snapshot change must change that fingerprint automatically so the dashboard can annotate the boundary instead of silently joining unlike totals. -Pi native definition 17 keeps the memory outcome at exactly the nonce plus one -final LF. Its authoritative native-write JSON spells that byte as `\u000a` -instead of `\n`; both decode to identical content. The installed Pi tools -preserve the 33-byte Unicode-escaped payload and the 32-byte negative control -exactly. The strict managed-file and fresh-task byte comparisons, single native -write/read, cross-root denial, model, low thinking, permissions and deadlines are -unchanged. Retain both previous missing-LF failures and their original catalog -fingerprints. This encoding correction permits one recorded changed memory -attempt, with zero automatic retries. Free byte preservation is diagnostic -evidence, not a live qualification pass. +Pi native definition 18 supplies one ordinary JSON write with the nonce followed +by exactly one LF, then asks for a complete native read. The independently +checked outcome remains 33 bytes, saved through the public managed-file API +and copied into a fresh task after controller restart. Missing LF, extra LF, +literal escapes, CRLF and stale values all fail. The cross-root denial, protected +parent seed, permissions, deadlines and zero automatic retries remain required. +The user approved Sonnet 4.6 through OpenRouter as the explicit Pi qualification +model; production model selection stays caller-controlled. Earlier DeepSeek +attempts and their fingerprints remain historical evidence and do not qualify +this new model/fixture combination. The explicit [stock-harness suite](STOCK-HARNESS.md) wraps existing profiles with `productionDefaultHireProfile`: omit only `instructionsBundle` so the public @@ -450,7 +450,7 @@ final line feed; both write and read retain the exact supplied bytes. The prompt orders one memory write, one complete native read, a separate expected cross-root write denial, and then completion. Native paths use the exact current absolute agent directory; shell-variable expansion is not assumed. An incorrect -memory result must be reported without a repeated rewrite loop. Native readback and the +memory result must be reported without claiming success. Native readback and the managed-file API must retain the exact bytes across a new task and controller restart. The byte graders remain unchanged. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 2d8b4535896..bd1b6260eda 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1132,7 +1132,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 10, excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial", "pi-native.runner-acpx-pi.local.native-pending-provider-death"], - definitionMetadata: { version: 17, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-write-arguments-with-json-unicode-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" }, + definitionMetadata: { version: 18, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-exact-native-read", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" }, }, { id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index fb178a4441f..ef5561ee82c 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -189,7 +189,7 @@ describe("Pi controls catalog admission", () => { // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 17, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 18, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-write-arguments-with-json-unicode-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 18, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 18, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-exact-native-read", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 9, taskCreation: "explicit-title-and-creation-response-id", profileVersion: 18, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); @@ -200,10 +200,10 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "e60021e6959cb2ed9b50b3d5b734b6ad5a78bafc7aca1ea1fca456fb3191f318", + "pi-native": "b31f8ab684568b56b1a8386fdac8793bc7e06a92372355d059642b3abb704b1a", "native-active-stop": "2d4fdeeb75bd531b309bd1b35cfa5680ceefdeee6b7155b068dd011ce9901980", - "rich-acp-warm-continuity": "45d0f6a824b061464fa18cb24b1414502b52959d5f47646cfc5079cbb8f2d98b", - "extended-harnesses": "77cc19a12c07411bca5f29f48162467bb291d0f47fd567cc1993151cf5c6c989", + "rich-acp-warm-continuity": "331b88d9538a132670789fb7864df7df5f4bf294ae19b62f53d8a02f901774f0", + "extended-harnesses": "c66ce8165c5a0aac7b03851307bb76637eecfed0c78a50606fb3f6e7dbcefcb4", }); expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(22); }); diff --git a/tests/runner-e2e/pi-native-cases.test.ts b/tests/runner-e2e/pi-native-cases.test.ts index c3783c9f2ed..615b859c4c7 100644 --- a/tests/runner-e2e/pi-native-cases.test.ts +++ b/tests/runner-e2e/pi-native-cases.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, piNativeMemoryPrompt, piNativeTasks } from "./pi-native-cases.js"; +import { gradePiNativeAnswers, gradePiNativeMemory, hasFailedPiWrite, hasPiCrossRootDenial, piNativeMemoryPrompt, piNativeTasks } from "./pi-native-cases.js"; import { runnerMatrix, runnerSuites } from "./catalog.js"; import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; @@ -13,16 +13,19 @@ describe("Pi native Product qualification", () => { const args = JSON.parse(encoded); expect(Object.keys(args)).toEqual(["path", "content"]); expect(args.path).toBe("/memory/pi-native.txt"); - expect(encoded).toContain('\\u000a"}'); - expect(encoded).not.toContain('\\\\u000a'); - const content = args.content; - expect(content).toBe(`${nonce}\n`); - expect(Buffer.byteLength(content)).toBe(33); - expect(Buffer.from(content).at(-1)).toBe(10); - expect(prompt).toContain("Do not trim or repeat the memory write"); - expect(prompt).toContain("Replace only the prefix in path"); - expect(prompt).toContain("Copy the content string directly into the native write arguments"); - expect(prompt).toContain("including its final line feed"); + expect(args.content).toBe(`${nonce}\n`); + expect(Buffer.byteLength(args.content)).toBe(33); + expect(Buffer.from(args.content).at(-1)).toBe(10); + expect(prompt).toContain("Use native read once, without offset or limit"); + expect(prompt).toContain("Do not retry or work around it"); + }); + + it("requires the current nonce and exactly one LF at every memory readback", () => { + const nonce = "0123456789abcdef0123456789abcdef"; + expect(gradePiNativeMemory(`${nonce}\n`, nonce)).toBe(true); + for (const wrong of [undefined, null, {}, nonce, `${nonce}\n\n`, `${nonce}\r\n`, `${nonce}\\n`, `${nonce}\\u000a`, `${"f".repeat(32)}\n`]) { + expect(gradePiNativeMemory(wrong, nonce)).toBe(false); + } }); it("selects five local and five remote Pi cases without changing the basic extended matrix", () => { const suite = runnerSuites.find(row => row.id === "pi-native")!; diff --git a/tests/runner-e2e/pi-native-cases.ts b/tests/runner-e2e/pi-native-cases.ts index fe7478826e0..6e1a7a006d6 100644 --- a/tests/runner-e2e/pi-native-cases.ts +++ b/tests/runner-e2e/pi-native-cases.ts @@ -24,24 +24,23 @@ export function piNativeFinish(marker: string): string { return `After verifying the requested outcome, call paperclip_finish once with reportedWorkDisposition done, summary ${marker}, the current completion contract revision, satisfied objective criterion with evidenceRefs [], no remaining work, evidence [], and verification []. Wait for acceptance, then reply exactly ${marker}. Do not create unrelated work.`; } +/** Exact saved bytes; neither missing evidence nor newline normalization passes. */ +export function gradePiNativeMemory(actual: unknown, nonce: string): boolean { + return typeof actual === "string" && actual === `${nonce}\n`; +} + export function piNativeMemoryPrompt(nonce: string, outsidePath?: string): string { - // A Unicode JSON escape makes the required final byte explicit while keeping - // the decoded write content identical to the original LF fixture. - const writeArguments = JSON.stringify({ path: `/${PI_NATIVE_MEMORY_PATH}`, content: `${nonce}\n` }) - .replace(/\\n"}$/, '\\u000a"}'); + const writeArguments = JSON.stringify({ path: `/${PI_NATIVE_MEMORY_PATH}`, content: `${nonce}\n` }); return [ - "Follow these steps in order. Use native file tools; do not use bash or the instructions API.", - "1. Copy the exact canonical absolute AGENT_HOME path from this turn's runtime instructions. Native file paths do not expand $AGENT_HOME. Do not use a remembered path or reconstruct any directory name.", - `2. Call native write exactly once with the JSON arguments below. Replace only the prefix in path with the exact absolute directory copied in step 1. Copy the content string directly into the native write arguments, including its final \\u000a escape. This is the only content specification: the JSON string decodes to ${Buffer.byteLength(nonce, "utf8") + 1} UTF-8 bytes, ending with exactly one line-feed byte (U+000A). The Unicode escape is part of content, not a closing delimiter. Do not reconstruct content from the visible nonce, omit the escape, or double-escape it into literal backslash-and-u characters. This is personal memory, not a task deliverable. Do not edit AGENTS.md or a project file.`, - "Native write stores exactly the content argument; it never adds a newline. Include the final JSON newline escape in the write argument itself. Native read without offset or limit returns exact file text, including any final newline; a read result without that newline means verification failed.", + "Save this personal memory using native write once. Replace with the canonical absolute AGENT_HOME directory from this turn's runtime instructions:", "```json", writeArguments, "```", - "3. Use native read once, without offset or limit, to verify that complete file, including its final line feed. Do not trim or repeat the memory write. If either memory operation fails or the content differs from step 2, report the failure and end your turn without claiming success.", + `The file must contain exactly ${Buffer.byteLength(nonce, "utf8") + 1} UTF-8 bytes: the supplied text followed by one LF byte. The JSON \\n denotes that LF, not literal backslash-and-n characters. Native write does not add it for you.`, + "Use native read once, without offset or limit, to verify the complete file. If writing or verification fails, report the failure without claiming success. Leave other agent files unchanged; use neither bash nor the instructions API.", ...(outsidePath === undefined ? [] : [ - `4. After successful memory verification, attempt native write exactly once to ${JSON.stringify(outsidePath)} with content forbidden. This unassigned root must be denied. That denial is the expected result of this step, not a memory-operation failure. Do not retry or work around it.`, + `Then attempt native write once to ${JSON.stringify(outsidePath)} with content forbidden. This unassigned path must be denied; that denial is expected. Do not retry or work around it.`, ]), - "Paperclip collects the memory file after the provider stops. Do not claim it is already saved to managed storage.", piNativeFinish("PI-NATIVE-MEMORY-STAGED"), ].join("\n"); } diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index c17b9ffc039..da78a701f9d 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -8,7 +8,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; -import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js"; +import { gradePiNativeAnswers, gradePiNativeMemory, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -183,7 +183,7 @@ export async function runPiNativeFlow(input: { const personal = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`); const firstEvents = await events(runs[0]!.id); await input.evidence("pi-agent-files-first-save.json", { personal, run: runs[0], events: firstEvents }); - check("registered-file-saved", personal.content === `${retained}\n`, "Public managed-file API contains the exact native-write bytes"); + check("registered-file-saved", gradePiNativeMemory(personal.content, retained), "Public managed-file API contains the exact native-write bytes"); check("stopped-save-receipt", firstEvents.some(row => row.eventType === "instruction_save" && row.payload?.state === "saved"), "Provider stop produced a durable file-save receipt"); const crossRootIntact = remote ? hasUnchangedPiRemoteTarget(currentBaseline, await finishRemote("cross-root-final"), "@cross-root") : await absent(outside); check("cross-root-denied", crossRootIntact && hasPiCrossRootDenial(firstEvents), "A single native write recorded the exact cross-root denial reason and the isolated target remains unchanged"); @@ -195,7 +195,7 @@ export async function runPiNativeFlow(input: { ].join("\n"), { targets: ["pi-agent-memory-proof.txt"] }); await settle(2); if (remote) await finishRemote("memory-readback-final"); - check("fresh-run-readback", await readWorkspace("pi-agent-memory-proof.txt") === `${retained}\n`, "A new issue after server restart copied the undisclosed saved agent-file bytes"); + check("fresh-run-readback", gradePiNativeMemory(await readWorkspace("pi-agent-memory-proof.txt"), retained), "A new issue after server restart copied the undisclosed saved agent-file bytes"); const current = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`); check("persistent-bytes-unchanged", current.content === personal.content, "Fresh-run readback preserved the saved managed bytes"); const parentSeed = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PARENT_SEED_PATH)}`);