From 926104e4d65613f32a554a26a1bfb20e2099e0cc Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 5 Oct 2026 17:13:42 -0500 Subject: [PATCH] Order Pi memory verification and cross-root denial before completion --- tests/runner-e2e/FIXTURES.md | 6 +++-- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/pi-controls-evidence.test.ts | 4 ++-- tests/runner-e2e/pi-native-cases.ts | 22 +++++++++++++------ tests/runner-e2e/pi-native-flow.ts | 4 ++-- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 8c1f5ff9551..347d2931a31 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -396,8 +396,10 @@ form as well as a copied runner. The agent-memory fixture requires the nonce's UTF-8 bytes followed by exactly one line-feed byte (`0x0A`). Its prompt states that byte contract in plain text: the browser editor can normalize backslash escapes in a quoted JSON example. -One native write and one complete native read bound the first task; an incorrect -result must be reported without a repeated rewrite loop. Native readback and the +The prompt orders one memory write, one complete native read, a separate expected +cross-root write denial, and then completion. Native paths use the exact current +absolute agent directory; shell-variable expansion is not assumed. An incorrect +memory result must be reported without a repeated rewrite loop. Native readback and the managed-file API must retain the exact bytes across a new task and controller restart. The byte graders remain unchanged. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 020a76673b1..0ce64140814 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1114,7 +1114,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"), environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 10, excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial", "pi-native.runner-acpx-pi.local.native-pending-provider-death"], - definitionMetadata: { version: 9, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "plain-byte-contract-bounded-native-write-read", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" }, + definitionMetadata: { version: 10, qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-memory-write-read-cross-root-denial", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request" }, }, { id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true, diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index c058bc765d8..02992941663 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -189,7 +189,7 @@ describe("Pi controls catalog admission", () => { // Extended v3 states the strict file oracle's task-wide Bash limit. const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile; expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter"); - expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 9, profileVersion: 14, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "plain-byte-contract-bounded-native-write-read", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); + expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 10, profileVersion: 14, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-memory-write-read-cross-root-denial", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" }); expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 8, profileVersion: 14, remoteProcessIdentity: "observer-pid-startTicks-bootId", controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" }); @@ -200,7 +200,7 @@ describe("Pi controls catalog admission", () => { } const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)])); expect(hashes).toEqual({ - "pi-native": "7e5ba4f0856614757b8bac3f66759bc90b3439038d9eea67fdcf7c997462a7a2", + "pi-native": "cb4845932393a7bc8ddd2a92f765d85c756bba7b9c295490047869e1ab71cb7f", "native-active-stop": "99682b2b106d816a011834fae5a944ed7729958893709d5b83a19b6f595e7e4d", "rich-acp-warm-continuity": "036c0faebc2f6eee5cd22ea38887c9c22650a83561fd08ee83473a565b11bb00", "extended-harnesses": "121f4cf75267bcdb003e5ed59e165755b2ac0c3d31c82dc217a62c8abe3d1b4d", diff --git a/tests/runner-e2e/pi-native-cases.ts b/tests/runner-e2e/pi-native-cases.ts index a517dd17fef..0758ab90d18 100644 --- a/tests/runner-e2e/pi-native-cases.ts +++ b/tests/runner-e2e/pi-native-cases.ts @@ -22,6 +22,20 @@ export function piNativeFinish(marker: string): string { return `After verifying the requested outcome, call paperclip_finish once with reportedWorkDisposition done, summary ${marker}, the current completion contract revision, satisfied objective criterion, no remaining work, evidence [], and verification []. Wait for acceptance, then reply exactly ${marker}. Do not create unrelated work.`; } +export function piNativeMemoryPrompt(nonce: string, outsidePath?: string): string { + return [ + "Follow these steps in order. Use native file tools; do not use bash or the instructions API.", + "1. Copy the exact canonical absolute AGENT_HOME path from this turn's runtime instructions. Native file paths do not expand $AGENT_HOME. Do not use a remembered path or reconstruct any directory name.", + `2. Use native write once to create ${PI_NATIVE_MEMORY_PATH} under that absolute directory. Set content to the ${Buffer.byteLength(nonce, "utf8")} UTF-8 bytes of ${nonce}, followed by exactly one line-feed byte (U+000A). This is personal memory, not a task deliverable. Do not edit AGENTS.md or a project file.`, + "3. Use native read once, without offset or limit, to verify that complete file. Do not trim or repeat the memory write. If either memory operation fails, report the failure and end your turn without claiming success.", + ...(outsidePath === undefined ? [] : [ + `4. After successful memory verification, make a separate native write attempt exactly once to ${JSON.stringify(outsidePath)} with content forbidden. This unassigned root must be denied. That denial is the expected result of this step, not a memory-operation failure. Do not retry or work around it.`, + ]), + "Paperclip collects the memory file after the provider stops. Do not claim it is already saved to managed storage.", + piNativeFinish("PI-NATIVE-MEMORY-STAGED"), + ].join("\n"); +} + export function piNativePrompt(id: string, nonce: string): string { if (id === "native-pending-provider-death") return [ 'Call paperclip_native_question exactly once with {method:"input",title:"Pi native provider death",placeholder:"Answer"}. Wait for the real answer; never guess it or substitute semantic request_human_input.', @@ -42,13 +56,7 @@ export function piNativePrompt(id: string, nonce: string): string { "Keep the actual four tool result objects in order. Using the native write tool, write their JSON array to pi-native-answers.json in the task workspace. This is an internal verification file, not a deliverable. Do not infer, change, or prefill any answer.", piNativeFinish(`PI-NATIVE-${id}-${nonce}`), ].join("\n"); - if (id === "agent-files-fresh-run") return [ - "Find the exact registered AGENT_HOME working-copy path in your runtime instructions. Use native write/read file tools for this task, not bash or the instructions API.", - `Use native write exactly once to create ${PI_NATIVE_MEMORY_PATH} inside that AGENT_HOME. Its content must be the ${Buffer.byteLength(nonce, "utf8")} UTF-8 bytes of ${nonce}, followed by exactly one line-feed byte (U+000A). Set the write tool's content argument to that complete string, including the final line feed. This is personal memory, not a task deliverable. Do not edit AGENTS.md or a project file instead.`, - "Use native read exactly once, without offset or limit, to verify the complete file. Do not trim its content, repeat the write, or use bash to inspect or repair it. If the native result is incorrect, report the failure and end your turn without claiming success.", - "Do not claim that the file is already saved to managed storage; Paperclip collects it after the provider stops.", - piNativeFinish("PI-NATIVE-MEMORY-STAGED"), - ].join("\n"); + if (id === "agent-files-fresh-run") return piNativeMemoryPrompt(nonce); if (id === "human-permission-denial") return [ "Attempt native write exactly once with relative path pi-human-denied.txt and content forbidden. Wait for the actual browser permission decision. Do not retry, use bash or an API, write another path, or work around denial.", "After the native write returns its denial, finish this negative test. Native denial is the expected result; never claim that the file was written.", diff --git a/tests/runner-e2e/pi-native-flow.ts b/tests/runner-e2e/pi-native-flow.ts index dc319d620bd..dd7171935ec 100644 --- a/tests/runner-e2e/pi-native-flow.ts +++ b/tests/runner-e2e/pi-native-flow.ts @@ -8,7 +8,7 @@ import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { collectRunEvents } from "./run-observations.js"; import { createTaskThroughUi } from "./user-actions.js"; -import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, piNativeFinish } from "./pi-native-cases.js"; +import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js"; import type { LiveFixtureValues } from "./live-fixtures.js"; import type { MatrixExecution } from "./types.js"; @@ -153,7 +153,7 @@ export async function runPiNativeFlow(input: { } else if (execution.task.id === "agent-files-fresh-run") { const retained = randomBytes(16).toString("hex"); const outside = resolve(input.workspacePath, "..", `pi-unassigned-${nonce}.txt`); if (!remote) check("unassigned-initially-absent", await absent(outside), "The isolated cross-root marker did not already exist"); - const prompt = (outsidePath: string) => `${execution.task.buildPrompt(retained)}\nBefore finishing, attempt native write exactly once to ${JSON.stringify(outsidePath)} with content forbidden. This intentionally unassigned root must be denied. Never use bash or another operation to work around its failure.`; + const prompt = (outsidePath: string) => piNativeMemoryPrompt(retained, outsidePath); await create(execution.task.buildTitle(nonce), remote ? fixture => { if (!fixture.outsideTarget) throw new Error("Remote cross-root target is absent"); return prompt(fixture.outsideTarget); } : prompt(outside), { crossRoot: { initialText: `unchanged-${randomBytes(16).toString("hex")}` } }); await settle(1); const personal = await api.get(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`);