test(pi): qualify Sonnet and simplify exact-byte memory fixture

This commit is contained in:
Dotta committed 2026-10-08 05:10:06 -05:00
1 parent b548b4f4c5
commit 0104942b02
8 files changed
+75 -44

No files matched your search

+11 -11
View File
@@ -31,16 +31,16 @@ profile, model qualification, environment, task, or ranking-snapshot change
must change that fingerprint automatically so the dashboard can annotate the
boundary instead of silently joining unlike totals.
Pi native definition 17 keeps the memory outcome at exactly the nonce plus one
final LF. Its authoritative native-write JSON spells that byte as `\u000a`
instead of `\n`; both decode to identical content. The installed Pi tools
preserve the 33-byte Unicode-escaped payload and the 32-byte negative control
exactly. The strict managed-file and fresh-task byte comparisons, single native
write/read, cross-root denial, model, low thinking, permissions and deadlines are
unchanged. Retain both previous missing-LF failures and their original catalog
fingerprints. This encoding correction permits one recorded changed memory
attempt, with zero automatic retries. Free byte preservation is diagnostic
evidence, not a live qualification pass.
Pi native definition 18 supplies one ordinary JSON write with the nonce followed
by exactly one LF, then asks for a complete native read. The independently
checked outcome remains 33 bytes, saved through the public managed-file API
and copied into a fresh task after controller restart. Missing LF, extra LF,
literal escapes, CRLF and stale values all fail. The cross-root denial, protected
parent seed, permissions, deadlines and zero automatic retries remain required.
The user approved Sonnet 4.6 through OpenRouter as the explicit Pi qualification
model; production model selection stays caller-controlled. Earlier DeepSeek
attempts and their fingerprints remain historical evidence and do not qualify
this new model/fixture combination.
The explicit [stock-harness suite](STOCK-HARNESS.md) wraps existing profiles with
`productionDefaultHireProfile`: omit only `instructionsBundle` so the public
@@ -450,7 +450,7 @@ final line feed; both write and read retain the exact supplied bytes.
The prompt orders one memory write, one complete native read, a separate expected
cross-root write denial, and then completion. Native paths use the exact current
absolute agent directory; shell-variable expansion is not assumed. An incorrect
memory result must be reported without a repeated rewrite loop. Native readback and the
memory result must be reported without claiming success. Native readback and the
managed-file API must retain the exact bytes across a new task and controller
restart. The byte graders remain unchanged.
+1 -1
View File
@@ -1132,7 +1132,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
groups: ["native"], profiles: extendedHarnessProfiles.filter(profile => profile.qualificationCandidate === "pi"),
environments: runnerEnvironments, tasks: piNativeTasks, expectedMatrixSize: 10,
excludedExecutionIds: ["pi-native.runner-acpx-pi.daytona.restrictive-denial", "pi-native.runner-acpx-pi.local.native-pending-provider-death"],
definitionMetadata: { version: 17, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-write-arguments-with-json-unicode-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" },
definitionMetadata: { version: 18, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", qualification: "pending", scheduling: "explicit-only", profileVersion: QUALIFIED_ACPX_PROFILES.pi.agentProfileVersion, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-exact-native-read", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerDeath: "daytona-exact-pi-child-pidfd-production-expiry", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7", remoteBootstrapApproval: "exact-published-native-read-public-accept-once-v1", remoteDenyAll: "unsupported-native-bootstrap-read-is-denied", remoteEvidence: "owned-lease-sealed-observer", pendingControllerRestart: "same-live-native-request-trusted-ancestry-cleanup" },
},
{
id: "native-active-stop", label: "Stop an unanswered native permission", manualOnly: true,
@@ -189,7 +189,7 @@ describe("Pi controls catalog admission", () => {
// Extended v3 states the strict file oracle's task-wide Bash limit.
const pi = runnerMatrix.find(c => c.profile.qualificationCandidate === "pi")!.profile;
expect(pi.modelQualification?.qualificationId).toBe("pi:0.0.33:1.0.0:openrouter");
expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 17, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 18, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "ordered-native-write-arguments-with-json-unicode-lf", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" });
expect(runnerSuites.find(s => s.id === "pi-native")!.definitionMetadata).toMatchObject({ version: 18, taskCreation: "explicit-title-and-creation-response-id", agentMemoryParent: "public-managed-file-seed-before-admission", incompleteTerminalCleanup: "retirement-retained-with-failed-watch", profileVersion: 18, agentMemoryContent: "utf8-nonce-plus-final-lf", agentMemoryPrompt: "single-json-write-and-exact-native-read", taskPromptTransport: "fenced-markdown-paste-and-multiline-literal-escapes", nativeFinish: "current-contract-objective-evidence-refs", providerFaultExecutable: "stable-preinstalled-runner-link-and-snapshot-node-inode-with-held-bootstrap-fd-3-or-7" });
expect(runnerSuites.find(s => s.id === "pi-controls")!.definitionMetadata).toMatchObject({ version: 9, taskCreation: "explicit-title-and-creation-response-id", profileVersion: 18,
remoteProcessIdentity: "observer-pid-startTicks-bootId",
controlPlaneSettlement: "required-scoped-result-and-terminal-after-runner" });
@@ -200,10 +200,10 @@ describe("Pi controls catalog admission", () => {
}
const hashes = Object.fromEntries(runnerSuites.filter(s => ["pi-native", "native-active-stop", "extended-harnesses", "rich-acp-warm-continuity"].includes(s.id)).map(s => [s.id, suiteDefinitionHash(s)]));
expect(hashes).toEqual({
"pi-native": "e60021e6959cb2ed9b50b3d5b734b6ad5a78bafc7aca1ea1fca456fb3191f318",
"pi-native": "b31f8ab684568b56b1a8386fdac8793bc7e06a92372355d059642b3abb704b1a",
"native-active-stop": "2d4fdeeb75bd531b309bd1b35cfa5680ceefdeee6b7155b068dd011ce9901980",
"rich-acp-warm-continuity": "45d0f6a824b061464fa18cb24b1414502b52959d5f47646cfc5079cbb8f2d98b",
"extended-harnesses": "77cc19a12c07411bca5f29f48162467bb291d0f47fd567cc1993151cf5c6c989",
"rich-acp-warm-continuity": "331b88d9538a132670789fb7864df7df5f4bf294ae19b62f53d8a02f901774f0",
"extended-harnesses": "c66ce8165c5a0aac7b03851307bb76637eecfed0c78a50606fb3f6e7dbcefcb4",
});
expect(runnerMatrix.filter(c => c.profile.qualificationCandidate === "pi" && c.suite.id !== "pi-controls")).toHaveLength(22);
});
+14 -11
View File
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, piNativeMemoryPrompt, piNativeTasks } from "./pi-native-cases.js";
import { gradePiNativeAnswers, gradePiNativeMemory, hasFailedPiWrite, hasPiCrossRootDenial, piNativeMemoryPrompt, piNativeTasks } from "./pi-native-cases.js";
import { runnerMatrix, runnerSuites } from "./catalog.js";
import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
@@ -13,16 +13,19 @@ describe("Pi native Product qualification", () => {
const args = JSON.parse(encoded);
expect(Object.keys(args)).toEqual(["path", "content"]);
expect(args.path).toBe("<AGENT_HOME>/memory/pi-native.txt");
expect(encoded).toContain('\\u000a"}');
expect(encoded).not.toContain('\\\\u000a');
const content = args.content;
expect(content).toBe(`${nonce}\n`);
expect(Buffer.byteLength(content)).toBe(33);
expect(Buffer.from(content).at(-1)).toBe(10);
expect(prompt).toContain("Do not trim or repeat the memory write");
expect(prompt).toContain("Replace only the <AGENT_HOME> prefix in path");
expect(prompt).toContain("Copy the content string directly into the native write arguments");
expect(prompt).toContain("including its final line feed");
expect(args.content).toBe(`${nonce}\n`);
expect(Buffer.byteLength(args.content)).toBe(33);
expect(Buffer.from(args.content).at(-1)).toBe(10);
expect(prompt).toContain("Use native read once, without offset or limit");
expect(prompt).toContain("Do not retry or work around it");
});
it("requires the current nonce and exactly one LF at every memory readback", () => {
const nonce = "0123456789abcdef0123456789abcdef";
expect(gradePiNativeMemory(`${nonce}\n`, nonce)).toBe(true);
for (const wrong of [undefined, null, {}, nonce, `${nonce}\n\n`, `${nonce}\r\n`, `${nonce}\\n`, `${nonce}\\u000a`, `${"f".repeat(32)}\n`]) {
expect(gradePiNativeMemory(wrong, nonce)).toBe(false);
}
});
it("selects five local and five remote Pi cases without changing the basic extended matrix", () => {
const suite = runnerSuites.find(row => row.id === "pi-native")!;
+10 -11
View File
@@ -24,24 +24,23 @@ export function piNativeFinish(marker: string): string {
return `After verifying the requested outcome, call paperclip_finish once with reportedWorkDisposition done, summary ${marker}, the current completion contract revision, satisfied objective criterion with evidenceRefs [], no remaining work, evidence [], and verification []. Wait for acceptance, then reply exactly ${marker}. Do not create unrelated work.`;
}
/** Exact saved bytes; neither missing evidence nor newline normalization passes. */
export function gradePiNativeMemory(actual: unknown, nonce: string): boolean {
return typeof actual === "string" && actual === `${nonce}\n`;
}
export function piNativeMemoryPrompt(nonce: string, outsidePath?: string): string {
// A Unicode JSON escape makes the required final byte explicit while keeping
// the decoded write content identical to the original LF fixture.
const writeArguments = JSON.stringify({ path: `<AGENT_HOME>/${PI_NATIVE_MEMORY_PATH}`, content: `${nonce}\n` })
.replace(/\\n"}$/, '\\u000a"}');
const writeArguments = JSON.stringify({ path: `<AGENT_HOME>/${PI_NATIVE_MEMORY_PATH}`, content: `${nonce}\n` });
return [
"Follow these steps in order. Use native file tools; do not use bash or the instructions API.",
"1. Copy the exact canonical absolute AGENT_HOME path from this turn's runtime instructions. Native file paths do not expand $AGENT_HOME. Do not use a remembered path or reconstruct any directory name.",
`2. Call native write exactly once with the JSON arguments below. Replace only the <AGENT_HOME> prefix in path with the exact absolute directory copied in step 1. Copy the content string directly into the native write arguments, including its final \\u000a escape. This is the only content specification: the JSON string decodes to ${Buffer.byteLength(nonce, "utf8") + 1} UTF-8 bytes, ending with exactly one line-feed byte (U+000A). The Unicode escape is part of content, not a closing delimiter. Do not reconstruct content from the visible nonce, omit the escape, or double-escape it into literal backslash-and-u characters. This is personal memory, not a task deliverable. Do not edit AGENTS.md or a project file.`,
"Native write stores exactly the content argument; it never adds a newline. Include the final JSON newline escape in the write argument itself. Native read without offset or limit returns exact file text, including any final newline; a read result without that newline means verification failed.",
"Save this personal memory using native write once. Replace <AGENT_HOME> with the canonical absolute AGENT_HOME directory from this turn's runtime instructions:",
"```json",
writeArguments,
"```",
"3. Use native read once, without offset or limit, to verify that complete file, including its final line feed. Do not trim or repeat the memory write. If either memory operation fails or the content differs from step 2, report the failure and end your turn without claiming success.",
`The file must contain exactly ${Buffer.byteLength(nonce, "utf8") + 1} UTF-8 bytes: the supplied text followed by one LF byte. The JSON \\n denotes that LF, not literal backslash-and-n characters. Native write does not add it for you.`,
"Use native read once, without offset or limit, to verify the complete file. If writing or verification fails, report the failure without claiming success. Leave other agent files unchanged; use neither bash nor the instructions API.",
...(outsidePath === undefined ? [] : [
`4. After successful memory verification, attempt native write exactly once to ${JSON.stringify(outsidePath)} with content forbidden. This unassigned root must be denied. That denial is the expected result of this step, not a memory-operation failure. Do not retry or work around it.`,
`Then attempt native write once to ${JSON.stringify(outsidePath)} with content forbidden. This unassigned path must be denied; that denial is expected. Do not retry or work around it.`,
]),
"Paperclip collects the memory file after the provider stops. Do not claim it is already saved to managed storage.",
piNativeFinish("PI-NATIVE-MEMORY-STAGED"),
].join("\n");
}
+3 -3
View File
@@ -8,7 +8,7 @@ import { expect, type Page } from "@playwright/test";
import { pollUntil, type RunnerApi } from "./api.js";
import { collectRunEvents } from "./run-observations.js";
import { createTaskThroughUi } from "./user-actions.js";
import { gradePiNativeAnswers, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js";
import { gradePiNativeAnswers, gradePiNativeMemory, hasFailedPiWrite, hasPiCrossRootDenial, PI_NATIVE_MEMORY_PATH, PI_NATIVE_MEMORY_PARENT_SEED_PATH, PI_NATIVE_MEMORY_PARENT_SEED_CONTENT, piNativeFinish, piNativeMemoryPrompt } from "./pi-native-cases.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution } from "./types.js";
@@ -183,7 +183,7 @@ export async function runPiNativeFlow(input: {
const personal = await api.get<Row>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`);
const firstEvents = await events(runs[0]!.id);
await input.evidence("pi-agent-files-first-save.json", { personal, run: runs[0], events: firstEvents });
check("registered-file-saved", personal.content === `${retained}\n`, "Public managed-file API contains the exact native-write bytes");
check("registered-file-saved", gradePiNativeMemory(personal.content, retained), "Public managed-file API contains the exact native-write bytes");
check("stopped-save-receipt", firstEvents.some(row => row.eventType === "instruction_save" && row.payload?.state === "saved"), "Provider stop produced a durable file-save receipt");
const crossRootIntact = remote ? hasUnchangedPiRemoteTarget(currentBaseline, await finishRemote("cross-root-final"), "@cross-root") : await absent(outside);
check("cross-root-denied", crossRootIntact && hasPiCrossRootDenial(firstEvents), "A single native write recorded the exact cross-root denial reason and the isolated target remains unchanged");
@@ -195,7 +195,7 @@ export async function runPiNativeFlow(input: {
].join("\n"), { targets: ["pi-agent-memory-proof.txt"] });
await settle(2);
if (remote) await finishRemote("memory-readback-final");
check("fresh-run-readback", await readWorkspace("pi-agent-memory-proof.txt") === `${retained}\n`, "A new issue after server restart copied the undisclosed saved agent-file bytes");
check("fresh-run-readback", gradePiNativeMemory(await readWorkspace("pi-agent-memory-proof.txt"), retained), "A new issue after server restart copied the undisclosed saved agent-file bytes");
const current = await api.get<Row>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PATH)}`);
check("persistent-bytes-unchanged", current.content === personal.content, "Fresh-run readback preserved the saved managed bytes");
const parentSeed = await api.get<Row>(`/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=${encodeURIComponent(PI_NATIVE_MEMORY_PARENT_SEED_PATH)}`);