test: separate instruction proof nonce from fixture identity

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-27 18:33:40 -05:00
1 parent ffac79eb55
commit 30b391e7de
3 files changed
+10 -6

No files matched your search

+2 -1
View File
@@ -128,7 +128,8 @@ for the same agent. The first edits only its registered private instruction file
using ordinary filesystem tools; the oracle checks exact canonical bytes and a
new cleanup revision bound to that run. The harness restarts Paperclip and creates
a fresh task, which must upload a downloaded text attachment containing the saved
nonce, which is absent from the second task's title and prompt. The browser then
nonce. This value is independent of fixture names and absent from the second
task's title and prompt. The browser then
inspects history and restores the baseline as a new revision. A third task edits
its private copy while a board API save changes the canonical revision. Cleanup
must preserve the conflict. The browser compares the current content and explicitly
+1 -1
View File
@@ -929,7 +929,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
environments: [localEnvironment, runnerEnvironments.find(environment => environment.id === "daytona")!], tasks: [instructionPersistenceTask],
excludedExecutionIds: ["instruction-persistence.legacy-codex.daytona.private-copy-persists"],
expectedMatrixSize: 3, manualOnly: true,
definitionMetadata: { version: 2, oracle: "canonical-revision-blind-attachment-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" },
definitionMetadata: { version: 3, oracle: "canonical-revision-independent-nonce-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" },
},
{
id: "continuation-accounting", label: "Continuation accounting baseline", manualOnly: true,
+7 -4
View File
@@ -1,4 +1,4 @@
import { createHash } from "node:crypto";
import { createHash, randomBytes } from "node:crypto";
import { expect, type Page } from "@playwright/test";
import { pollUntil, type RunnerApi } from "./api.js";
import { captureFirstTaskAttachments } from "./first-task-attachments.js";
@@ -43,10 +43,13 @@ export async function runInstructionPersistenceFlow(input: {
evidence(name: string, data: unknown): Promise<void>;
}) {
const { page, api, fixtures, execution, nonce } = input;
// Fixture names contain the campaign nonce. Use an unrelated value that the
// fresh task can obtain only from the saved entry (or forbidden task history).
const persistedNonce = randomBytes(16).toString("hex");
const filePath = `/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=AGENTS.md`;
const before = await api.get<Row>(filePath);
if (typeof before.content !== "string" || !before.revision?.id) throw new Error("Managed instructions must expose a canonical baseline revision");
const expectedContent = `${before.content}\n${instructionNonceLine(nonce)}`;
const expectedContent = `${before.content}\n${instructionNonceLine(persistedNonce)}`;
let issue: Row = {};
let runs: Row[] = [];
async function create(title: string, prompt: string) {
@@ -75,7 +78,7 @@ export async function runInstructionPersistenceFlow(input: {
await page.reload();
await expect(page.getByTestId("issue-detail-header").getByRole("button", { name: "Change status (current: Done)", exact: true })).toBeVisible();
}
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce));
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(persistedNonce));
await settle(1);
const firstRunId = runs[0]!.id;
const after = await pollUntil({ label: "stopped instruction cleanup revision", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000),
@@ -98,7 +101,7 @@ export async function runInstructionPersistenceFlow(input: {
const proof = attachments.find(row => row.originalFilename === "instruction-proof.txt" || row.name === "instruction-proof.txt");
const final = await api.get<Row>(filePath);
expect(final.revision.id).toBe(after.revision.id);
const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(nonce) });
const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(persistedNonce) });
await expect(page.getByTestId("task-chat-agent-bubble").filter({ hasText: execution.task.buildVisibleMarker(nonce) }).last()).toBeVisible();
await input.capture("final-state", "Fresh task downloaded the persisted instruction nonce", "final-state.png");
expect(checks.filter(check => !check.passed), "Independent instruction persistence checks").toEqual([]);