From 30b391e7de7b91f41d031ae92bceba3c846620bf Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 27 Sep 2026 18:20:54 -0500 Subject: [PATCH] test: separate instruction proof nonce from fixture identity Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 3 ++- tests/runner-e2e/catalog.ts | 2 +- tests/runner-e2e/instruction-persistence.ts | 11 +++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 463201bab03..046835ed84f 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -128,7 +128,8 @@ for the same agent. The first edits only its registered private instruction file using ordinary filesystem tools; the oracle checks exact canonical bytes and a new cleanup revision bound to that run. The harness restarts Paperclip and creates a fresh task, which must upload a downloaded text attachment containing the saved -nonce, which is absent from the second task's title and prompt. The browser then +nonce. This value is independent of fixture names and absent from the second +task's title and prompt. The browser then inspects history and restores the baseline as a new revision. A third task edits its private copy while a board API save changes the canonical revision. Cleanup must preserve the conflict. The browser compares the current content and explicitly diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 04e208592a3..e4e06e60326 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -929,7 +929,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment, runnerEnvironments.find(environment => environment.id === "daytona")!], tasks: [instructionPersistenceTask], excludedExecutionIds: ["instruction-persistence.legacy-codex.daytona.private-copy-persists"], expectedMatrixSize: 3, manualOnly: true, - definitionMetadata: { version: 2, oracle: "canonical-revision-blind-attachment-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" }, + definitionMetadata: { version: 3, oracle: "canonical-revision-independent-nonce-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" }, }, { id: "continuation-accounting", label: "Continuation accounting baseline", manualOnly: true, diff --git a/tests/runner-e2e/instruction-persistence.ts b/tests/runner-e2e/instruction-persistence.ts index ae7bcc9110d..2a54db93ad2 100644 --- a/tests/runner-e2e/instruction-persistence.ts +++ b/tests/runner-e2e/instruction-persistence.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { expect, type Page } from "@playwright/test"; import { pollUntil, type RunnerApi } from "./api.js"; import { captureFirstTaskAttachments } from "./first-task-attachments.js"; @@ -43,10 +43,13 @@ export async function runInstructionPersistenceFlow(input: { evidence(name: string, data: unknown): Promise; }) { const { page, api, fixtures, execution, nonce } = input; + // Fixture names contain the campaign nonce. Use an unrelated value that the + // fresh task can obtain only from the saved entry (or forbidden task history). + const persistedNonce = randomBytes(16).toString("hex"); const filePath = `/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=AGENTS.md`; const before = await api.get(filePath); if (typeof before.content !== "string" || !before.revision?.id) throw new Error("Managed instructions must expose a canonical baseline revision"); - const expectedContent = `${before.content}\n${instructionNonceLine(nonce)}`; + const expectedContent = `${before.content}\n${instructionNonceLine(persistedNonce)}`; let issue: Row = {}; let runs: Row[] = []; async function create(title: string, prompt: string) { @@ -75,7 +78,7 @@ export async function runInstructionPersistenceFlow(input: { await page.reload(); await expect(page.getByTestId("issue-detail-header").getByRole("button", { name: "Change status (current: Done)", exact: true })).toBeVisible(); } - await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce)); + await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(persistedNonce)); await settle(1); const firstRunId = runs[0]!.id; const after = await pollUntil({ label: "stopped instruction cleanup revision", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), @@ -98,7 +101,7 @@ export async function runInstructionPersistenceFlow(input: { const proof = attachments.find(row => row.originalFilename === "instruction-proof.txt" || row.name === "instruction-proof.txt"); const final = await api.get(filePath); expect(final.revision.id).toBe(after.revision.id); - const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(nonce) }); + const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(persistedNonce) }); await expect(page.getByTestId("task-chat-agent-bubble").filter({ hasText: execution.task.buildVisibleMarker(nonce) }).last()).toBeVisible(); await input.capture("final-state", "Fresh task downloaded the persisted instruction nonce", "final-state.png"); expect(checks.filter(check => !check.passed), "Independent instruction persistence checks").toEqual([]);