From 10c89e0d391887da91adfa1c55cfd954497f4428 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 27 Sep 2026 17:26:18 -0500 Subject: [PATCH] test: conceal persisted nonce from fresh instruction task Co-Authored-By: Paperclip --- tests/runner-e2e/instruction-persistence.test.ts | 6 +++++- tests/runner-e2e/instruction-persistence.ts | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/instruction-persistence.test.ts b/tests/runner-e2e/instruction-persistence.test.ts index 6f478b0f5fd..8b19f90e940 100644 --- a/tests/runner-e2e/instruction-persistence.test.ts +++ b/tests/runner-e2e/instruction-persistence.test.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; import { describe, expect, it } from "vitest"; -import { gradeInstructionPersistence } from "./instruction-persistence.js"; +import { gradeInstructionPersistence, instructionPersistenceTask } from "./instruction-persistence.js"; import { runnerMatrix } from "./catalog.js"; const expectedContent = "original\nInstruction persistence nonce: fixture\n"; const valid = () => ({ before: { revision: { id: "before" } }, @@ -9,6 +9,10 @@ const valid = () => ({ before: { revision: { id: "before" } }, proof: { body: "Instruction persistence nonce: fixture\n", contentVerified: true }, expectedProof: "Instruction persistence nonce: fixture\n" }); describe("instruction persistence independent oracle", () => { + it("does not disclose the persisted nonce in the fresh-task marker", () => { + const nonce = "only-the-instruction-entry-reveals-this"; + expect(instructionPersistenceTask.buildVisibleMarker(nonce)).not.toContain(nonce); + }); it("accepts exact cleanup content and a verified downloaded fresh-task proof", () => { expect(gradeInstructionPersistence(valid()).every(row => row.passed)).toBe(true); }); diff --git a/tests/runner-e2e/instruction-persistence.ts b/tests/runner-e2e/instruction-persistence.ts index 0483b74cefb..fb3219e6487 100644 --- a/tests/runner-e2e/instruction-persistence.ts +++ b/tests/runner-e2e/instruction-persistence.ts @@ -15,7 +15,7 @@ export const instructionPersistenceTask: RunnerTaskFixture = { expectedRunCount: 2, attemptTimeoutMs: { local: 20 * 60_000, daytona: 20 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: nonce => `Persist private instructions ${nonce}`, - buildVisibleMarker: nonce => `INSTRUCTIONS-VERIFIED-${nonce}`, + buildVisibleMarker: () => "INSTRUCTIONS-VERIFIED", buildPrompt: nonce => [ "Edit your own registered writable agent instruction entry with ordinary filesystem tools. The runtime guidance gives its exact private path.", `Preserve its existing bytes and append exactly one newline followed by this line and a final newline: Instruction persistence nonce: ${nonce}`, @@ -86,7 +86,7 @@ export async function runInstructionPersistenceFlow(input: { // A new server and a new issue cannot pass by retaining model conversation. await input.restart(); expect((await api.get(filePath)).content).toBe(expectedContent); - await create(`Read persisted instructions ${nonce}`, [ + await create("Read persisted instructions", [ "Read your own loaded agent instruction entry (or its current registered private copy) using ordinary filesystem tools.", "Find the line beginning 'Instruction persistence nonce: '. Copy that entire line plus one final newline into instruction-proof.txt. Do not infer the value from this task title or other task history. Do not change your instructions.", "Upload instruction-proof.txt as a text/plain task attachment named instruction-proof.txt using the normal artifact workflow. A local file alone is insufficient.",