test: conceal persisted nonce from fresh instruction task

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-27 17:26:18 -05:00
1 parent 3b5808109c
commit 10c89e0d39
2 files changed
+7 -3

No files matched your search

@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { describe, expect, it } from "vitest";
import { gradeInstructionPersistence } from "./instruction-persistence.js";
import { gradeInstructionPersistence, instructionPersistenceTask } from "./instruction-persistence.js";
import { runnerMatrix } from "./catalog.js";
const expectedContent = "original\nInstruction persistence nonce: fixture\n";
const valid = () => ({ before: { revision: { id: "before" } },
@@ -9,6 +9,10 @@ const valid = () => ({ before: { revision: { id: "before" } },
proof: { body: "Instruction persistence nonce: fixture\n", contentVerified: true },
expectedProof: "Instruction persistence nonce: fixture\n" });
describe("instruction persistence independent oracle", () => {
it("does not disclose the persisted nonce in the fresh-task marker", () => {
const nonce = "only-the-instruction-entry-reveals-this";
expect(instructionPersistenceTask.buildVisibleMarker(nonce)).not.toContain(nonce);
});
it("accepts exact cleanup content and a verified downloaded fresh-task proof", () => {
expect(gradeInstructionPersistence(valid()).every(row => row.passed)).toBe(true);
});
+2 -2
View File
@@ -15,7 +15,7 @@ export const instructionPersistenceTask: RunnerTaskFixture = {
expectedRunCount: 2, attemptTimeoutMs: { local: 20 * 60_000, daytona: 20 * 60_000 },
expectedTerminalState: { issue: "done", run: "succeeded" },
buildTitle: nonce => `Persist private instructions ${nonce}`,
buildVisibleMarker: nonce => `INSTRUCTIONS-VERIFIED-${nonce}`,
buildVisibleMarker: () => "INSTRUCTIONS-VERIFIED",
buildPrompt: nonce => [
"Edit your own registered writable agent instruction entry with ordinary filesystem tools. The runtime guidance gives its exact private path.",
`Preserve its existing bytes and append exactly one newline followed by this line and a final newline: Instruction persistence nonce: ${nonce}`,
@@ -86,7 +86,7 @@ export async function runInstructionPersistenceFlow(input: {
// A new server and a new issue cannot pass by retaining model conversation.
await input.restart();
expect((await api.get<Row>(filePath)).content).toBe(expectedContent);
await create(`Read persisted instructions ${nonce}`, [
await create("Read persisted instructions", [
"Read your own loaded agent instruction entry (or its current registered private copy) using ordinary filesystem tools.",
"Find the line beginning 'Instruction persistence nonce: '. Copy that entire line plus one final newline into instruction-proof.txt. Do not infer the value from this task title or other task history. Do not change your instructions.",
"Upload instruction-proof.txt as a text/plain task attachment named instruction-proof.txt using the normal artifact workflow. A local file alone is insufficient.",