mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-24 10:13:11 +02:00
test: separate instruction proof nonce from fixture identity
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
4df2d3a8d7
commit
6abe6cf899
3 files changed
+10
-6
No files matched your search
@@ -128,7 +128,8 @@ for the same agent. The first edits only its registered private instruction file
|
||||
using ordinary filesystem tools; the oracle checks exact canonical bytes and a
|
||||
new cleanup revision bound to that run. The harness restarts Paperclip and creates
|
||||
a fresh task, which must upload a downloaded text attachment containing the saved
|
||||
nonce, which is absent from the second task's title and prompt. The browser then
|
||||
nonce. This value is independent of fixture names and absent from the second
|
||||
task's title and prompt. The browser then
|
||||
inspects history and restores the baseline as a new revision. A third task edits
|
||||
its private copy while a board API save changes the canonical revision. Cleanup
|
||||
must preserve the conflict. The browser compares the current content and explicitly
|
||||
|
||||
@@ -919,7 +919,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
|
||||
environments: [localEnvironment, runnerEnvironments.find(environment => environment.id === "daytona")!], tasks: [instructionPersistenceTask],
|
||||
excludedExecutionIds: ["instruction-persistence.legacy-codex.daytona.private-copy-persists"],
|
||||
expectedMatrixSize: 3, manualOnly: true,
|
||||
definitionMetadata: { version: 2, oracle: "canonical-revision-blind-attachment-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" },
|
||||
definitionMetadata: { version: 3, oracle: "canonical-revision-independent-nonce-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" },
|
||||
},
|
||||
{
|
||||
id: "continuation-accounting", label: "Continuation accounting baseline", manualOnly: true,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { expect, type Page } from "@playwright/test";
|
||||
import { pollUntil, type RunnerApi } from "./api.js";
|
||||
import { captureFirstTaskAttachments } from "./first-task-attachments.js";
|
||||
@@ -43,10 +43,13 @@ export async function runInstructionPersistenceFlow(input: {
|
||||
evidence(name: string, data: unknown): Promise<void>;
|
||||
}) {
|
||||
const { page, api, fixtures, execution, nonce } = input;
|
||||
// Fixture names contain the campaign nonce. Use an unrelated value that the
|
||||
// fresh task can obtain only from the saved entry (or forbidden task history).
|
||||
const persistedNonce = randomBytes(16).toString("hex");
|
||||
const filePath = `/api/agents/${fixtures.agent.id}/instructions-bundle/file?path=AGENTS.md`;
|
||||
const before = await api.get<Row>(filePath);
|
||||
if (typeof before.content !== "string" || !before.revision?.id) throw new Error("Managed instructions must expose a canonical baseline revision");
|
||||
const expectedContent = `${before.content}\n${instructionNonceLine(nonce)}`;
|
||||
const expectedContent = `${before.content}\n${instructionNonceLine(persistedNonce)}`;
|
||||
let issue: Row = {};
|
||||
let runs: Row[] = [];
|
||||
async function create(title: string, prompt: string) {
|
||||
@@ -75,7 +78,7 @@ export async function runInstructionPersistenceFlow(input: {
|
||||
await page.reload();
|
||||
await expect(page.getByTestId("issue-detail-header").getByRole("button", { name: "Change status (current: Done)", exact: true })).toBeVisible();
|
||||
}
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(nonce));
|
||||
await create(execution.task.buildTitle(nonce), execution.task.buildPrompt(persistedNonce));
|
||||
await settle(1);
|
||||
const firstRunId = runs[0]!.id;
|
||||
const after = await pollUntil({ label: "stopped instruction cleanup revision", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000),
|
||||
@@ -98,7 +101,7 @@ export async function runInstructionPersistenceFlow(input: {
|
||||
const proof = attachments.find(row => row.originalFilename === "instruction-proof.txt" || row.name === "instruction-proof.txt");
|
||||
const final = await api.get<Row>(filePath);
|
||||
expect(final.revision.id).toBe(after.revision.id);
|
||||
const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(nonce) });
|
||||
const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(persistedNonce) });
|
||||
await expect(page.getByTestId("task-chat-agent-bubble").filter({ hasText: execution.task.buildVisibleMarker(nonce) }).last()).toBeVisible();
|
||||
await input.capture("final-state", "Fresh task downloaded the persisted instruction nonce", "final-state.png");
|
||||
expect(checks.filter(check => !check.passed), "Independent instruction persistence checks").toEqual([]);
|
||||
|
||||
Reference in new issue
Block a user