From 91320e9c0773628b74178aedc9adfd75a96cfb3c Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 27 Sep 2026 18:14:27 -0500 Subject: [PATCH] test: verify instruction history and conflict resolution in browser Co-Authored-By: Paperclip --- tests/runner-e2e/FIXTURES.md | 9 ++- tests/runner-e2e/README.md | 11 +++- tests/runner-e2e/catalog.ts | 2 +- .../instruction-persistence.test.ts | 2 +- tests/runner-e2e/instruction-persistence.ts | 62 ++++++++++++++++++- 5 files changed, 76 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 2ed4c7a85a1..6d7620384b0 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -231,6 +231,11 @@ hooks or database mutations are used by the fixture. The `instruction_persistence` flow uses production managed instructions and the public versioned entry endpoint. It preserves the initial content, requires an ordinary private file edit to create a cleanup-source revision, restarts the server, -and creates a fresh browser task. Its final oracle downloads and verifies the +and creates a fresh browser task without disclosing the nonce. Its readback oracle downloads and verifies the attachment bytes and SHA-256, rather than accepting a filename or model claim. -Both tasks' runs count toward billing and teardown. The suite is explicit-only. +The browser inspects history and restores exact baseline bytes. A third task +uploads a ready attachment and waits in an ordinary bounded shell command while +the board changes the canonical file through the public API. Stopped cleanup must +preserve the original candidate as a conflict. The browser reviews the current +content and explicitly saves the preserved edit against that revision. All three +tasks' runs count toward billing and teardown. The suite is explicit-only. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 0779660bf7a..b829951a290 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -123,13 +123,18 @@ Paperclip server while the interaction is waiting, reloads that state, and then resumes it. The suite has no Daytona cells. `instruction-persistence` is an explicit-only three-cell workflow: legacy and -native Codex locally, plus native Codex on Daytona. Each creates two browser tasks +native Codex locally, plus native Codex on Daytona. Each creates three browser tasks for the same agent. The first edits only its registered private instruction file using ordinary filesystem tools; the oracle checks exact canonical bytes and a new cleanup revision bound to that run. The harness restarts Paperclip and creates a fresh task, which must upload a downloaded text attachment containing the saved -nonce. It rejects tool/API saves, stale revisions, missing downloads, and incorrect -bytes. The deadline is twenty minutes per cell, with two expected provider runs; +nonce, which is absent from the second task's title and prompt. The browser then +inspects history and restores the baseline as a new revision. A third task edits +its private copy while a board API save changes the canonical revision. Cleanup +must preserve the conflict. The browser compares the current content and explicitly +saves the preserved candidate; the oracle checks exact bytes and parent revision. +It rejects tool/API cleanup saves, stale revisions, missing downloads, and incorrect +bytes. The deadline is twenty minutes per cell, with three expected provider runs; normal instance/Daytona cleanup, screenshots, evidence, and billing apply. Run with `pnpm test:e2e:runner -- --suite instruction-persistence`. The unchanged-provider warm lifecycle is independently covered by `daytona-warm-continuity`. diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 946bc177866..5eca058827a 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -919,7 +919,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [localEnvironment, runnerEnvironments.find(environment => environment.id === "daytona")!], tasks: [instructionPersistenceTask], excludedExecutionIds: ["instruction-persistence.legacy-codex.daytona.private-copy-persists"], expectedMatrixSize: 3, manualOnly: true, - definitionMetadata: { version: 1, oracle: "canonical-revision-and-downloaded-attachment", providerTurns: 2, restart: "between-tasks", instructions: "production" }, + definitionMetadata: { version: 2, oracle: "canonical-revision-blind-attachment-and-browser-conflict-resolution", providerTurns: 3, restart: "between-tasks", instructions: "production" }, }, { id: "continuation-accounting", label: "Continuation accounting baseline", manualOnly: true, diff --git a/tests/runner-e2e/instruction-persistence.test.ts b/tests/runner-e2e/instruction-persistence.test.ts index 8b19f90e940..5c1b4030524 100644 --- a/tests/runner-e2e/instruction-persistence.test.ts +++ b/tests/runner-e2e/instruction-persistence.test.ts @@ -30,6 +30,6 @@ describe("instruction persistence independent oracle", () => { it("registers exactly three explicit cells without changing scheduled campaigns", () => { const cells = runnerMatrix.filter(row => row.suite.id === "instruction-persistence"); expect(cells.map(row => `${row.profile.id}.${row.environment.id}`)).toEqual(["legacy-codex.local", "runner-codex.local", "runner-codex.daytona"]); - expect(cells.every(row => row.suite.manualOnly && row.task.expectedRunCount === 2)).toBe(true); + expect(cells.every(row => row.suite.manualOnly && row.task.expectedRunCount === 3)).toBe(true); }); }); diff --git a/tests/runner-e2e/instruction-persistence.ts b/tests/runner-e2e/instruction-persistence.ts index 1bbb87326ad..ae7bcc9110d 100644 --- a/tests/runner-e2e/instruction-persistence.ts +++ b/tests/runner-e2e/instruction-persistence.ts @@ -12,7 +12,7 @@ export const instructionNonceLine = (nonce: string) => `Instruction persistence export const instructionPersistenceTask: RunnerTaskFixture = { id: "private-copy-persists", label: "Private instruction edit survives a fresh task", groups: [], workMode: "standard", flow: "instruction_persistence", - expectedRunCount: 2, attemptTimeoutMs: { local: 20 * 60_000, daytona: 20 * 60_000 }, + expectedRunCount: 3, attemptTimeoutMs: { local: 20 * 60_000, daytona: 20 * 60_000 }, expectedTerminalState: { issue: "done", run: "succeeded" }, buildTitle: nonce => `Persist private instructions ${nonce}`, buildVisibleMarker: () => "INSTRUCTIONS-VERIFIED", @@ -99,10 +99,66 @@ export async function runInstructionPersistenceFlow(input: { const final = await api.get(filePath); expect(final.revision.id).toBe(after.revision.id); const checks = gradeInstructionPersistence({ before, after, firstRunId, expectedContent, proof, expectedProof: instructionNonceLine(nonce) }); - await input.evidence("api-state.json", { issue, runs, checks, canonicalInstructions: final, attachments }); - await input.evidence("instruction-persistence.json", { checks, before, after, final, issues: [runs[0]?.nativeIssueId, issue.id], runs, attachments }); await expect(page.getByTestId("task-chat-agent-bubble").filter({ hasText: execution.task.buildVisibleMarker(nonce) }).last()).toBeVisible(); await input.capture("final-state", "Fresh task downloaded the persisted instruction nonce", "final-state.png"); expect(checks.filter(check => !check.passed), "Independent instruction persistence checks").toEqual([]); + + const instructionsUrl = `/${fixtures.company.issuePrefix}/agents/${fixtures.agent.id}/instructions`; + await page.goto(instructionsUrl); + await page.getByRole("button", { name: "History", exact: true }).click(); + await page.getByRole("button", { name: before.revision.id.slice(0, 8), exact: true }).click(); + await expect(page.getByText("Changes from selected revision to current", { exact: true })).toBeVisible(); + await page.getByRole("button", { name: "Restore as new revision", exact: true }).click(); + const restored = await pollUntil({ label: "browser restored instruction revision", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), + load: () => api.get(filePath), accept: row => row.content === before.content && row.revision?.source === "restore" && row.revision?.restoredFromRevisionId === before.revision.id }); + expect(restored.revision.parentRevisionId).toBe(after.revision.id); + checks.push({ id: "browser-history-and-restore", passed: true, detail: "Browser inspected history and restored exact baseline bytes as a new revision" }); + + // The provider publishes an ordinary attachment before a bounded command wait. + // A board edit during that run creates a real CAS conflict at stopped cleanup. + const conflictSuffix = `\nPreserved instruction candidate: ${nonce}\n`; + const expectedCandidate = `${restored.content}${conflictSuffix}`; + await create("Preserve a concurrent instruction edit", [ + `Append exactly this UTF-8 suffix to your current registered writable instruction entry, represented as a JSON string: ${JSON.stringify(conflictSuffix)}`, + "Decode the JSON string once. Preserve all existing bytes. Do not use an instruction revision tool or instructions API.", + "After the file edit, upload a text/plain task attachment named instruction-candidate-ready.txt with the text ready. Use the ordinary artifact workflow.", + "Then execute the ordinary shell command sleep 45 and wait for it to finish. This gives the board time to edit the canonical instructions concurrently. Do not complete the task before that command finishes.", + "After the wait completes, reply Candidate edit ready and complete the task. Do not change the instructions again or claim that they saved.", + ].join("\n")); + await pollUntil({ label: "provider staged concurrent instruction edit", deadlineAt: input.deadlineAt, + load: () => api.get(`/api/issues/${issue.id}/attachments`), + accept: rows => rows.some(row => row.originalFilename === "instruction-candidate-ready.txt" || row.name === "instruction-candidate-ready.txt") }); + const active = await api.get(`/api/issues/${issue.id}/runs`); + expect(active.some(row => row.status === "running")).toBe(true); + const boardContent = `${restored.content}\nConcurrent board instruction edit.\n`; + const boardResponse = await api.request.put(`/api/agents/${fixtures.agent.id}/instructions-bundle/file`, { + data: { path: "AGENTS.md", content: boardContent, baseRevisionId: restored.revision.id }, + }); + expect(boardResponse.ok()).toBe(true); + const board = await boardResponse.json() as Row; + await settle(3); + const conflictRunId = runs[2]!.id; + const candidatePath = `/api/agents/${fixtures.agent.id}/instructions-bundle/candidates`; + const candidates = await pollUntil({ label: "preserved instruction conflict", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), + load: () => api.get(candidatePath), accept: rows => rows.some(row => row.runId === conflictRunId && row.state === "conflict") }); + const candidate = candidates.find(row => row.runId === conflictRunId)!; + expect(candidate.content).toBe(expectedCandidate); + expect(candidate.baseRevisionId).toBe(restored.revision.id); + expect((await api.get(filePath)).content).toBe(boardContent); + await page.goto(instructionsUrl); + await page.getByRole("button", { name: "Review preserved edits", exact: true }).click(); + await page.getByText("Compare current instructions", { exact: true }).click(); + await expect(page.locator("details pre")).toHaveText(boardContent); + // Explicitly choose the preserved candidate after reviewing the board edit. + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + const resolved = await pollUntil({ label: "browser resolved preserved instruction candidate", deadlineAt: Math.min(input.deadlineAt, Date.now() + 30_000), + load: () => api.get(filePath), accept: row => row.content === expectedCandidate && row.revision?.parentRevisionId === board.revision.id && row.revision?.source === "api" }); + await expect(page.getByRole("button", { name: "Review preserved edits", exact: true })).toHaveCount(0); + expect((await api.get(candidatePath)).some(row => row.runId === conflictRunId)).toBe(false); + checks.push({ id: "browser-preserved-conflict-resolution", passed: true, detail: "Concurrent canonical edit survived cleanup; browser review and explicit save resolved the preserved candidate" }); + await input.evidence("api-state.json", { issue, runs, checks, canonicalInstructions: resolved, attachments }); + await input.evidence("instruction-persistence.json", { checks, before, after, final, restored, board, candidate, resolved, runs, attachments }); + await page.goto(`/${fixtures.company.issuePrefix}/issues/${issue.identifier ?? issue.id}`); + await input.capture("conflict-resolved", "Stopped run preserved its concurrent instruction edit for explicit review", "conflict-resolved.png"); return { issue, runs, checks }; }