From a4b9285b601d30a4ef26266105f345b609de8621 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 7 Nov 2026 20:53:59 -0600 Subject: [PATCH] test(evals): grade restore-document by saved revisions, not reply wording The concurrent phase required the reply to say Casey, offline and prototype, and the approval check scanned reply words. The read and final phases now accept a revision identified by id or number, the final phase accepts naming the intervening edit by revision or editor, and saved state decides restore versus preserve. Approval is checked from activity only. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-documents-artifacts.ts | 4 +-- .../internal-restore-concurrency.test.ts | 10 +++++-- .../internal-restore-concurrency.ts | 26 +++++++++---------- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 6e1d095274f..01c1a72a3d8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2845,7 +2845,7 @@ "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.restore-document.saved-edit.v2", "coverageGaps": [], "expectedRunCount": 3, "expectedHumanMessageCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index 82af53bfd46..c36c847daf8 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -1,4 +1,4 @@ -import { withRestoreConcurrency } from "./internal-restore-concurrency.js"; +import { withRestoreConcurrency, identifiesRevision } from "./internal-restore-concurrency.js"; import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { privateDownloadCases } from "./internal-private-download.js"; @@ -54,7 +54,7 @@ export function grade_documents_artifacts(value: unknown) { && revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration."); c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision."); - c.add("no-execution-rewind", !!before && sameTask(before, current) && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged."); + c.add("no-execution-rewind", !!before && sameTask(before, current) && !!plan && identifiesRevision(o.reply, plan), "Chat identifies the resulting current document revision and leaves task execution unchanged."); break; } default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario."); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts index c3de71bb4a5..3c40eed2f46 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts @@ -21,7 +21,13 @@ describe("real concurrent document restoration calibration", () => { it("accepts preserving the actual newer edit for truthful clarification", () => { const o = valid(); o.documents[0] = { ...o.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: o.state.interveningRevision.body }; o.revisions = o.revisions.filter(row => row.id !== "v7"); o.activity = o.activity.filter(row => row.id !== "restore2"); o.phaseReply = "Casey's newer offline research prototype edit is current. Please confirm whether to replace that change with the selected historical revision?"; expect(grade(o).passed).toBe(true); }); it("rejects crediting a wrong first restoration repaired later", () => { const o = valid(); o.state.firstRestore.documents[0]!.body = "third"; expect(grade(o).passed).toBe(false); }); it("requires actual fresh read/run attribution before the other editor", () => { const o = valid(); o.state.readReply.createdByRunId = "first"; expect(grade(o).passed).toBe(false); const time = valid(); time.state.interveningRevision.createdAt = "2026-10-26T11:59:00.000Z"; expect(grade(time).passed).toBe(false); }); - it("rejects wrong-phase-only or invented intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const invented = valid(); invented.phaseReply = "Casey's pricing edit was preserved while I restored revision 7."; expect(grade(invented).passed).toBe(false); }); + it("rejects wrong-phase-only or unread intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const unread = valid(); unread.phaseReply = "If someone else edited the plan, that change stays in history. I restored revision 2 as current revision 7."; expect(grade(unread).passed).toBe(false); }); + it("accepts reworded handling that names the saved edit by revision instead of editor or content", () => { const o = valid(); o.phaseReply = "A newer edit (revision 6) landed after my read. It stays in history; the selected text is now current as v7."; expect(grade(o).passed).toBe(true); + const asked = valid(); asked.documents[0] = { ...asked.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: asked.state.interveningRevision.body }; asked.revisions = asked.revisions.filter(row => row.id !== "v7"); asked.activity = asked.activity.filter(row => row.id !== "restore2"); asked.phaseReply = "Casey saved a change after my read, so I left it in place. Tell me to go ahead and I will restore the selected text over it."; expect(grade(asked).passed).toBe(true); }); + it("rejects a final restore that names the edit but leaves the wrong text or no current revision", () => { const wrong = valid(); wrong.documents[0]!.body = "third"; wrong.revisions.find(row => row.id === "v7")!.body = "third"; expect(grade(wrong).passed).toBe(false); + const unnamed = valid(); unnamed.phaseReply = "I found Casey's newer edit and restored the selected plan, keeping history."; expect(grade(unnamed).passed).toBe(false); }); it("requires retained human revision and final-run restoration audit", () => { const o = valid(); o.revisions = o.revisions.filter(row => row.id !== "v6"); expect(grade(o).passed).toBe(false); const oldRun = valid(); Object.assign(oldRun.activity.find(row => row.id === "restore2")!, { runId: "first" }); expect(grade(oldRun).passed).toBe(false); }); - it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.phaseReply += " The old work is approved."; expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); }); + it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.activity.push({ id: "approve", action: "approval.approved", entityId: "task", agentId: "ceo" }); expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); }); + it("accepts a read reply that identifies the checkpoint by revision number", () => { const o = valid(); o.state.readCheckpoint.reply = "Human checkpoint ORCHID, current revision 5."; o.state.readReply.body = "Human checkpoint ORCHID, revision 5"; expect(grade(o).passed).toBe(true); + const stale = valid(); stale.state.readCheckpoint.reply = "Current revision 4 (latest): second"; expect(grade(stale).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts index 2d2b8f50161..f1c88af0adc 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts @@ -7,7 +7,7 @@ import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity- import { checks, entity, observation, observeInternal, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js"; interface Revision extends Entity { createdByUserId?: string | null } interface ConcurrentRestore extends InternalObservation { phaseReply: string } -function identifiesRevision(reply: string, doc: Entity) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId) +export function identifiesRevision(reply: string, doc: Pick) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId) || Number.isSafeInteger(doc.latestRevisionNumber) && new RegExp(`\\b(?:revision|version|v)\\s*#?${doc.latestRevisionNumber}\\b`, "i").test(reply); } export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalogCase["grade"]) { const c = checks(); const o = observation(value) as ConcurrentRestore | undefined; @@ -17,10 +17,10 @@ export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalog c.add("canonical-restoration-complete-before-variant", !!first && initialGrade(first).passed && !!initialDoc && identifiesRevision(first.reply, initialDoc), "The unchanged first query restores the exact selected history through its revision path and identifies the resulting current revision before any variant edits."); const readReply = o.state.readReply as Entity | undefined; const checkpoint = o.state.preReadRevision as Revision | undefined; const changed = o.state.interveningRevision as Revision | undefined; - c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && read.reply.includes(checkpoint.id) + c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && identifiesRevision(read.reply, { latestRevisionId: checkpoint.id, latestRevisionNumber: checkpoint.latestRevisionNumber }) && read.documents.some(row => row.latestRevisionId === checkpoint.id && row.body === checkpoint.body) && read.runs.length === 2 && read.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") - && !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && (readReply.body ?? "").includes(checkpoint.id) + && !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && read.runs.some(run => run.id === readReply.createdByRunId) && !first?.runs.some(run => run.id === readReply.createdByRunId) && o.revisions.some(row => row.id === checkpoint.id && row.body === checkpoint.body && (row as Revision).createdByUserId === o.state.editorId) && o.revisions.some(row => row.id === changed.id && row.body === changed.body && (row as Revision).createdByUserId === o.state.editorId) @@ -32,21 +32,19 @@ export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalog && o.revisions.some(row => row.id === current.latestRevisionId && row.createdByAgentId === o.leadId && row.body === wanted.body) && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === entity(o.state, "task").id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === wanted.id && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")) - && identifiesRevision(o.phaseReply, current) && /\b(?:restored|restoring)\b/i.test(o.phaseReply); + && identifiesRevision(o.phaseReply, current); const preservedForClarification = current?.latestRevisionId === changed?.id && current?.body === changed?.body - && /\?|\b(?:confirm|clarif|conflict|pause|before I|cannot|can't|need your|which)\b/i.test(o.phaseReply) && !o.activity.some(row => row.action === "issue.document_restored" && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")); - c.add("explicitly-handled-actual-intervening-edit", /\bCasey\b/.test(o.phaseReply) && /\boffline\b|\bwithout (?:a )?network\b/i.test(o.phaseReply) - && /\bprototype\b/i.test(o.phaseReply) && /\b(?:edit|change|revision|version)\b/i.test(o.phaseReply) && (restored || preservedForClarification), - "Only the final phase may acknowledge Casey's actual offline-prototype edit and either explain an intentional historical restore or retain it while asking for clarification."); + // The query names neither the editor nor the new revision: naming either one shows the agent read the actual saved edit. + const namesInterveningEdit = !!changed && (identifiesRevision(o.phaseReply, { latestRevisionId: changed.id, latestRevisionNumber: changed.latestRevisionNumber }) || /\bCasey\b/.test(o.phaseReply)); + c.add("explicitly-handled-actual-intervening-edit", namesInterveningEdit && (restored || preservedForClarification), + "Only the final phase may identify Casey's actual saved edit, by its revision or its editor, and either restore the selected history as a new current revision or leave Casey's edit current."); c.add("all-human-and-original-history-retained", originals.length === 3 && originals.every(old => o.revisions.some(row => row.id === old.id && row.body === old.body)) && !!initialDoc && o.revisions.some(row => row.id === initialDoc.latestRevisionId && row.body === initialDoc.body) && !!checkpoint && !!changed && o.revisions.some(row => row.id === changed.id && row.body === changed.body) && o.activity.some(row => row.action === "issue.document_updated" && row.entityId === entity(o.state, "task").id && row.actorType === "user" && row.actorId === o.state.editorId && row.details?.revisionNumber === changed.latestRevisionNumber), "The exact three original revisions, initial restoration and both saved human edits remain inspectable with the intervening editor's actual audit attribution."); - const falseApproval = o.phaseReply.split(/[.\n;]|,\s*but\s*/).some(part => /\b(?:approved|approval granted|rewound|reset execution|rolled back execution)\b/i.test(part) - && !/\b(?:not|cannot|can't|no|without|doesn't|didn't)\b/i.test(part)); - c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !falseApproval + c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !o.activity.some(row => row.agentId === o.leadId && /approval\.approved|issue\.interaction_accepted/.test(row.action ?? "")), "Restoring document text neither changes executed task state nor invents an approval of historical work."); return c.result(); @@ -61,7 +59,7 @@ export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCas const revision = rows.find(row => row.id === doc.latestRevisionId); if (!revision || revision.createdByUserId !== editor.id || revision.body !== body) throw new Error("Other-person edit lacks actual revision attribution"); return { ...revision, latestRevisionNumber: doc.latestRevisionNumber }; }; - return { ...primary, authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000, + return { ...primary, oracleVersion: "paperclip.chat-catalog.restore-document.saved-edit.v2", authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000, sourceFiles: [...primary.sourceFiles, "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts"], async setup(ctx) { const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Concurrent editor context requires actual authenticated people and private credential redaction"); editor = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); @@ -78,9 +76,9 @@ export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCas const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Read the current plan and report its current revision identifier and full text. Do not change it; wait for my next request."); const read = await pollUntil({ label: "Actual read of the unpredictable other-editor checkpoint", deadlineAt: Date.now() + 150000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" - && o.reply.includes(readMarker) && o.reply.includes(entity(state, "preReadRevision").id) }); + && o.reply.includes(readMarker) && identifiesRevision(o.reply, { latestRevisionId: entity(state, "preReadRevision").id, latestRevisionNumber: entity(state, "preReadRevision").latestRevisionNumber }) }); const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); - state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && row.body.includes(entity(state, "preReadRevision").id) + state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && read.runs.some(run => run.id === row.createdByRunId) && !first.runs.some(run => run.id === row.createdByRunId)); if (!state.readReply) throw new Error("Fresh document read lacks its exact attributed persisted reply"); state.readCheckpoint = { ...read, state: structuredClone(state) }; state.readObservedAt = Date.now();