diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 6e1d095274f..01c1a72a3d8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2845,7 +2845,7 @@ "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.restore-document.saved-edit.v2", "coverageGaps": [], "expectedRunCount": 3, "expectedHumanMessageCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index 82af53bfd46..c36c847daf8 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -1,4 +1,4 @@ -import { withRestoreConcurrency } from "./internal-restore-concurrency.js"; +import { withRestoreConcurrency, identifiesRevision } from "./internal-restore-concurrency.js"; import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { privateDownloadCases } from "./internal-private-download.js"; @@ -54,7 +54,7 @@ export function grade_documents_artifacts(value: unknown) { && revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration."); c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision."); - c.add("no-execution-rewind", !!before && sameTask(before, current) && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged."); + c.add("no-execution-rewind", !!before && sameTask(before, current) && !!plan && identifiesRevision(o.reply, plan), "Chat identifies the resulting current document revision and leaves task execution unchanged."); break; } default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario."); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts index c3de71bb4a5..3c40eed2f46 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts @@ -21,7 +21,13 @@ describe("real concurrent document restoration calibration", () => { it("accepts preserving the actual newer edit for truthful clarification", () => { const o = valid(); o.documents[0] = { ...o.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: o.state.interveningRevision.body }; o.revisions = o.revisions.filter(row => row.id !== "v7"); o.activity = o.activity.filter(row => row.id !== "restore2"); o.phaseReply = "Casey's newer offline research prototype edit is current. Please confirm whether to replace that change with the selected historical revision?"; expect(grade(o).passed).toBe(true); }); it("rejects crediting a wrong first restoration repaired later", () => { const o = valid(); o.state.firstRestore.documents[0]!.body = "third"; expect(grade(o).passed).toBe(false); }); it("requires actual fresh read/run attribution before the other editor", () => { const o = valid(); o.state.readReply.createdByRunId = "first"; expect(grade(o).passed).toBe(false); const time = valid(); time.state.interveningRevision.createdAt = "2026-10-26T11:59:00.000Z"; expect(grade(time).passed).toBe(false); }); - it("rejects wrong-phase-only or invented intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const invented = valid(); invented.phaseReply = "Casey's pricing edit was preserved while I restored revision 7."; expect(grade(invented).passed).toBe(false); }); + it("rejects wrong-phase-only or unread intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const unread = valid(); unread.phaseReply = "If someone else edited the plan, that change stays in history. I restored revision 2 as current revision 7."; expect(grade(unread).passed).toBe(false); }); + it("accepts reworded handling that names the saved edit by revision instead of editor or content", () => { const o = valid(); o.phaseReply = "A newer edit (revision 6) landed after my read. It stays in history; the selected text is now current as v7."; expect(grade(o).passed).toBe(true); + const asked = valid(); asked.documents[0] = { ...asked.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: asked.state.interveningRevision.body }; asked.revisions = asked.revisions.filter(row => row.id !== "v7"); asked.activity = asked.activity.filter(row => row.id !== "restore2"); asked.phaseReply = "Casey saved a change after my read, so I left it in place. Tell me to go ahead and I will restore the selected text over it."; expect(grade(asked).passed).toBe(true); }); + it("rejects a final restore that names the edit but leaves the wrong text or no current revision", () => { const wrong = valid(); wrong.documents[0]!.body = "third"; wrong.revisions.find(row => row.id === "v7")!.body = "third"; expect(grade(wrong).passed).toBe(false); + const unnamed = valid(); unnamed.phaseReply = "I found Casey's newer edit and restored the selected plan, keeping history."; expect(grade(unnamed).passed).toBe(false); }); it("requires retained human revision and final-run restoration audit", () => { const o = valid(); o.revisions = o.revisions.filter(row => row.id !== "v6"); expect(grade(o).passed).toBe(false); const oldRun = valid(); Object.assign(oldRun.activity.find(row => row.id === "restore2")!, { runId: "first" }); expect(grade(oldRun).passed).toBe(false); }); - it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.phaseReply += " The old work is approved."; expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); }); + it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.activity.push({ id: "approve", action: "approval.approved", entityId: "task", agentId: "ceo" }); expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); }); + it("accepts a read reply that identifies the checkpoint by revision number", () => { const o = valid(); o.state.readCheckpoint.reply = "Human checkpoint ORCHID, current revision 5."; o.state.readReply.body = "Human checkpoint ORCHID, revision 5"; expect(grade(o).passed).toBe(true); + const stale = valid(); stale.state.readCheckpoint.reply = "Current revision 4 (latest): second"; expect(grade(stale).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts index 2d2b8f50161..f1c88af0adc 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts @@ -7,7 +7,7 @@ import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity- import { checks, entity, observation, observeInternal, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js"; interface Revision extends Entity { createdByUserId?: string | null } interface ConcurrentRestore extends InternalObservation { phaseReply: string } -function identifiesRevision(reply: string, doc: Entity) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId) +export function identifiesRevision(reply: string, doc: Pick) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId) || Number.isSafeInteger(doc.latestRevisionNumber) && new RegExp(`\\b(?:revision|version|v)\\s*#?${doc.latestRevisionNumber}\\b`, "i").test(reply); } export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalogCase["grade"]) { const c = checks(); const o = observation(value) as ConcurrentRestore | undefined; @@ -17,10 +17,10 @@ export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalog c.add("canonical-restoration-complete-before-variant", !!first && initialGrade(first).passed && !!initialDoc && identifiesRevision(first.reply, initialDoc), "The unchanged first query restores the exact selected history through its revision path and identifies the resulting current revision before any variant edits."); const readReply = o.state.readReply as Entity | undefined; const checkpoint = o.state.preReadRevision as Revision | undefined; const changed = o.state.interveningRevision as Revision | undefined; - c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && read.reply.includes(checkpoint.id) + c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && identifiesRevision(read.reply, { latestRevisionId: checkpoint.id, latestRevisionNumber: checkpoint.latestRevisionNumber }) && read.documents.some(row => row.latestRevisionId === checkpoint.id && row.body === checkpoint.body) && read.runs.length === 2 && read.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") - && !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && (readReply.body ?? "").includes(checkpoint.id) + && !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && read.runs.some(run => run.id === readReply.createdByRunId) && !first?.runs.some(run => run.id === readReply.createdByRunId) && o.revisions.some(row => row.id === checkpoint.id && row.body === checkpoint.body && (row as Revision).createdByUserId === o.state.editorId) && o.revisions.some(row => row.id === changed.id && row.body === changed.body && (row as Revision).createdByUserId === o.state.editorId) @@ -32,21 +32,19 @@ export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalog && o.revisions.some(row => row.id === current.latestRevisionId && row.createdByAgentId === o.leadId && row.body === wanted.body) && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === entity(o.state, "task").id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === wanted.id && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")) - && identifiesRevision(o.phaseReply, current) && /\b(?:restored|restoring)\b/i.test(o.phaseReply); + && identifiesRevision(o.phaseReply, current); const preservedForClarification = current?.latestRevisionId === changed?.id && current?.body === changed?.body - && /\?|\b(?:confirm|clarif|conflict|pause|before I|cannot|can't|need your|which)\b/i.test(o.phaseReply) && !o.activity.some(row => row.action === "issue.document_restored" && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")); - c.add("explicitly-handled-actual-intervening-edit", /\bCasey\b/.test(o.phaseReply) && /\boffline\b|\bwithout (?:a )?network\b/i.test(o.phaseReply) - && /\bprototype\b/i.test(o.phaseReply) && /\b(?:edit|change|revision|version)\b/i.test(o.phaseReply) && (restored || preservedForClarification), - "Only the final phase may acknowledge Casey's actual offline-prototype edit and either explain an intentional historical restore or retain it while asking for clarification."); + // The query names neither the editor nor the new revision: naming either one shows the agent read the actual saved edit. + const namesInterveningEdit = !!changed && (identifiesRevision(o.phaseReply, { latestRevisionId: changed.id, latestRevisionNumber: changed.latestRevisionNumber }) || /\bCasey\b/.test(o.phaseReply)); + c.add("explicitly-handled-actual-intervening-edit", namesInterveningEdit && (restored || preservedForClarification), + "Only the final phase may identify Casey's actual saved edit, by its revision or its editor, and either restore the selected history as a new current revision or leave Casey's edit current."); c.add("all-human-and-original-history-retained", originals.length === 3 && originals.every(old => o.revisions.some(row => row.id === old.id && row.body === old.body)) && !!initialDoc && o.revisions.some(row => row.id === initialDoc.latestRevisionId && row.body === initialDoc.body) && !!checkpoint && !!changed && o.revisions.some(row => row.id === changed.id && row.body === changed.body) && o.activity.some(row => row.action === "issue.document_updated" && row.entityId === entity(o.state, "task").id && row.actorType === "user" && row.actorId === o.state.editorId && row.details?.revisionNumber === changed.latestRevisionNumber), "The exact three original revisions, initial restoration and both saved human edits remain inspectable with the intervening editor's actual audit attribution."); - const falseApproval = o.phaseReply.split(/[.\n;]|,\s*but\s*/).some(part => /\b(?:approved|approval granted|rewound|reset execution|rolled back execution)\b/i.test(part) - && !/\b(?:not|cannot|can't|no|without|doesn't|didn't)\b/i.test(part)); - c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !falseApproval + c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !o.activity.some(row => row.agentId === o.leadId && /approval\.approved|issue\.interaction_accepted/.test(row.action ?? "")), "Restoring document text neither changes executed task state nor invents an approval of historical work."); return c.result(); @@ -61,7 +59,7 @@ export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCas const revision = rows.find(row => row.id === doc.latestRevisionId); if (!revision || revision.createdByUserId !== editor.id || revision.body !== body) throw new Error("Other-person edit lacks actual revision attribution"); return { ...revision, latestRevisionNumber: doc.latestRevisionNumber }; }; - return { ...primary, authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000, + return { ...primary, oracleVersion: "paperclip.chat-catalog.restore-document.saved-edit.v2", authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000, sourceFiles: [...primary.sourceFiles, "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts"], async setup(ctx) { const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Concurrent editor context requires actual authenticated people and private credential redaction"); editor = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); @@ -78,9 +76,9 @@ export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCas const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Read the current plan and report its current revision identifier and full text. Do not change it; wait for my next request."); const read = await pollUntil({ label: "Actual read of the unpredictable other-editor checkpoint", deadlineAt: Date.now() + 150000, load: () => observeInternal(ctx, state), accept: o => o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" - && o.reply.includes(readMarker) && o.reply.includes(entity(state, "preReadRevision").id) }); + && o.reply.includes(readMarker) && identifiesRevision(o.reply, { latestRevisionId: entity(state, "preReadRevision").id, latestRevisionNumber: entity(state, "preReadRevision").latestRevisionNumber }) }); const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); - state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && row.body.includes(entity(state, "preReadRevision").id) + state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && read.runs.some(run => run.id === row.createdByRunId) && !first.runs.some(run => run.id === row.createdByRunId)); if (!state.readReply) throw new Error("Fresh document read lacks its exact attributed persisted reply"); state.readCheckpoint = { ...read, state: structuredClone(state) }; state.readObservedAt = Date.now();