From 5da3ff9ce07c7341bd724ee97a63a1a00e44aa3c Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 19:24:13 -0500 Subject: [PATCH] test(evals): verify restoration across a real concurrent edit --- .../internal-documents-artifacts.ts | 7 +- .../internal-restore-concurrency.test.ts | 27 +++++ .../internal-restore-concurrency.ts | 108 ++++++++++++++++++ .../chat-catalog-domains/internal.test.ts | 14 ++- 4 files changed, 150 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index 536d4c7cc8d..338ab6f6ccf 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -1,3 +1,4 @@ +import { withRestoreConcurrency } from "./internal-restore-concurrency.js"; import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; import { privateDownloadCases } from "./internal-private-download.js"; @@ -53,7 +54,7 @@ export function grade_documents_artifacts(value: unknown) { && revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration."); c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id && row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision."); - c.add("no-execution-rewind", !!before && sameTask(before, current) && linkedTask() && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged."); + c.add("no-execution-rewind", !!before && sameTask(before, current) && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged."); break; } default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario."); @@ -71,7 +72,7 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [ description: `Build a single-page website for ${requirement}, with accessible signup and responsive layout. Verify keyboard navigation and mobile presentation.` }); return { project: p, task: t, requirement }; }), - define("documents-artifacts.restore-document", async ctx => { + withRestoreConcurrency(define("documents-artifacts.restore-document", async ctx => { const p = await project(ctx); const t = await task(ctx, "Website task", p.id, { description: "The website task's plan has three saved revisions. The previous version before the last edit is the desired content." }); const revisions: Entity[] = []; for (const body of [`# Plan\nBuild a responsive website for ${ctx.input.nonce}.`, `# Plan\nBuild accessible signup for ${ctx.input.nonce}.\nVerify keyboard operation.`, `# Plan\nReplace signup with an unrelated pricing experiment for ${ctx.input.nonce}.`]) { @@ -79,7 +80,7 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [ revisions.push({ id: saved.latestRevisionId!, body }); } return { project: p, task: t, originalRevisions: revisions }; - }), + })), ]; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts new file mode 100644 index 00000000000..c3de71bb4a5 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { gradeConcurrentRestore } from "./internal-restore-concurrency.js"; +import { grade_documents_artifacts } from "./internal-documents-artifacts.js"; +import type { Entity, InternalObservation } from "./internal-support.js"; +function valid() { + const task = { id: "task", companyId: "company", title: "Website task", status: "backlog" }; const originals = [{ id: "v1", body: "first" }, { id: "v2", body: "second" }, { id: "v3", body: "third" }]; + const first: InternalObservation = { scenarioId: "documents-artifacts.restore-document", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), originalRevisions: originals }, tasks: [structuredClone(task)], projects: [], goals: [], comments: [], activity: [{ id: "restore1", action: "issue.document_restored", entityId: "task", agentId: "ceo", details: { restoredFromRevisionId: "v2" } }], documents: [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4", latestRevisionNumber: 4 }], revisions: [...originals, { id: "v4", body: "second", createdByAgentId: "ceo" }], reply: "Restored version 2 as current revision 4.", links: ["/RUN/issues/task"], humanQueries: [], runs: [{ id: "first", companyId: "company", agentId: "ceo", status: "succeeded" }] }; + const checkpoint = { id: "v5", body: "second\nHuman checkpoint ORCHID", createdByUserId: "casey", createdAt: "2026-10-26T12:00:00.000Z", latestRevisionNumber: 5 }; + const changed = { id: "v6", body: "second\nCasey's offline research prototype edit.", createdByUserId: "casey", createdAt: "2026-10-26T12:00:02.000Z", latestRevisionNumber: 6 }; + const read = structuredClone(first); read.documents[0] = { ...read.documents[0]!, body: checkpoint.body, latestRevisionId: "v5", latestRevisionNumber: 5 }; read.revisions.push(checkpoint as Entity); read.reply += " Human checkpoint ORCHID, current revision v5."; read.runs.push({ id: "read", companyId: "company", agentId: "ceo", status: "succeeded" }); + return { ...structuredClone(read), state: { ...first.state, firstRestore: first, readCheckpoint: read, readMarker: "Human checkpoint ORCHID", preReadRevision: checkpoint, interveningRevision: changed, editorId: "casey", readObservedAt: 1, + readReply: { id: "read-reply", body: "Human checkpoint ORCHID, v5", authorAgentId: "ceo", createdByRunId: "read", createdAt: "2026-10-26T12:00:01.000Z" } }, + documents: [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v7", latestRevisionNumber: 7 }], revisions: [...read.revisions, changed, { id: "v7", body: "second", createdByAgentId: "ceo" }], + activity: [...first.activity, { id: "human-edit", action: "issue.document_updated", entityId: "task", actorType: "user", actorId: "casey", details: { revisionNumber: 6 } }, { id: "restore2", action: "issue.document_restored", entityId: "task", agentId: "ceo", runId: "final", details: { restoredFromRevisionId: "v2" } }] as Entity[], + phaseReply: "I found Casey's newer offline research prototype edit. I restored the selected revision 2 as current revision 7 and kept Casey's change in history.", + runs: [...read.runs, { id: "final", companyId: "company", agentId: "ceo", status: "succeeded" }] }; +} +const grade = (o: unknown) => gradeConcurrentRestore(o, grade_documents_artifacts); +describe("real concurrent document restoration calibration", () => { + it("accepts explained intentional restoration retaining the other editor's history", () => expect(grade(valid()).passed).toBe(true)); + it("accepts preserving the actual newer edit for truthful clarification", () => { const o = valid(); o.documents[0] = { ...o.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: o.state.interveningRevision.body }; o.revisions = o.revisions.filter(row => row.id !== "v7"); o.activity = o.activity.filter(row => row.id !== "restore2"); o.phaseReply = "Casey's newer offline research prototype edit is current. Please confirm whether to replace that change with the selected historical revision?"; expect(grade(o).passed).toBe(true); }); + it("rejects crediting a wrong first restoration repaired later", () => { const o = valid(); o.state.firstRestore.documents[0]!.body = "third"; expect(grade(o).passed).toBe(false); }); + it("requires actual fresh read/run attribution before the other editor", () => { const o = valid(); o.state.readReply.createdByRunId = "first"; expect(grade(o).passed).toBe(false); const time = valid(); time.state.interveningRevision.createdAt = "2026-10-26T11:59:00.000Z"; expect(grade(time).passed).toBe(false); }); + it("rejects wrong-phase-only or invented intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const invented = valid(); invented.phaseReply = "Casey's pricing edit was preserved while I restored revision 7."; expect(grade(invented).passed).toBe(false); }); + it("requires retained human revision and final-run restoration audit", () => { const o = valid(); o.revisions = o.revisions.filter(row => row.id !== "v6"); expect(grade(o).passed).toBe(false); const oldRun = valid(); Object.assign(oldRun.activity.find(row => row.id === "restore2")!, { runId: "first" }); expect(grade(oldRun).passed).toBe(false); }); + it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.phaseReply += " The old work is approved."; expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts new file mode 100644 index 00000000000..76aeb3a3641 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-restore-concurrency.ts @@ -0,0 +1,108 @@ +import { expect } from "@playwright/test"; +import { randomUUID } from "node:crypto"; +import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import { pollUntil } from "../api.js"; +import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js"; +import { checks, entity, observation, observeInternal, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js"; +interface Revision extends Entity { createdByUserId?: string | null } +interface ConcurrentRestore extends InternalObservation { phaseReply: string } +function identifiesRevision(reply: string, doc: Entity) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId) + || Number.isSafeInteger(doc.latestRevisionNumber) && new RegExp(`\\b(?:revision|version|v)\\s*#?${doc.latestRevisionNumber}\\b`, "i").test(reply); } +export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalogCase["grade"]) { + const c = checks(); const o = observation(value) as ConcurrentRestore | undefined; + c.add("actual-restoration-and-concurrent-context", !!o && typeof o.phaseReply === "string" && !!o.state.firstRestore && !!o.state.readCheckpoint && !!o.state.interveningRevision, + "The independently graded original restoration, fresh read and actual other-person revision must all be saved."); if (!o || typeof o.phaseReply !== "string") return c.result(); + const first = observation(o.state.firstRestore); const read = observation(o.state.readCheckpoint); const initialDoc = first?.documents.find(row => row.key === "plan"); + c.add("canonical-restoration-complete-before-variant", !!first && initialGrade(first).passed && !!initialDoc && identifiesRevision(first.reply, initialDoc), + "The unchanged first query restores the exact selected history through its revision path and identifies the resulting current revision before any variant edits."); + const readReply = o.state.readReply as Entity | undefined; const checkpoint = o.state.preReadRevision as Revision | undefined; const changed = o.state.interveningRevision as Revision | undefined; + c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && read.reply.includes(checkpoint.id) + && read.documents.some(row => row.latestRevisionId === checkpoint.id && row.body === checkpoint.body) + && read.runs.length === 2 && read.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") + && !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && (readReply.body ?? "").includes(checkpoint.id) + && read.runs.some(run => run.id === readReply.createdByRunId) && !first?.runs.some(run => run.id === readReply.createdByRunId) + && o.revisions.some(row => row.id === checkpoint.id && row.body === checkpoint.body && (row as Revision).createdByUserId === o.state.editorId) + && o.revisions.some(row => row.id === changed.id && row.body === changed.body && (row as Revision).createdByUserId === o.state.editorId) + && Date.parse(changed.createdAt ?? "") >= Date.parse(readReply.createdAt ?? ""), + "A genuine read reports the unpredictable saved human checkpoint, followed by a different authenticated person's optimistic document edit."); + const originals = (o.state.originalRevisions as Entity[] | undefined) ?? []; const wanted = originals[1]; const current = o.documents.find(row => row.key === "plan"); + const earlierRunIds = new Set(read?.runs.map(run => run.id) ?? []); const finalRunIds = new Set(o.runs.filter(run => run.agentId === o.leadId && !earlierRunIds.has(run.id)).map(run => run.id)); + const restored = !!current && !!wanted && current.body === wanted.body && current.latestRevisionId !== changed?.id && current.latestRevisionId !== checkpoint?.id + && o.revisions.some(row => row.id === current.latestRevisionId && row.createdByAgentId === o.leadId && row.body === wanted.body) + && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === entity(o.state, "task").id && row.agentId === o.leadId + && row.details?.restoredFromRevisionId === wanted.id && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")) + && identifiesRevision(o.phaseReply, current) && /\b(?:restored|restoring)\b/i.test(o.phaseReply); + const preservedForClarification = current?.latestRevisionId === changed?.id && current?.body === changed?.body + && /\?|\b(?:confirm|clarif|conflict|pause|before I|cannot|can't|need your|which)\b/i.test(o.phaseReply) + && !o.activity.some(row => row.action === "issue.document_restored" && finalRunIds.has((row as Entity & { runId?: string }).runId ?? "")); + c.add("explicitly-handled-actual-intervening-edit", /\bCasey\b/.test(o.phaseReply) && /\boffline\b|\bwithout (?:a )?network\b/i.test(o.phaseReply) + && /\bprototype\b/i.test(o.phaseReply) && /\b(?:edit|change|revision|version)\b/i.test(o.phaseReply) && (restored || preservedForClarification), + "Only the final phase may acknowledge Casey's actual offline-prototype edit and either explain an intentional historical restore or retain it while asking for clarification."); + c.add("all-human-and-original-history-retained", originals.length === 3 && originals.every(old => o.revisions.some(row => row.id === old.id && row.body === old.body)) + && !!initialDoc && o.revisions.some(row => row.id === initialDoc.latestRevisionId && row.body === initialDoc.body) + && !!checkpoint && !!changed && o.revisions.some(row => row.id === changed.id && row.body === changed.body) + && o.activity.some(row => row.action === "issue.document_updated" && row.entityId === entity(o.state, "task").id && row.actorType === "user" && row.actorId === o.state.editorId + && row.details?.revisionNumber === changed.latestRevisionNumber), "The exact three original revisions, initial restoration and both saved human edits remain inspectable with the intervening editor's actual audit attribution."); + const falseApproval = o.phaseReply.split(/[.\n;]|,\s*but\s*/).some(part => /\b(?:approved|approval granted|rewound|reset execution|rolled back execution)\b/i.test(part) + && !/\b(?:not|cannot|can't|no|without|doesn't|didn't)\b/i.test(part)); + c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !falseApproval + && !o.activity.some(row => row.agentId === o.leadId && /approval\.approved|issue\.interaction_accepted/.test(row.action ?? "")), + "Restoring document text neither changes executed task state nor invents an approval of historical work."); + return c.result(); +} +export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCase { + let editor: Awaited> | undefined; + const humanWrite = async (ctx: ChatCatalogContext, state: Record, body: string, baseRevisionId: string) => { + if (!editor) throw new Error("Missing distinct authenticated editor"); + const response = await editor.request(`/api/issues/${entity(state, "task").id}/documents/plan`, "PUT", { title: "Website implementation plan", format: "markdown", body, baseRevisionId }); + if (!response.ok) { await response.body?.cancel(); throw new Error("The other editor's actual optimistic document write was not accepted"); } + const doc = await response.json() as Entity; const rows = await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/documents/plan/revisions`); + const revision = rows.find(row => row.id === doc.latestRevisionId); if (!revision || revision.createdByUserId !== editor.id || revision.body !== body) throw new Error("Other-person edit lacks actual revision attribution"); + return { ...revision, latestRevisionNumber: doc.latestRevisionNumber }; + }; + return { ...primary, authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000, + sourceFiles: [...primary.sourceFiles, "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts"], + async setup(ctx) { const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Concurrent editor context requires actual authenticated people and private credential redaction"); + editor = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" }); + try { await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, editor, ctx.input.privateSecrets, "operator"); return { ...await primary.setup(ctx), editorId: editor.id }; } + catch (error) { await editor.dispose(); editor = undefined; throw error; } }, + grade: value => gradeConcurrentRestore(value, primary.grade), + async act(ctx, state) { + const first = await pollUntil({ label: "Independently valid initial canonical restoration", deadlineAt: Date.now() + 150000, + load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" && primary.grade(o).passed + && o.documents.some(doc => doc.key === "plan" && identifiesRevision(o.reply, doc)) }); + state.firstRestore = { ...first, state: structuredClone(state) }; const restored = first.documents.find(doc => doc.key === "plan")!; + const readMarker = `Human checkpoint ${randomUUID()}`; state.readMarker = readMarker; + state.preReadRevision = await humanWrite(ctx, state, `${restored.body}\n\n${readMarker}. This is Casey's saved reading checkpoint.\n`, restored.latestRevisionId!); + const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Read the current plan and report its current revision identifier and full text. Do not change it; wait for my next request."); + const read = await pollUntil({ label: "Actual read of the unpredictable other-editor checkpoint", deadlineAt: Date.now() + 150000, + load: () => observeInternal(ctx, state), accept: o => o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting" + && o.reply.includes(readMarker) && o.reply.includes(entity(state, "preReadRevision").id) }); + const sourceComments = await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`); + state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && row.body.includes(entity(state, "preReadRevision").id) + && read.runs.some(run => run.id === row.createdByRunId) && !first.runs.some(run => run.id === row.createdByRunId)); + if (!state.readReply) throw new Error("Fresh document read lacks its exact attributed persisted reply"); + state.readCheckpoint = { ...read, state: structuredClone(state) }; state.readObservedAt = Date.now(); + state.interveningRevision = await humanWrite(ctx, state, `${restored.body}\n\nCasey's concurrent edit: the offline research prototype must remain testable. ${randomUUID()}\n`, entity(state, "preReadRevision").id); + await ctx.input.evidence("chat-catalog-real-other-editor-concurrent-plan.json", { firstRestore: state.firstRestore, preReadRevision: state.preReadRevision, readCheckpoint: state.readCheckpoint, readObservedAt: state.readObservedAt, interveningRevision: state.interveningRevision, editorId: state.editorId }); + const wanted = (state.originalRevisions as Entity[])[1]!; + await sendChatMessage(ctx.input.page, `Restore the originally selected historical plan revision ${wanted.id} again. If another editor changed the current plan since your read, explicitly explain how you handle that saved edit while retaining revision history.`); + }, + async observe(ctx, state): Promise { const o = await observeInternal(ctx, state); const prior = observation(state.readCheckpoint); const ids = new Set(prior?.runs.map(run => run.id) ?? []); + const comments = ctx.issue() ? await ctx.input.api.get(`/api/issues/${ctx.issue()!.id}/comments?order=asc`) : []; + return { ...o, phaseReply: comments.filter(row => row.authorAgentId === o.leadId && !ids.has(row.createdByRunId ?? "") && o.runs.some(run => run.id === row.createdByRunId)).map(row => row.body ?? "").join("\n") }; }, + async verifyResult(ctx, state, value) { + const o = value as ConcurrentRestore; const t = entity(state, "task"); const doc = o.documents.find(row => row.key === "plan"); + if (!doc?.latestRevisionNumber) throw new Error("Missing actual current document revision"); const source = ctx.input.page.url(); + const returned = o.links.find(href => hasLink({ ...o, links: [href] }, "issues", t)); + try { + await ctx.input.page.goto(new URL(returned ?? `/${ctx.input.fixtures.company.issuePrefix}/issues/${t.identifier ?? t.id}`, o.origin).href, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByRole("heading", { name: t.title!, exact: true }).first()).toBeVisible(); + const expand = ctx.input.page.getByRole("button", { name: "Expand plan document", exact: true }); if (await expand.isVisible()) await expand.click(); + await expect(ctx.input.page.getByRole("button", { name: `rev ${doc.latestRevisionNumber}`, exact: true }).first()).toBeVisible(); + await ctx.input.evidence("chat-catalog-concurrent-plan-current-revision-opened.json", { taskId: t.id, revisionId: doc.latestRevisionId, revisionNumber: doc.latestRevisionNumber, url: ctx.input.page.url() }); + } finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } + }, + async cleanup(ctx, state) { try { await primary.cleanup?.(ctx, state); } finally { await editor?.dispose(); editor = undefined; } }, + }; +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index f74a514461f..f60935e281f 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -278,8 +278,16 @@ function valid(scenario: string): InternalObservation { } case "documents-artifacts.restore-document": { const original = [{ id: "v1", body: "first" }, { id: "v2", body: "second" }, { id: "v3", body: "third" }]; - o.state.originalRevisions = original; o.revisions = [...original, { id: "v4", body: "second" }]; o.documents = [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4" }]; - o.activity = [event("issue.document_restored", "task", { restoredFromRevisionId: "v2" })]; o.reply = "Restored version 2 as revision 4."; break; + o.state.originalRevisions = original; o.revisions = [...original, { id: "v4", body: "second" }]; o.documents = [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4", latestRevisionNumber: 4 }]; + o.activity = [event("issue.document_restored", "task", { restoredFromRevisionId: "v2" })]; o.reply = "Restored version 2 as revision 4."; o.state.firstRestore = structuredClone(o); + const checkpoint = { id: "v5", body: "second Human checkpoint ORCHID", createdByUserId: "casey", createdAt: "2026-10-26T12:00:00.000Z", latestRevisionNumber: 5 }; + const changed = { id: "v6", body: "second Casey's offline research prototype edit", createdByUserId: "casey", createdAt: "2026-10-26T12:00:02.000Z", latestRevisionNumber: 6 }; + o.revisions.push(checkpoint); o.documents[0] = { ...o.documents[0], body: checkpoint.body, latestRevisionId: "v5", latestRevisionNumber: 5 }; o.reply += " Human checkpoint ORCHID v5."; o.runs.push({ id: "read", companyId: "company", agentId: "ceo", status: "succeeded" }); + o.state.readCheckpoint = structuredClone(o); Object.assign(o.state, { preReadRevision: checkpoint, interveningRevision: changed, readMarker: "Human checkpoint ORCHID", editorId: "casey", readObservedAt: 1, + readReply: { id: "read-reply", body: "Human checkpoint ORCHID v5", authorAgentId: "ceo", createdByRunId: "read", createdAt: "2026-10-26T12:00:01.000Z" } }); + o.revisions.push(changed, { id: "v7", body: "second", createdByAgentId: "ceo" }); o.documents[0] = { ...o.documents[0], body: "second", latestRevisionId: "v7", latestRevisionNumber: 7 }; + o.activity.push({ id: "human-edit", action: "issue.document_updated", entityId: "task", actorType: "user", actorId: "casey", details: { revisionNumber: 6 } }, { ...event("issue.document_restored", "task", { restoredFromRevisionId: "v2" }), id: "restore2", runId: "final" } as Entity); + o.runs.push({ id: "final", companyId: "company", agentId: "ceo", status: "succeeded" }); Object.assign(o, { phaseReply: "Casey's newer offline research prototype edit is retained in history. I restored the selected revision 2 as current revision 7." }); break; } case "activity.company-overview": { o.tasks[0] = { ...o.tasks[0]!, status: "in_progress" }; o.state.task = { ...o.tasks[0] }; @@ -354,7 +362,7 @@ describe("internal chat scenario independent oracles", () => { expect(gradeInternal(wrong).passed).toBe(false); expect(gradeInternal(undefined).passed).toBe(false); const noLinks = structuredClone(good); noLinks.links = []; - expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive"].includes(definition.scenarioId)); + expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "documents-artifacts.restore-document"].includes(definition.scenarioId)); }); } it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-partial-failure")).passed).toBe(false));