test(evals): verify restoration across a real concurrent edit

This commit is contained in:
Dotta committed 2026-10-26 19:24:13 -05:00
1 parent 747eb78c7b
commit 5da3ff9ce0
4 files changed
+150 -6

No files matched your search

@@ -1,3 +1,4 @@
import { withRestoreConcurrency } from "./internal-restore-concurrency.js";
import { projectMarkdownToText, resolveProjectionRange, createDocumentAnchorSelector } from "../../../packages/shared/src/document-anchors.js";
import type { ChatCatalogCase } from "../chat-catalog-contract.js";
import { privateDownloadCases } from "./internal-private-download.js";
@@ -53,7 +54,7 @@ export function grade_documents_artifacts(value: unknown) {
&& revisions.every(original => o.revisions.some(row => row.id === original.id && row.body === original.body)), "All three historical revisions remain unmodified alongside the restoration.");
c.add("restore-path-attribution", !!before && o.activity.some(row => row.action === "issue.document_restored" && row.entityId === before.id
&& row.agentId === o.leadId && row.details?.restoredFromRevisionId === previous?.id), "An independently observed restore audit identifies the exact source revision.");
c.add("no-execution-rewind", !!before && sameTask(before, current) && linkedTask() && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged.");
c.add("no-execution-rewind", !!before && sameTask(before, current) && /\b(?:revision|version)\b/i.test(o.reply), "Chat identifies a document revision and leaves task execution unchanged.");
break;
}
default: c.add("implemented-scenario", false, "No independent oracle is implemented for this scenario.");
@@ -71,7 +72,7 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [
description: `Build a single-page website for ${requirement}, with accessible signup and responsive layout. Verify keyboard navigation and mobile presentation.` });
return { project: p, task: t, requirement };
}),
define("documents-artifacts.restore-document", async ctx => {
withRestoreConcurrency(define("documents-artifacts.restore-document", async ctx => {
const p = await project(ctx); const t = await task(ctx, "Website task", p.id, { description: "The website task's plan has three saved revisions. The previous version before the last edit is the desired content." });
const revisions: Entity[] = [];
for (const body of [`# Plan\nBuild a responsive website for ${ctx.input.nonce}.`, `# Plan\nBuild accessible signup for ${ctx.input.nonce}.\nVerify keyboard operation.`, `# Plan\nReplace signup with an unrelated pricing experiment for ${ctx.input.nonce}.`]) {
@@ -79,7 +80,7 @@ export const documents_artifactsCases: readonly ChatCatalogCase[] = [
revisions.push({ id: saved.latestRevisionId!, body });
}
return { project: p, task: t, originalRevisions: revisions };
}),
})),
];
@@ -0,0 +1,27 @@
import { describe, expect, it } from "vitest";
import { gradeConcurrentRestore } from "./internal-restore-concurrency.js";
import { grade_documents_artifacts } from "./internal-documents-artifacts.js";
import type { Entity, InternalObservation } from "./internal-support.js";
function valid() {
const task = { id: "task", companyId: "company", title: "Website task", status: "backlog" }; const originals = [{ id: "v1", body: "first" }, { id: "v2", body: "second" }, { id: "v3", body: "third" }];
const first: InternalObservation = { scenarioId: "documents-artifacts.restore-document", origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: { task: structuredClone(task), originalRevisions: originals }, tasks: [structuredClone(task)], projects: [], goals: [], comments: [], activity: [{ id: "restore1", action: "issue.document_restored", entityId: "task", agentId: "ceo", details: { restoredFromRevisionId: "v2" } }], documents: [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4", latestRevisionNumber: 4 }], revisions: [...originals, { id: "v4", body: "second", createdByAgentId: "ceo" }], reply: "Restored version 2 as current revision 4.", links: ["/RUN/issues/task"], humanQueries: [], runs: [{ id: "first", companyId: "company", agentId: "ceo", status: "succeeded" }] };
const checkpoint = { id: "v5", body: "second\nHuman checkpoint ORCHID", createdByUserId: "casey", createdAt: "2026-10-26T12:00:00.000Z", latestRevisionNumber: 5 };
const changed = { id: "v6", body: "second\nCasey's offline research prototype edit.", createdByUserId: "casey", createdAt: "2026-10-26T12:00:02.000Z", latestRevisionNumber: 6 };
const read = structuredClone(first); read.documents[0] = { ...read.documents[0]!, body: checkpoint.body, latestRevisionId: "v5", latestRevisionNumber: 5 }; read.revisions.push(checkpoint as Entity); read.reply += " Human checkpoint ORCHID, current revision v5."; read.runs.push({ id: "read", companyId: "company", agentId: "ceo", status: "succeeded" });
return { ...structuredClone(read), state: { ...first.state, firstRestore: first, readCheckpoint: read, readMarker: "Human checkpoint ORCHID", preReadRevision: checkpoint, interveningRevision: changed, editorId: "casey", readObservedAt: 1,
readReply: { id: "read-reply", body: "Human checkpoint ORCHID, v5", authorAgentId: "ceo", createdByRunId: "read", createdAt: "2026-10-26T12:00:01.000Z" } },
documents: [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v7", latestRevisionNumber: 7 }], revisions: [...read.revisions, changed, { id: "v7", body: "second", createdByAgentId: "ceo" }],
activity: [...first.activity, { id: "human-edit", action: "issue.document_updated", entityId: "task", actorType: "user", actorId: "casey", details: { revisionNumber: 6 } }, { id: "restore2", action: "issue.document_restored", entityId: "task", agentId: "ceo", runId: "final", details: { restoredFromRevisionId: "v2" } }] as Entity[],
phaseReply: "I found Casey's newer offline research prototype edit. I restored the selected revision 2 as current revision 7 and kept Casey's change in history.",
runs: [...read.runs, { id: "final", companyId: "company", agentId: "ceo", status: "succeeded" }] };
}
const grade = (o: unknown) => gradeConcurrentRestore(o, grade_documents_artifacts);
describe("real concurrent document restoration calibration", () => {
it("accepts explained intentional restoration retaining the other editor's history", () => expect(grade(valid()).passed).toBe(true));
it("accepts preserving the actual newer edit for truthful clarification", () => { const o = valid(); o.documents[0] = { ...o.documents[0]!, latestRevisionId: "v6", latestRevisionNumber: 6, body: o.state.interveningRevision.body }; o.revisions = o.revisions.filter(row => row.id !== "v7"); o.activity = o.activity.filter(row => row.id !== "restore2"); o.phaseReply = "Casey's newer offline research prototype edit is current. Please confirm whether to replace that change with the selected historical revision?"; expect(grade(o).passed).toBe(true); });
it("rejects crediting a wrong first restoration repaired later", () => { const o = valid(); o.state.firstRestore.documents[0]!.body = "third"; expect(grade(o).passed).toBe(false); });
it("requires actual fresh read/run attribution before the other editor", () => { const o = valid(); o.state.readReply.createdByRunId = "first"; expect(grade(o).passed).toBe(false); const time = valid(); time.state.interveningRevision.createdAt = "2026-10-26T11:59:00.000Z"; expect(grade(time).passed).toBe(false); });
it("rejects wrong-phase-only or invented intervening-edit acknowledgement", () => { const o = valid(); o.reply += o.phaseReply; o.phaseReply = "Restored revision 7."; expect(grade(o).passed).toBe(false); const invented = valid(); invented.phaseReply = "Casey's pricing edit was preserved while I restored revision 7."; expect(grade(invented).passed).toBe(false); });
it("requires retained human revision and final-run restoration audit", () => { const o = valid(); o.revisions = o.revisions.filter(row => row.id !== "v6"); expect(grade(o).passed).toBe(false); const oldRun = valid(); Object.assign(oldRun.activity.find(row => row.id === "restore2")!, { runId: "first" }); expect(grade(oldRun).passed).toBe(false); });
it("rejects retroactive approval or execution rewind", () => { const o = valid(); o.phaseReply += " The old work is approved."; expect(grade(o).passed).toBe(false); const rewind = valid(); rewind.tasks[0]!.status = "done"; expect(grade(rewind).passed).toBe(false); });
});
@@ -0,0 +1,108 @@
import { expect } from "@playwright/test";
import { randomUUID } from "node:crypto";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { pollUntil } from "../api.js";
import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
import { checks, entity, observation, observeInternal, sameTask, hasLink, type Entity, type InternalObservation } from "./internal-support.js";
interface Revision extends Entity { createdByUserId?: string | null }
interface ConcurrentRestore extends InternalObservation { phaseReply: string }
function identifiesRevision(reply: string, doc: Entity) { return !!doc.latestRevisionId && reply.includes(doc.latestRevisionId)
|| Number.isSafeInteger(doc.latestRevisionNumber) && new RegExp(`\\b(?:revision|version|v)\\s*#?${doc.latestRevisionNumber}\\b`, "i").test(reply); }
export function gradeConcurrentRestore(value: unknown, initialGrade: ChatCatalogCase["grade"]) {
const c = checks(); const o = observation(value) as ConcurrentRestore | undefined;
c.add("actual-restoration-and-concurrent-context", !!o && typeof o.phaseReply === "string" && !!o.state.firstRestore && !!o.state.readCheckpoint && !!o.state.interveningRevision,
"The independently graded original restoration, fresh read and actual other-person revision must all be saved."); if (!o || typeof o.phaseReply !== "string") return c.result();
const first = observation(o.state.firstRestore); const read = observation(o.state.readCheckpoint); const initialDoc = first?.documents.find(row => row.key === "plan");
c.add("canonical-restoration-complete-before-variant", !!first && initialGrade(first).passed && !!initialDoc && identifiesRevision(first.reply, initialDoc),
"The unchanged first query restores the exact selected history through its revision path and identifies the resulting current revision before any variant edits.");
const readReply = o.state.readReply as Entity | undefined; const checkpoint = o.state.preReadRevision as Revision | undefined; const changed = o.state.interveningRevision as Revision | undefined;
c.add("fresh-read-before-other-editor-change", !!read && !!checkpoint && !!changed && read.reply.includes(String(o.state.readMarker)) && read.reply.includes(checkpoint.id)
&& read.documents.some(row => row.latestRevisionId === checkpoint.id && row.body === checkpoint.body)
&& read.runs.length === 2 && read.runs.every(run => run.agentId === o.leadId && run.status === "succeeded")
&& !!readReply && readReply.authorAgentId === o.leadId && (readReply.body ?? "").includes(String(o.state.readMarker)) && (readReply.body ?? "").includes(checkpoint.id)
&& read.runs.some(run => run.id === readReply.createdByRunId) && !first?.runs.some(run => run.id === readReply.createdByRunId)
&& o.revisions.some(row => row.id === checkpoint.id && row.body === checkpoint.body && (row as Revision).createdByUserId === o.state.editorId)
&& o.revisions.some(row => row.id === changed.id && row.body === changed.body && (row as Revision).createdByUserId === o.state.editorId)
&& Date.parse(changed.createdAt ?? "") >= Date.parse(readReply.createdAt ?? ""),
"A genuine read reports the unpredictable saved human checkpoint, followed by a different authenticated person's optimistic document edit.");
const originals = (o.state.originalRevisions as Entity[] | undefined) ?? []; const wanted = originals[1]; const current = o.documents.find(row => row.key === "plan");
const earlierRunIds = new Set(read?.runs.map(run => run.id) ?? []); const finalRunIds = new Set(o.runs.filter(run => run.agentId === o.leadId && !earlierRunIds.has(run.id)).map(run => run.id));
const restored = !!current && !!wanted && current.body === wanted.body && current.latestRevisionId !== changed?.id && current.latestRevisionId !== checkpoint?.id
&& o.revisions.some(row => row.id === current.latestRevisionId && row.createdByAgentId === o.leadId && row.body === wanted.body)
&& o.activity.some(row => row.action === "issue.document_restored" && row.entityId === entity(o.state, "task").id && row.agentId === o.leadId
&& row.details?.restoredFromRevisionId === wanted.id && finalRunIds.has((row as Entity & { runId?: string }).runId ?? ""))
&& identifiesRevision(o.phaseReply, current) && /\b(?:restored|restoring)\b/i.test(o.phaseReply);
const preservedForClarification = current?.latestRevisionId === changed?.id && current?.body === changed?.body
&& /\?|\b(?:confirm|clarif|conflict|pause|before I|cannot|can't|need your|which)\b/i.test(o.phaseReply)
&& !o.activity.some(row => row.action === "issue.document_restored" && finalRunIds.has((row as Entity & { runId?: string }).runId ?? ""));
c.add("explicitly-handled-actual-intervening-edit", /\bCasey\b/.test(o.phaseReply) && /\boffline\b|\bwithout (?:a )?network\b/i.test(o.phaseReply)
&& /\bprototype\b/i.test(o.phaseReply) && /\b(?:edit|change|revision|version)\b/i.test(o.phaseReply) && (restored || preservedForClarification),
"Only the final phase may acknowledge Casey's actual offline-prototype edit and either explain an intentional historical restore or retain it while asking for clarification.");
c.add("all-human-and-original-history-retained", originals.length === 3 && originals.every(old => o.revisions.some(row => row.id === old.id && row.body === old.body))
&& !!initialDoc && o.revisions.some(row => row.id === initialDoc.latestRevisionId && row.body === initialDoc.body)
&& !!checkpoint && !!changed && o.revisions.some(row => row.id === changed.id && row.body === changed.body)
&& o.activity.some(row => row.action === "issue.document_updated" && row.entityId === entity(o.state, "task").id && row.actorType === "user" && row.actorId === o.state.editorId
&& row.details?.revisionNumber === changed.latestRevisionNumber), "The exact three original revisions, initial restoration and both saved human edits remain inspectable with the intervening editor's actual audit attribution.");
const falseApproval = o.phaseReply.split(/[.\n;]|,\s*but\s*/).some(part => /\b(?:approved|approval granted|rewound|reset execution|rolled back execution)\b/i.test(part)
&& !/\b(?:not|cannot|can't|no|without|doesn't|didn't)\b/i.test(part));
c.add("no-retroactive-approval-or-execution-rewind", sameTask(entity(o.state, "task"), o.tasks.find(row => row.id === entity(o.state, "task").id)) && !falseApproval
&& !o.activity.some(row => row.agentId === o.leadId && /approval\.approved|issue\.interaction_accepted/.test(row.action ?? "")),
"Restoring document text neither changes executed task state nor invents an approval of historical work.");
return c.result();
}
export function withRestoreConcurrency(primary: ChatCatalogCase): ChatCatalogCase {
let editor: Awaited<ReturnType<typeof createAuthenticatedPerson>> | undefined;
const humanWrite = async (ctx: ChatCatalogContext, state: Record<string, unknown>, body: string, baseRevisionId: string) => {
if (!editor) throw new Error("Missing distinct authenticated editor");
const response = await editor.request(`/api/issues/${entity(state, "task").id}/documents/plan`, "PUT", { title: "Website implementation plan", format: "markdown", body, baseRevisionId });
if (!response.ok) { await response.body?.cancel(); throw new Error("The other editor's actual optimistic document write was not accepted"); }
const doc = await response.json() as Entity; const rows = await ctx.input.api.get<Revision[]>(`/api/issues/${entity(state, "task").id}/documents/plan/revisions`);
const revision = rows.find(row => row.id === doc.latestRevisionId); if (!revision || revision.createdByUserId !== editor.id || revision.body !== body) throw new Error("Other-person edit lacks actual revision attribution");
return { ...revision, latestRevisionNumber: doc.latestRevisionNumber };
};
return { ...primary, authentication: "human-session", coverageGaps: [], expectedRunCount: 3, minimumRunCount: 3, expectedHumanMessageCount: 3, attemptTimeoutMs: 660000,
sourceFiles: [...primary.sourceFiles, "chat-catalog-domains/internal-restore-concurrency.ts", "chat-catalog-domains/identity-auth.ts"],
async setup(ctx) { const browser = ctx.input.page.context().browser(); if (!browser || !ctx.input.privateSecrets) throw new Error("Concurrent editor context requires actual authenticated people and private credential redaction");
editor = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets: ctx.input.privateSecrets, name: "Casey" });
try { await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, editor, ctx.input.privateSecrets, "operator"); return { ...await primary.setup(ctx), editorId: editor.id }; }
catch (error) { await editor.dispose(); editor = undefined; throw error; } },
grade: value => gradeConcurrentRestore(value, primary.grade),
async act(ctx, state) {
const first = await pollUntil({ label: "Independently valid initial canonical restoration", deadlineAt: Date.now() + 150000,
load: () => observeInternal(ctx, state), accept: o => o.runs.length === 1 && o.runs[0]?.status === "succeeded" && ctx.issue()?.conversationState === "waiting" && primary.grade(o).passed
&& o.documents.some(doc => doc.key === "plan" && identifiesRevision(o.reply, doc)) });
state.firstRestore = { ...first, state: structuredClone(state) }; const restored = first.documents.find(doc => doc.key === "plan")!;
const readMarker = `Human checkpoint ${randomUUID()}`; state.readMarker = readMarker;
state.preReadRevision = await humanWrite(ctx, state, `${restored.body}\n\n${readMarker}. This is Casey's saved reading checkpoint.\n`, restored.latestRevisionId!);
const { sendChatMessage } = await import("../chat-flow.js"); await sendChatMessage(ctx.input.page, "Read the current plan and report its current revision identifier and full text. Do not change it; wait for my next request.");
const read = await pollUntil({ label: "Actual read of the unpredictable other-editor checkpoint", deadlineAt: Date.now() + 150000,
load: () => observeInternal(ctx, state), accept: o => o.runs.length === 2 && o.runs.every(run => run.status === "succeeded") && ctx.issue()?.conversationState === "waiting"
&& o.reply.includes(readMarker) && o.reply.includes(entity(state, "preReadRevision").id) });
const sourceComments = await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`);
state.readReply = sourceComments.find(row => row.authorAgentId === ctx.input.fixtures.agent.id && row.body?.includes(readMarker) && row.body.includes(entity(state, "preReadRevision").id)
&& read.runs.some(run => run.id === row.createdByRunId) && !first.runs.some(run => run.id === row.createdByRunId));
if (!state.readReply) throw new Error("Fresh document read lacks its exact attributed persisted reply");
state.readCheckpoint = { ...read, state: structuredClone(state) }; state.readObservedAt = Date.now();
state.interveningRevision = await humanWrite(ctx, state, `${restored.body}\n\nCasey's concurrent edit: the offline research prototype must remain testable. ${randomUUID()}\n`, entity(state, "preReadRevision").id);
await ctx.input.evidence("chat-catalog-real-other-editor-concurrent-plan.json", { firstRestore: state.firstRestore, preReadRevision: state.preReadRevision, readCheckpoint: state.readCheckpoint, readObservedAt: state.readObservedAt, interveningRevision: state.interveningRevision, editorId: state.editorId });
const wanted = (state.originalRevisions as Entity[])[1]!;
await sendChatMessage(ctx.input.page, `Restore the originally selected historical plan revision ${wanted.id} again. If another editor changed the current plan since your read, explicitly explain how you handle that saved edit while retaining revision history.`);
},
async observe(ctx, state): Promise<ConcurrentRestore> { const o = await observeInternal(ctx, state); const prior = observation(state.readCheckpoint); const ids = new Set(prior?.runs.map(run => run.id) ?? []);
const comments = ctx.issue() ? await ctx.input.api.get<Entity[]>(`/api/issues/${ctx.issue()!.id}/comments?order=asc`) : [];
return { ...o, phaseReply: comments.filter(row => row.authorAgentId === o.leadId && !ids.has(row.createdByRunId ?? "") && o.runs.some(run => run.id === row.createdByRunId)).map(row => row.body ?? "").join("\n") }; },
async verifyResult(ctx, state, value) {
const o = value as ConcurrentRestore; const t = entity(state, "task"); const doc = o.documents.find(row => row.key === "plan");
if (!doc?.latestRevisionNumber) throw new Error("Missing actual current document revision"); const source = ctx.input.page.url();
const returned = o.links.find(href => hasLink({ ...o, links: [href] }, "issues", t));
try {
await ctx.input.page.goto(new URL(returned ?? `/${ctx.input.fixtures.company.issuePrefix}/issues/${t.identifier ?? t.id}`, o.origin).href, { waitUntil: "domcontentloaded" });
await expect(ctx.input.page.getByRole("heading", { name: t.title!, exact: true }).first()).toBeVisible();
const expand = ctx.input.page.getByRole("button", { name: "Expand plan document", exact: true }); if (await expand.isVisible()) await expand.click();
await expect(ctx.input.page.getByRole("button", { name: `rev ${doc.latestRevisionNumber}`, exact: true }).first()).toBeVisible();
await ctx.input.evidence("chat-catalog-concurrent-plan-current-revision-opened.json", { taskId: t.id, revisionId: doc.latestRevisionId, revisionNumber: doc.latestRevisionNumber, url: ctx.input.page.url() });
} finally { await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }
},
async cleanup(ctx, state) { try { await primary.cleanup?.(ctx, state); } finally { await editor?.dispose(); editor = undefined; } },
};
}
@@ -278,8 +278,16 @@ function valid(scenario: string): InternalObservation {
}
case "documents-artifacts.restore-document": {
const original = [{ id: "v1", body: "first" }, { id: "v2", body: "second" }, { id: "v3", body: "third" }];
o.state.originalRevisions = original; o.revisions = [...original, { id: "v4", body: "second" }]; o.documents = [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4" }];
o.activity = [event("issue.document_restored", "task", { restoredFromRevisionId: "v2" })]; o.reply = "Restored version 2 as revision 4."; break;
o.state.originalRevisions = original; o.revisions = [...original, { id: "v4", body: "second" }]; o.documents = [{ id: "doc", key: "plan", body: "second", latestRevisionId: "v4", latestRevisionNumber: 4 }];
o.activity = [event("issue.document_restored", "task", { restoredFromRevisionId: "v2" })]; o.reply = "Restored version 2 as revision 4."; o.state.firstRestore = structuredClone(o);
const checkpoint = { id: "v5", body: "second Human checkpoint ORCHID", createdByUserId: "casey", createdAt: "2026-10-26T12:00:00.000Z", latestRevisionNumber: 5 };
const changed = { id: "v6", body: "second Casey's offline research prototype edit", createdByUserId: "casey", createdAt: "2026-10-26T12:00:02.000Z", latestRevisionNumber: 6 };
o.revisions.push(checkpoint); o.documents[0] = { ...o.documents[0], body: checkpoint.body, latestRevisionId: "v5", latestRevisionNumber: 5 }; o.reply += " Human checkpoint ORCHID v5."; o.runs.push({ id: "read", companyId: "company", agentId: "ceo", status: "succeeded" });
o.state.readCheckpoint = structuredClone(o); Object.assign(o.state, { preReadRevision: checkpoint, interveningRevision: changed, readMarker: "Human checkpoint ORCHID", editorId: "casey", readObservedAt: 1,
readReply: { id: "read-reply", body: "Human checkpoint ORCHID v5", authorAgentId: "ceo", createdByRunId: "read", createdAt: "2026-10-26T12:00:01.000Z" } });
o.revisions.push(changed, { id: "v7", body: "second", createdByAgentId: "ceo" }); o.documents[0] = { ...o.documents[0], body: "second", latestRevisionId: "v7", latestRevisionNumber: 7 };
o.activity.push({ id: "human-edit", action: "issue.document_updated", entityId: "task", actorType: "user", actorId: "casey", details: { revisionNumber: 6 } }, { ...event("issue.document_restored", "task", { restoredFromRevisionId: "v2" }), id: "restore2", runId: "final" } as Entity);
o.runs.push({ id: "final", companyId: "company", agentId: "ceo", status: "succeeded" }); Object.assign(o, { phaseReply: "Casey's newer offline research prototype edit is retained in history. I restored the selected revision 2 as current revision 7." }); break;
}
case "activity.company-overview": {
o.tasks[0] = { ...o.tasks[0]!, status: "in_progress" }; o.state.task = { ...o.tasks[0] };
@@ -354,7 +362,7 @@ describe("internal chat scenario independent oracles", () => {
expect(gradeInternal(wrong).passed).toBe(false);
expect(gradeInternal(undefined).passed).toBe(false);
const noLinks = structuredClone(good); noLinks.links = [];
expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive"].includes(definition.scenarioId));
expect(gradeInternal(noLinks).passed).toBe(["goals.align-existing-work", "projects.add-repository", "projects.replace-repository", "tasks.completion-with-gate", "documents-artifacts.private-download", "inbox-search.archive-done", "inbox-search.restore-archive", "documents-artifacts.restore-document"].includes(definition.scenarioId));
});
}
it("never treats an unimplemented scenario as passing", () => expect(gradeInternal(base("inbox-search.archive-partial-failure")).passed).toBe(false));