test(evals): identify reject-and-revise's draft by structure, not the setup's scope tag

The draft is introduction copy; whether it repeats the plan's internal
"Acorn research launch <nonce>" tag is the model's choice, and a correct run
left it out. The draft is now a new document separate from the plan, written
by the run and linked from the reply; the opened link must show that saved
document's first line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -4268,7 +4268,7 @@
       "chat-scenarios/questions-and-approvals.yaml",
       "../../server/src/services/chat-completion-delivery.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.plan-review.v3",
+    "oracleVersion": "paperclip.chat-catalog.plan-review.v4",
     "expectedRunCount": 3,
     "minimumRunCount": 1,
     "turnTimeoutMs": 120000,
@@ -4290,7 +4290,7 @@
       "chat-scenarios/questions-and-approvals.yaml",
       "../../server/src/services/chat-completion-delivery.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.plan-review.v3",
+    "oracleVersion": "paperclip.chat-catalog.plan-review.v4",
     "expectedRunCount": 3,
     "minimumRunCount": 2,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts

@@ -44,6 +44,7 @@ describe("versioned approval and revised publication scope", () => {
   it("allows actual attributed local chat draft without unnecessarily requiring delegation", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.tasks = []; o.runs = o.runs.filter(run => run.agentId === "ceo"); o.documents[0] = { ...o.documents[0]!, issueId: "chat" }; o.revisions[0] = { ...o.revisions[0]!, createdByRunId: "lead-run", createdByAgentId: "ceo" }; o.links = ["#document-plan"]; expect(d.grade(o).passed).toBe(true); o.links = []; expect(d.grade(o).passed).toBe(false); });
   it("does not grade the draft's or reply's wording", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); const marker = String(o.state.scopeMarker);
     o.documents[0] = { ...o.documents[0]!, body: `# Introduction copy\n${marker}\nA notebook for small science teams.` }; o.reply = "Held back the page. Your copy is linked above."; expect(d.grade(o).passed).toBe(true);
-    o.documents[0] = { ...o.documents[0]!, body: "# Introduction copy without the launch scope" }; expect(d.grade(o).passed).toBe(false); });
+    o.documents[0] = { ...o.documents[0]!, body: "# Research Notes\nA calmer place for your team's experiments." }; expect(d.grade(o).passed).toBe(true);
+    o.documents[0] = { ...o.documents[0]!, id: "plan" }; expect(d.grade(o).passed).toBe(false); });
   it("rejects an otherwise real draft revision written by an unrelated run", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false); });
 });

tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts

@@ -40,12 +40,12 @@ function gradePlanReviewOutcome(value: unknown) {
   } else if (o.scenarioId === "questions-and-approvals.reject-and-revise") {
     c.add("publication-decision-declined", ["rejected", "superseded"].includes(current?.status ?? "") && !o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === original?.id)
       && o.state.rejectedReceiptVisible === true, "The old publication request remains visibly declined or superseded and is never accepted.");
-    c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => (row.body ?? "").includes(String(o.state.scopeMarker))
+    c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => row.id !== plan?.id && !!row.body?.trim()
       && !!row.latestRevisionId && row.latestRevisionId !== plan?.latestRevisionId && o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id
         && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId && [o.leadId, worker?.id].includes(run.agentId)))
       && (row.issueId === o.state.sourceId ? o.links.some(link => link.includes(`#document-${encodeURIComponent(row.key!)}`)) : o.tasks.some(task => task.id === row.issueId && hasLink(o, "issues", task))))
       && o.state.sourceRetainsConversation === true && o.state.noPublishedWorkProduct === true,
-      "The revised scope produces an actual attributed new local document carrying the setup's scope marker, linked from the reply, without a published delivery record. Its wording and the reply's wording are not graded.");
+      "The revised scope produces an actual attributed new local document, separate from the declined plan and linked from the reply, without a published delivery record. Its wording (including whether it repeats the setup's internal scope tag) and the reply's wording are not graded.");
   } else c.add("known-plan-workflow", false, "No oracle exists for this workflow.");
   return c.result();
 }
@@ -143,7 +143,7 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a
       supersedeOnUserComment: false, target: { type: "issue_document", issueId: source.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } }));
   await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { sourceId: source.id, personId: session.user.id, worker, plan, original, scopeMarker };
 }, gradePlanReview, {
-  sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v3", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000,
+  sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v4", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000,
   runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 },
   async act(ctx, state) {
     const originalDeadline = ctx.observationDeadlineAt;
@@ -186,13 +186,16 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a
   async verifyResult(ctx, state, value) {
     const o = identityObservation(value);
     if (scenarioId.endsWith("review-plan") || o?.tasks.length) { await verifyInternalResult(ctx, state, value); return; }
-    const doc = o?.documents.find(row => o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && revision.createdByRunId)
-      && (row.body ?? "").includes(String(state.scopeMarker)));
+    const plan = state.plan as Entity | undefined;
+    const doc = o?.documents.find(row => row.id !== plan?.id && !!row.body?.trim() && o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && revision.createdByRunId)
+      && o.links.some(link => link.includes(`#document-${encodeURIComponent(row.key!)}`)));
     if (!doc) throw new Error("Actual local review draft is missing");
     const href = await ctx.input.page.locator('[id^="comment-"] a[href]').evaluateAll(links => links.map(link => link.getAttribute("href")!));
     const reviewLink = href.find(link => link.includes(`#document-${encodeURIComponent(doc.key!)}`));
     if (!reviewLink) throw new Error("Actual draft review link missing from reply");
     await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(reviewLink)}]`).first().click();
-    await expect(ctx.input.page.getByText(String(state.scopeMarker), { exact: false }).first()).toBeVisible();
+    // The opened view must show the saved draft itself: its first line of text.
+    const firstLine = doc.body!.split("\n").map(line => line.replace(/^[#>*\-\s]+|[*_`]+/g, "").trim()).find(Boolean)!;
+    await expect(ctx.input.page.getByText(firstLine, { exact: false }).first()).toBeVisible();
   },
 }));