From ae20a630312bfac4f5219a530682ee4070e0ca82 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 03:19:52 -0600 Subject: [PATCH] test(evals): identify reject-and-revise's draft by structure, not the setup's scope tag The draft is introduction copy; whether it repeats the plan's internal "Acorn research launch " tag is the model's choice, and a correct run left it out. The draft is now a new document separate from the plan, written by the run and linked from the reply; the opened link must show that saved document's first line. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 4 ++-- .../identity-plan-review.test.ts | 3 ++- .../chat-catalog-domains/identity-plan-review.ts | 15 +++++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index e0730b0cfde..ddd8f28fa42 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4268,7 +4268,7 @@ "chat-scenarios/questions-and-approvals.yaml", "../../server/src/services/chat-completion-delivery.ts" ], - "oracleVersion": "paperclip.chat-catalog.plan-review.v3", + "oracleVersion": "paperclip.chat-catalog.plan-review.v4", "expectedRunCount": 3, "minimumRunCount": 1, "turnTimeoutMs": 120000, @@ -4290,7 +4290,7 @@ "chat-scenarios/questions-and-approvals.yaml", "../../server/src/services/chat-completion-delivery.ts" ], - "oracleVersion": "paperclip.chat-catalog.plan-review.v3", + "oracleVersion": "paperclip.chat-catalog.plan-review.v4", "expectedRunCount": 3, "minimumRunCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts index 231bf190d00..b782d285071 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts @@ -44,6 +44,7 @@ describe("versioned approval and revised publication scope", () => { it("allows actual attributed local chat draft without unnecessarily requiring delegation", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.tasks = []; o.runs = o.runs.filter(run => run.agentId === "ceo"); o.documents[0] = { ...o.documents[0]!, issueId: "chat" }; o.revisions[0] = { ...o.revisions[0]!, createdByRunId: "lead-run", createdByAgentId: "ceo" }; o.links = ["#document-plan"]; expect(d.grade(o).passed).toBe(true); o.links = []; expect(d.grade(o).passed).toBe(false); }); it("does not grade the draft's or reply's wording", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); const marker = String(o.state.scopeMarker); o.documents[0] = { ...o.documents[0]!, body: `# Introduction copy\n${marker}\nA notebook for small science teams.` }; o.reply = "Held back the page. Your copy is linked above."; expect(d.grade(o).passed).toBe(true); - o.documents[0] = { ...o.documents[0]!, body: "# Introduction copy without the launch scope" }; expect(d.grade(o).passed).toBe(false); }); + o.documents[0] = { ...o.documents[0]!, body: "# Research Notes\nA calmer place for your team's experiments." }; expect(d.grade(o).passed).toBe(true); + o.documents[0] = { ...o.documents[0]!, id: "plan" }; expect(d.grade(o).passed).toBe(false); }); it("rejects an otherwise real draft revision written by an unrelated run", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts index d694430961c..eb5fe4073ff 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts @@ -40,12 +40,12 @@ function gradePlanReviewOutcome(value: unknown) { } else if (o.scenarioId === "questions-and-approvals.reject-and-revise") { c.add("publication-decision-declined", ["rejected", "superseded"].includes(current?.status ?? "") && !o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === original?.id) && o.state.rejectedReceiptVisible === true, "The old publication request remains visibly declined or superseded and is never accepted."); - c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => (row.body ?? "").includes(String(o.state.scopeMarker)) + c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => row.id !== plan?.id && !!row.body?.trim() && !!row.latestRevisionId && row.latestRevisionId !== plan?.latestRevisionId && o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId && [o.leadId, worker?.id].includes(run.agentId))) && (row.issueId === o.state.sourceId ? o.links.some(link => link.includes(`#document-${encodeURIComponent(row.key!)}`)) : o.tasks.some(task => task.id === row.issueId && hasLink(o, "issues", task)))) && o.state.sourceRetainsConversation === true && o.state.noPublishedWorkProduct === true, - "The revised scope produces an actual attributed new local document carrying the setup's scope marker, linked from the reply, without a published delivery record. Its wording and the reply's wording are not graded."); + "The revised scope produces an actual attributed new local document, separate from the declined plan and linked from the reply, without a published delivery record. Its wording (including whether it repeats the setup's internal scope tag) and the reply's wording are not graded."); } else c.add("known-plan-workflow", false, "No oracle exists for this workflow."); return c.result(); } @@ -143,7 +143,7 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a supersedeOnUserComment: false, target: { type: "issue_document", issueId: source.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } })); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { sourceId: source.id, personId: session.user.id, worker, plan, original, scopeMarker }; }, gradePlanReview, { - sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v3", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, + sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v4", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, async act(ctx, state) { const originalDeadline = ctx.observationDeadlineAt; @@ -186,13 +186,16 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a async verifyResult(ctx, state, value) { const o = identityObservation(value); if (scenarioId.endsWith("review-plan") || o?.tasks.length) { await verifyInternalResult(ctx, state, value); return; } - const doc = o?.documents.find(row => o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && revision.createdByRunId) - && (row.body ?? "").includes(String(state.scopeMarker))); + const plan = state.plan as Entity | undefined; + const doc = o?.documents.find(row => row.id !== plan?.id && !!row.body?.trim() && o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && revision.createdByRunId) + && o.links.some(link => link.includes(`#document-${encodeURIComponent(row.key!)}`))); if (!doc) throw new Error("Actual local review draft is missing"); const href = await ctx.input.page.locator('[id^="comment-"] a[href]').evaluateAll(links => links.map(link => link.getAttribute("href")!)); const reviewLink = href.find(link => link.includes(`#document-${encodeURIComponent(doc.key!)}`)); if (!reviewLink) throw new Error("Actual draft review link missing from reply"); await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(reviewLink)}]`).first().click(); - await expect(ctx.input.page.getByText(String(state.scopeMarker), { exact: false }).first()).toBeVisible(); + // The opened view must show the saved draft itself: its first line of text. + const firstLine = doc.body!.split("\n").map(line => line.replace(/^[#>*\-\s]+|[*_`]+/g, "").trim()).find(Boolean)!; + await expect(ctx.input.page.getByText(firstLine, { exact: false }).first()).toBeVisible(); }, }));