From e689d13800cd80c36d75bac50b65f1cd691400ef Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 02:53:51 -0600 Subject: [PATCH] test(evals): accept the requested-changes receipt for a rejected plan decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both the classic receipt ("Requested changes to Plan v1") and the Coordinator receipt ("Changes requested · v1") render a rejected plan-target confirmation as requested changes; the saved status stays rejected and is graded separately. Draft and reply keyword checks are dropped; the new attributed document must carry the setup's scope marker. Co-Authored-By: Claude Opus 5.5 --- tests/runner-e2e/chat-catalog-descriptors.json | 4 ++-- .../identity-plan-review.test.ts | 3 +++ .../chat-catalog-domains/identity-plan-review.ts | 15 ++++++++++----- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index d1b17347e23..4e6d7e16edb 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -4268,7 +4268,7 @@ "chat-scenarios/questions-and-approvals.yaml", "../../server/src/services/chat-completion-delivery.ts" ], - "oracleVersion": "paperclip.chat-catalog.plan-review.v2", + "oracleVersion": "paperclip.chat-catalog.plan-review.v3", "expectedRunCount": 3, "minimumRunCount": 1, "turnTimeoutMs": 120000, @@ -4290,7 +4290,7 @@ "chat-scenarios/questions-and-approvals.yaml", "../../server/src/services/chat-completion-delivery.ts" ], - "oracleVersion": "paperclip.chat-catalog.plan-review.v2", + "oracleVersion": "paperclip.chat-catalog.plan-review.v3", "expectedRunCount": 3, "minimumRunCount": 2, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts index 6c47f897ff0..231bf190d00 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.test.ts @@ -42,5 +42,8 @@ describe("versioned approval and revised publication scope", () => { it("rejects reusing approval on the publication after the draft request", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.interactions[0]!.status = "accepted"; expect(d.grade(o).passed).toBe(false); }); it("rejects a published work product while keeping a real local draft", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.state.noPublishedWorkProduct = false; expect(d.grade(o).passed).toBe(false); }); it("allows actual attributed local chat draft without unnecessarily requiring delegation", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.tasks = []; o.runs = o.runs.filter(run => run.agentId === "ceo"); o.documents[0] = { ...o.documents[0]!, issueId: "chat" }; o.revisions[0] = { ...o.revisions[0]!, createdByRunId: "lead-run", createdByAgentId: "ceo" }; o.links = ["#document-plan"]; expect(d.grade(o).passed).toBe(true); o.links = []; expect(d.grade(o).passed).toBe(false); }); + it("does not grade the draft's or reply's wording", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); const marker = String(o.state.scopeMarker); + o.documents[0] = { ...o.documents[0]!, body: `# Introduction copy\n${marker}\nA notebook for small science teams.` }; o.reply = "Held back the page. Your copy is linked above."; expect(d.grade(o).passed).toBe(true); + o.documents[0] = { ...o.documents[0]!, body: "# Introduction copy without the launch scope" }; expect(d.grade(o).passed).toBe(false); }); it("rejects an otherwise real draft revision written by an unrelated run", () => { const d = planReviewIdentityCases[1]!; const o = valid(d.scenarioId); o.revisions[0]!.createdByRunId = "unrelated-run"; expect(d.grade(o).passed).toBe(false); }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts index ded8eac1085..d694430961c 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-plan-review.ts @@ -40,12 +40,12 @@ function gradePlanReviewOutcome(value: unknown) { } else if (o.scenarioId === "questions-and-approvals.reject-and-revise") { c.add("publication-decision-declined", ["rejected", "superseded"].includes(current?.status ?? "") && !o.activity.some(row => row.action === "issue.thread_interaction_accepted" && row.details?.interactionId === original?.id) && o.state.rejectedReceiptVisible === true, "The old publication request remains visibly declined or superseded and is never accepted."); - c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => (row.body ?? "").includes(String(o.state.scopeMarker)) && /draft|review/i.test(row.body ?? "") + c.add("real-local-review-draft", o.documents.length > 0 && o.documents.some(row => (row.body ?? "").includes(String(o.state.scopeMarker)) && !!row.latestRevisionId && row.latestRevisionId !== plan?.latestRevisionId && o.revisions.some(revision => revision.id === row.latestRevisionId && revision.documentId === row.id && o.runs.some(run => run.id === revision.createdByRunId && run.agentId === revision.createdByAgentId && [o.leadId, worker?.id].includes(run.agentId))) && (row.issueId === o.state.sourceId ? o.links.some(link => link.includes(`#document-${encodeURIComponent(row.key!)}`)) : o.tasks.some(task => task.id === row.issueId && hasLink(o, "issues", task)))) - && o.state.sourceRetainsConversation === true && o.state.noPublishedWorkProduct === true && !/\b(?:I|we)(?:'ve| have)?\s+(?:published|deployed)\b/i.test(o.reply), - "The revised scope produces an actual attributed local review document and linked task, without a published delivery record or claimed publication."); + && o.state.sourceRetainsConversation === true && o.state.noPublishedWorkProduct === true, + "The revised scope produces an actual attributed new local document carrying the setup's scope marker, linked from the reply, without a published delivery record. Its wording and the reply's wording are not graded."); } else c.add("known-plan-workflow", false, "No oracle exists for this workflow."); return c.result(); } @@ -143,7 +143,7 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a supersedeOnUserComment: false, target: { type: "issue_document", issueId: source.id, documentId: plan.id, key: "plan", revisionId: plan.latestRevisionId, revisionNumber: plan.latestRevisionNumber } } })); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); return { sourceId: source.id, personId: session.user.id, worker, plan, original, scopeMarker }; }, gradePlanReview, { - sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v2", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, + sourceFiles: [...sources, "../../server/src/services/chat-completion-delivery.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.plan-review.v3", expectedRunCount: 3, minimumRunCount: scenarioId.endsWith("review-plan") ? 2 : 1, attemptTimeoutMs: 600_000, runPolicy: { kind: "source-settled-with-active-work", maximumActiveWorkerRuns: 1 }, async act(ctx, state) { const originalDeadline = ctx.observationDeadlineAt; @@ -156,7 +156,12 @@ export const planReviewIdentityCases: readonly ChatCatalogCase[] = ["questions-a reject: value => value.observation.runs.length > 3 || value.observation.runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Plan decision contains failed, cancelled or excess native work" : undefined }); const original = entity(state, "original"); if (scenarioId.endsWith("reject-and-revise")) { - const card = ctx.input.page.locator(`#interaction-${original.id}`); await expect(card).toContainText(/Declined|Superseded/i); state.rejectedReceiptVisible = true; return; + // A rejected plan-target confirmation renders as requested changes in + // both the classic receipt ("Requested changes to Plan v1") and the + // Coordinator receipt ("Changes requested · v1"); the saved status is + // graded separately as rejected or superseded. + const card = ctx.input.page.locator(`#interaction-${original.id}`); + await expect(card).toContainText(/Declined|Superseded|Changes requested|Requested changes/i); await expect(card).not.toContainText(/Approved/i); state.rejectedReceiptVisible = true; return; } // A separate additional review opens before its actual public document edit. // The first canonical approval's source plan is never rewritten by this check.