From 9d459f36a987f83beeeece43912398f0b2340948 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 19:45:07 -0600 Subject: [PATCH] fix(evals): anchor edit-before-resolution on the first revision that saves the pricing fix An earlier cosmetic save no longer satisfies the check; the first CEO revision carrying the $19 team-total fix must precede the resolution. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-documents-artifacts.ts | 12 +++++---- .../internal-pricing-oracle.test.ts | 26 ++++++++++++------- .../chat-catalog-domains/internal.test.ts | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 461ba165a13..b6929ed11ac 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -2859,7 +2859,7 @@ "chat-scenarios/documents-artifacts.yaml", "../../packages/shared/src/document-anchors.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v2", + "oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts index f8b4734b380..666b3eb3a2a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-documents-artifacts.ts @@ -37,12 +37,14 @@ export function grade_documents_artifacts(value: unknown) { c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId && unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId, "Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship."); - const edited = o.activity.filter(row => row.action === "issue.document_updated" && row.agentId === o.leadId) - .sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? ""))[0]; + const at = (row: Entity | undefined) => Date.parse(row?.createdAt ?? ""); const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id); - c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "") + const fixed = o.revisions.filter(row => row.createdByAgentId === o.leadId && (!plan || row.documentId === undefined || row.documentId === plan.id) + && monthlyTeamPricing(row.body ?? "") && (row.body ?? "").includes(String(o.state.supportText))).sort((a, b) => at(a) - at(b))[0]; + const audited = o.activity.some(row => row.action === "issue.document_updated" && row.agentId === o.leadId && at(row) <= at(resolved)); + c.add("edit-before-resolution", !!fixed && !!resolved && at(fixed) <= at(resolved) && audited && !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body), - "Saved revision and independent audit events establish the substantive edit before resolution."); + "The first CEO revision that saves the $19 team-total fix, with its audited edit, precedes the resolution; an unrelated earlier save does not count."); c.add("task-and-document-access", !!before && sameTask(before, current) && linkedTask(), "Chat links the changed document's task without modifying task execution."); break; } @@ -114,7 +116,7 @@ function resolveFeedback(): ChatCatalogCase { document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id }); return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment }; }); - return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v2", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) { + return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v3", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) { const o = await observeInternal(ctx, state); return { ...o, annotations: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) }; } }; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts index 1f3ed11a85a..dc965e349b0 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-pricing-oracle.test.ts @@ -31,7 +31,7 @@ it("preserves unrelated content, exact annotation relationships, saved revision const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`; const o = { scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo", state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [], - documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }], + documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo", createdAt: "2026-10-27T10:00:00Z" }], annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }], activity: [{ id: "edit", action: "issue.document_updated", agentId: "ceo", createdAt: "2026-10-27T10:00:00Z" }, { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-10-27T10:00:01Z" }], @@ -43,20 +43,26 @@ it("preserves unrelated content, exact annotation relationships, saved revision }); // Full cohort 9 (master b1ef1f3022): the CEO saved the edit at 00:41:59, resolved at 00:42:03 and saved an identical body -// again at 00:42:06. Activity is listed newest first, so the check read the later save and reported no edit before resolution. -it("establishes edit-before-resolution from the earliest CEO edit, whatever order activity is listed in", () => { +// again at 00:42:06. Activity is listed newest first, so the check read the later save. The graded edit is the first +// saved revision that carries the required $19 team-total fix, not any earlier save. +it("requires the first revision that saves the pricing fix to precede the resolution", () => { const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" }; const supportText = "Support is provided by the original launch team fixture-42."; - const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`; + const fixedBody = `$19 monthly total for the entire team, not per user.\n\n${supportText}`; + const cosmetic = `Pricing is $19 every month and $19 per user every month.\n\n${supportText}`; const edit = (id: string, at: string) => ({ id, action: "issue.document_updated", agentId: "ceo", createdAt: at }); + const revision = (id: string, body: string, at: string) => ({ id, documentId: "document", body, createdByAgentId: "ceo", createdAt: at }); const resolve = { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-11-12T00:42:03Z" }; - const o = (activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo", + const o = (revisions: Array>, activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo", state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [], - documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }], + documents: [{ id: "document", key: "plan", latestRevisionId: revisions.at(-1)!.id, body: revisions.at(-1)!.body }], revisions, annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }], activity, reply: "Fixed.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] }); - expect(grade_documents_artifacts(o([edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("edit", "2026-11-12T00:41:59Z")])).passed).toBe(true); - expect(grade_documents_artifacts(o([edit("edit", "2026-11-12T00:41:59Z"), resolve, edit("resave", "2026-11-12T00:42:06Z")])).passed).toBe(true); - expect(grade_documents_artifacts(o([edit("late", "2026-11-12T00:42:06Z"), resolve])).passed).toBe(false); - expect(grade_documents_artifacts(o([resolve])).passed).toBe(false); + expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z"), revision("resave", fixedBody, "2026-11-12T00:42:06Z")], + [edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true); + expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true); + expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z"), revision("fix", fixedBody, "2026-11-12T00:42:06Z")], + [edit("fix", "2026-11-12T00:42:06Z"), resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false); + expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z")], [resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false); + expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve])).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/internal.test.ts b/tests/runner-e2e/chat-catalog-domains/internal.test.ts index a531984d1c4..ac6ca1bc497 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal.test.ts @@ -292,7 +292,7 @@ function valid(scenario: string): InternalObservation { case "documents-artifacts.resolve-feedback": { o.state.pricingThread = { id: "pricing" }; o.state.siblingThread = { id: "sibling" }; o.state.originalRevisionId = "v1"; o.state.supportText = "Support remains unchanged."; const body = "$19 monthly total for the entire team, not per user. Support remains unchanged."; - o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo" }]; + o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo", createdAt: "2026-10-26T12:00:00Z" }]; o.annotations = [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "doc", originalRevisionId: "v1" }, { id: "sibling", status: "open" }]; o.activity = [{ ...event("issue.document_updated", "task"), createdAt: "2026-10-26T12:00:00Z" }, { ...event("issue.document_annotation_thread_resolved", "task", { threadId: "pricing" }), createdAt: "2026-10-26T12:00:01Z" }]; break;