fix(evals): anchor edit-before-resolution on the first revision that saves the pricing fix

An earlier cosmetic save no longer satisfies the check; the first CEO revision
carrying the $19 team-total fix must precede the resolution.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 19:45:07 -06:00
1 parent 649536ab2c
commit 9d459f36a9
4 files changed
+25 -17

No files matched your search

@@ -2859,7 +2859,7 @@
"chat-scenarios/documents-artifacts.yaml",
"../../packages/shared/src/document-anchors.ts"
],
"oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v2",
"oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v3",
"coverageGaps": [],
"expectedRunCount": 1,
"turnTimeoutMs": 120000,
@@ -37,12 +37,14 @@ export function grade_documents_artifacts(value: unknown) {
c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId
&& unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId,
"Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship.");
const edited = o.activity.filter(row => row.action === "issue.document_updated" && row.agentId === o.leadId)
.sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? ""))[0];
const at = (row: Entity | undefined) => Date.parse(row?.createdAt ?? "");
const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id);
c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "")
const fixed = o.revisions.filter(row => row.createdByAgentId === o.leadId && (!plan || row.documentId === undefined || row.documentId === plan.id)
&& monthlyTeamPricing(row.body ?? "") && (row.body ?? "").includes(String(o.state.supportText))).sort((a, b) => at(a) - at(b))[0];
const audited = o.activity.some(row => row.action === "issue.document_updated" && row.agentId === o.leadId && at(row) <= at(resolved));
c.add("edit-before-resolution", !!fixed && !!resolved && at(fixed) <= at(resolved) && audited
&& !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body),
"Saved revision and independent audit events establish the substantive edit before resolution.");
"The first CEO revision that saves the $19 team-total fix, with its audited edit, precedes the resolution; an unrelated earlier save does not count.");
c.add("task-and-document-access", !!before && sameTask(before, current) && linkedTask(), "Chat links the changed document's task without modifying task execution.");
break;
}
@@ -114,7 +116,7 @@ function resolveFeedback(): ChatCatalogCase {
document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id });
return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
});
return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v2", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v3", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
const o = await observeInternal(ctx, state);
return { ...o, annotations: await ctx.input.api.get<Entity[]>(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) };
} };
@@ -31,7 +31,7 @@ it("preserves unrelated content, exact annotation relationships, saved revision
const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
const o = { scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo", createdAt: "2026-10-27T10:00:00Z" }],
annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
activity: [{ id: "edit", action: "issue.document_updated", agentId: "ceo", createdAt: "2026-10-27T10:00:00Z" },
{ id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-10-27T10:00:01Z" }],
@@ -43,20 +43,26 @@ it("preserves unrelated content, exact annotation relationships, saved revision
});
// Full cohort 9 (master b1ef1f3022): the CEO saved the edit at 00:41:59, resolved at 00:42:03 and saved an identical body
// again at 00:42:06. Activity is listed newest first, so the check read the later save and reported no edit before resolution.
it("establishes edit-before-resolution from the earliest CEO edit, whatever order activity is listed in", () => {
// again at 00:42:06. Activity is listed newest first, so the check read the later save. The graded edit is the first
// saved revision that carries the required $19 team-total fix, not any earlier save.
it("requires the first revision that saves the pricing fix to precede the resolution", () => {
const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" };
const supportText = "Support is provided by the original launch team fixture-42.";
const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
const fixedBody = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
const cosmetic = `Pricing is $19 every month and $19 per user every month.\n\n${supportText}`;
const edit = (id: string, at: string) => ({ id, action: "issue.document_updated", agentId: "ceo", createdAt: at });
const revision = (id: string, body: string, at: string) => ({ id, documentId: "document", body, createdByAgentId: "ceo", createdAt: at });
const resolve = { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-11-12T00:42:03Z" };
const o = (activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
const o = (revisions: Array<ReturnType<typeof revision>>, activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
documents: [{ id: "document", key: "plan", latestRevisionId: revisions.at(-1)!.id, body: revisions.at(-1)!.body }], revisions,
annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
activity, reply: "Fixed.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] });
expect(grade_documents_artifacts(o([edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("edit", "2026-11-12T00:41:59Z")])).passed).toBe(true);
expect(grade_documents_artifacts(o([edit("edit", "2026-11-12T00:41:59Z"), resolve, edit("resave", "2026-11-12T00:42:06Z")])).passed).toBe(true);
expect(grade_documents_artifacts(o([edit("late", "2026-11-12T00:42:06Z"), resolve])).passed).toBe(false);
expect(grade_documents_artifacts(o([resolve])).passed).toBe(false);
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z"), revision("resave", fixedBody, "2026-11-12T00:42:06Z")],
[edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true);
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true);
expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z"), revision("fix", fixedBody, "2026-11-12T00:42:06Z")],
[edit("fix", "2026-11-12T00:42:06Z"), resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false);
expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z")], [resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false);
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve])).passed).toBe(false);
});
@@ -292,7 +292,7 @@ function valid(scenario: string): InternalObservation {
case "documents-artifacts.resolve-feedback": {
o.state.pricingThread = { id: "pricing" }; o.state.siblingThread = { id: "sibling" }; o.state.originalRevisionId = "v1"; o.state.supportText = "Support remains unchanged.";
const body = "$19 monthly total for the entire team, not per user. Support remains unchanged.";
o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo" }];
o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo", createdAt: "2026-10-26T12:00:00Z" }];
o.annotations = [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "doc", originalRevisionId: "v1" }, { id: "sibling", status: "open" }];
o.activity = [{ ...event("issue.document_updated", "task"), createdAt: "2026-10-26T12:00:00Z" },
{ ...event("issue.document_annotation_thread_resolved", "task", { threadId: "pricing" }), createdAt: "2026-10-26T12:00:01Z" }]; break;