mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
fix(evals): anchor edit-before-resolution on the first revision that saves the pricing fix
An earlier cosmetic save no longer satisfies the check; the first CEO revision carrying the $19 team-total fix must precede the resolution. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
649536ab2c
commit
9d459f36a9
4 files changed
+25
-17
No files matched your search
@@ -2859,7 +2859,7 @@
|
||||
"chat-scenarios/documents-artifacts.yaml",
|
||||
"../../packages/shared/src/document-anchors.ts"
|
||||
],
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v2",
|
||||
"oracleVersion": "paperclip.chat-catalog.internal.resolve-feedback.v3",
|
||||
"coverageGaps": [],
|
||||
"expectedRunCount": 1,
|
||||
"turnTimeoutMs": 120000,
|
||||
|
||||
@@ -37,12 +37,14 @@ export function grade_documents_artifacts(value: unknown) {
|
||||
c.add("correct-thread-resolved", annotation?.status === "resolved" && annotation.resolvedByAgentId === o.leadId
|
||||
&& unrelated?.status === "open" && annotation.documentId === plan?.id && annotation.originalRevisionId === o.state.originalRevisionId,
|
||||
"Only the anchored pricing annotation is resolved by the CEO, preserving its original document and revision relationship.");
|
||||
const edited = o.activity.filter(row => row.action === "issue.document_updated" && row.agentId === o.leadId)
|
||||
.sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? ""))[0];
|
||||
const at = (row: Entity | undefined) => Date.parse(row?.createdAt ?? "");
|
||||
const resolved = o.activity.find(row => row.action === "issue.document_annotation_thread_resolved" && row.agentId === o.leadId && row.details?.threadId === target.id);
|
||||
c.add("edit-before-resolution", !!edited && !!resolved && Date.parse(edited.createdAt ?? "") <= Date.parse(resolved.createdAt ?? "")
|
||||
const fixed = o.revisions.filter(row => row.createdByAgentId === o.leadId && (!plan || row.documentId === undefined || row.documentId === plan.id)
|
||||
&& monthlyTeamPricing(row.body ?? "") && (row.body ?? "").includes(String(o.state.supportText))).sort((a, b) => at(a) - at(b))[0];
|
||||
const audited = o.activity.some(row => row.action === "issue.document_updated" && row.agentId === o.leadId && at(row) <= at(resolved));
|
||||
c.add("edit-before-resolution", !!fixed && !!resolved && at(fixed) <= at(resolved) && audited
|
||||
&& !!plan && o.revisions.some(row => row.id === plan.latestRevisionId && row.createdByAgentId === o.leadId && row.body === plan.body),
|
||||
"Saved revision and independent audit events establish the substantive edit before resolution.");
|
||||
"The first CEO revision that saves the $19 team-total fix, with its audited edit, precedes the resolution; an unrelated earlier save does not count.");
|
||||
c.add("task-and-document-access", !!before && sameTask(before, current) && linkedTask(), "Chat links the changed document's task without modifying task execution.");
|
||||
break;
|
||||
}
|
||||
@@ -114,7 +116,7 @@ function resolveFeedback(): ChatCatalogCase {
|
||||
document: { id: doc.id, key: "plan", latestRevisionId: doc.latestRevisionId! }, annotationId: pricingThread.id });
|
||||
return { project: p, task: t, pricingThread, siblingThread, supportText, originalRevisionId: doc.latestRevisionId, selectedSource, factualContextAttachment: selectedSource.factualContextAttachment };
|
||||
});
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v2", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
|
||||
return { ...base, oracleVersion: "paperclip.chat-catalog.internal.resolve-feedback.v3", sourceFiles: [...base.sourceFiles, "../../packages/shared/src/document-anchors.ts"], async observe(ctx, state) {
|
||||
const o = await observeInternal(ctx, state);
|
||||
return { ...o, annotations: await ctx.input.api.get<Entity[]>(`/api/issues/${entity(state, "task").id}/documents/plan/annotations?status=all&includeComments=true`) };
|
||||
} };
|
||||
|
||||
@@ -31,7 +31,7 @@ it("preserves unrelated content, exact annotation relationships, saved revision
|
||||
const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
|
||||
const o = { scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
|
||||
state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
|
||||
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
|
||||
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo", createdAt: "2026-10-27T10:00:00Z" }],
|
||||
annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
|
||||
activity: [{ id: "edit", action: "issue.document_updated", agentId: "ceo", createdAt: "2026-10-27T10:00:00Z" },
|
||||
{ id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-10-27T10:00:01Z" }],
|
||||
@@ -43,20 +43,26 @@ it("preserves unrelated content, exact annotation relationships, saved revision
|
||||
});
|
||||
|
||||
// Full cohort 9 (master b1ef1f3022): the CEO saved the edit at 00:41:59, resolved at 00:42:03 and saved an identical body
|
||||
// again at 00:42:06. Activity is listed newest first, so the check read the later save and reported no edit before resolution.
|
||||
it("establishes edit-before-resolution from the earliest CEO edit, whatever order activity is listed in", () => {
|
||||
// again at 00:42:06. Activity is listed newest first, so the check read the later save. The graded edit is the first
|
||||
// saved revision that carries the required $19 team-total fix, not any earlier save.
|
||||
it("requires the first revision that saves the pricing fix to precede the resolution", () => {
|
||||
const task = { id: "task", companyId: "company", title: "Website task", status: "backlog", projectId: "project" };
|
||||
const supportText = "Support is provided by the original launch team fixture-42.";
|
||||
const body = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
|
||||
const fixedBody = `$19 monthly total for the entire team, not per user.\n\n${supportText}`;
|
||||
const cosmetic = `Pricing is $19 every month and $19 per user every month.\n\n${supportText}`;
|
||||
const edit = (id: string, at: string) => ({ id, action: "issue.document_updated", agentId: "ceo", createdAt: at });
|
||||
const revision = (id: string, body: string, at: string) => ({ id, documentId: "document", body, createdByAgentId: "ceo", createdAt: at });
|
||||
const resolve = { id: "resolve", action: "issue.document_annotation_thread_resolved", agentId: "ceo", details: { threadId: "pricing" }, createdAt: "2026-11-12T00:42:03Z" };
|
||||
const o = (activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
|
||||
const o = (revisions: Array<ReturnType<typeof revision>>, activity: unknown[]) => ({ scenarioId: "documents-artifacts.resolve-feedback", companyId: "company", origin: "https://fixture.test", leadId: "ceo",
|
||||
state: { task, pricingThread: { id: "pricing" }, siblingThread: { id: "sibling" }, supportText, originalRevisionId: "original" }, tasks: [task], projects: [], goals: [], comments: [],
|
||||
documents: [{ id: "document", key: "plan", latestRevisionId: "revision", body }], revisions: [{ id: "revision", body, createdByAgentId: "ceo" }],
|
||||
documents: [{ id: "document", key: "plan", latestRevisionId: revisions.at(-1)!.id, body: revisions.at(-1)!.body }], revisions,
|
||||
annotations: [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "document", originalRevisionId: "original" }, { id: "sibling", status: "open" }],
|
||||
activity, reply: "Fixed.", links: ["/RUN/issues/task"], humanQueries: [], runs: [] });
|
||||
expect(grade_documents_artifacts(o([edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("edit", "2026-11-12T00:41:59Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([edit("edit", "2026-11-12T00:41:59Z"), resolve, edit("resave", "2026-11-12T00:42:06Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([edit("late", "2026-11-12T00:42:06Z"), resolve])).passed).toBe(false);
|
||||
expect(grade_documents_artifacts(o([resolve])).passed).toBe(false);
|
||||
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z"), revision("resave", fixedBody, "2026-11-12T00:42:06Z")],
|
||||
[edit("resave", "2026-11-12T00:42:06Z"), resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve, edit("fix", "2026-11-12T00:41:59Z")])).passed).toBe(true);
|
||||
expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z"), revision("fix", fixedBody, "2026-11-12T00:42:06Z")],
|
||||
[edit("fix", "2026-11-12T00:42:06Z"), resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false);
|
||||
expect(grade_documents_artifacts(o([revision("cosmetic", cosmetic, "2026-11-12T00:41:59Z")], [resolve, edit("cosmetic", "2026-11-12T00:41:59Z")])).passed).toBe(false);
|
||||
expect(grade_documents_artifacts(o([revision("fix", fixedBody, "2026-11-12T00:41:59Z")], [resolve])).passed).toBe(false);
|
||||
});
|
||||
@@ -292,7 +292,7 @@ function valid(scenario: string): InternalObservation {
|
||||
case "documents-artifacts.resolve-feedback": {
|
||||
o.state.pricingThread = { id: "pricing" }; o.state.siblingThread = { id: "sibling" }; o.state.originalRevisionId = "v1"; o.state.supportText = "Support remains unchanged.";
|
||||
const body = "$19 monthly total for the entire team, not per user. Support remains unchanged.";
|
||||
o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo" }];
|
||||
o.documents = [{ id: "doc", key: "plan", body, latestRevisionId: "v2" }]; o.revisions = [{ id: "v2", body, createdByAgentId: "ceo", createdAt: "2026-10-26T12:00:00Z" }];
|
||||
o.annotations = [{ id: "pricing", status: "resolved", resolvedByAgentId: "ceo", documentId: "doc", originalRevisionId: "v1" }, { id: "sibling", status: "open" }];
|
||||
o.activity = [{ ...event("issue.document_updated", "task"), createdAt: "2026-10-26T12:00:00Z" },
|
||||
{ ...event("issue.document_annotation_thread_resolved", "task", { threadId: "pricing" }), createdAt: "2026-10-26T12:00:01Z" }]; break;
|
||||
|
||||
Reference in new issue
Block a user