From 574e14692aaadcb8f500c3bb26e7195cc40cff18 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 16:20:21 -0600 Subject: [PATCH] fix(evals): grade unnamed initial timeline attempts Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-timeline-narrative.test.ts | 4 ++-- .../internal-timeline-narrative.ts | 6 +++--- .../internal-timeline-native-milestone.test.ts | 16 +++++++++++++++- .../internal-work-timeline.ts | 2 +- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index bf9e1e9f779..cf25c60e17a 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v6", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v7", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts index f9fdd366ca7..b28b9c04217 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts @@ -20,8 +20,8 @@ it.each([ ])("retains consequential actor and outcome requirements: %s", reply => { expect(grade(reply)).toBe(false); }); -it("does not carry a section owner through an unrelated paragraph", () => { - expect(grade(account.replace("The verification", "\n\nThe verification"))).toBe(false); +it("does not infer the initial worker over an explicit contrary actor in another paragraph", () => { + expect(grade(account.replace("The verification command failed", "\n\nRiley ran verification and it failed"))).toBe(false); }); it("does not give a heading credit for another actor's command", () => { diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index 84178d37800..459fff2efd5 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -85,7 +85,7 @@ export function truthfulTimelineAccount(input: { resolvedPronouns.add(at + lead.length); return lead + owner; }); - const paragraph = { text, owner, start: offset, end: offset + text.length }; + const paragraph = { text, owner, namedWorkerHeading: owners.length > 0, start: offset, end: offset + text.length }; offset = paragraph.end + 2; return paragraph; }); @@ -93,7 +93,7 @@ export function truthfulTimelineAccount(input: { // Search milestone clauses, not the first occurrence of a worker or outcome. // A summary or heading may mention the repair before the ordered account. const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))] - .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, paragraphStart: paragraph?.start }; }); + .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, namedWorkerHeading: paragraph?.namedWorkerHeading, paragraphStart: paragraph?.start }; }); // A command noun can have ordinary modifiers, but never absorb a new event, // actor, negation or clause boundary while connecting this pause to failure. const commandBoundaries = [ @@ -108,7 +108,7 @@ export function truthfulTimelineAccount(input: { const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi; const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i"); - const failures = clauses.flatMap(({ text, at, owner }) => (mentions(text, firstWorker) || owner === firstWorker && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker)) + const failures = clauses.flatMap(({ text, at, owner, namedWorkerHeading }) => (mentions(text, firstWorker) || (owner === firstWorker || !namedWorkerHeading) && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker)) && new RegExp(`\\b${commandSubject}\\b`, "i").test(text) ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []); const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index }))); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 91278a4bc7d..8dab949e288 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -236,7 +236,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v6"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v7"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); @@ -383,3 +383,17 @@ it.each(["another agent", "a bot"])("retains explicit nonhuman handoff attributi o.reply = o.reply.replace("Then a human reassigned the task to Riley.", `The task was reassigned to Riley by ${actor}.`); expect(gradeWorkTimeline(o).passed).toBe(false); }); + +const unnamedAttempt = "The verification command failed with exit code 1 and the task paused. You reassigned the task to Riley. Riley repaired the label and the check passed."; +it("accepts a recorded failed attempt without requiring every worker's name", () => { + const o = fixture(); o.reply = unnamedAttempt; + expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + unnamedAttempt.replace("You reassigned the task to Riley. ", ""), + unnamedAttempt.replace("the check passed", "the check failed"), + unnamedAttempt.replace("The verification command failed", "Riley ran verification and it failed"), +])("keeps missing handoff, failed repair and explicit wrong initial actor rejected: %s", reply => { + const o = fixture(); o.reply = reply; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 3114b73a79a..8f915495687 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v6", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v7", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },