fix(evals): grade unnamed initial timeline attempts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -258,7 +258,7 @@
       "../../server/src/services/native-runtime/status-decision-committer.ts",
       "../../server/src/services/work-timeline.ts"
     ],
-    "oracleVersion": "paperclip.chat-catalog.work-timeline.v6",
+    "oracleVersion": "paperclip.chat-catalog.work-timeline.v7",
     "coverageGaps": [],
     "expectedRunCount": 3,
     "minimumRunCount": 3,

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts

@@ -20,8 +20,8 @@ it.each([
 ])("retains consequential actor and outcome requirements: %s", reply => {
   expect(grade(reply)).toBe(false);
 });
-it("does not carry a section owner through an unrelated paragraph", () => {
-  expect(grade(account.replace("The verification", "\n\nThe verification"))).toBe(false);
+it("does not infer the initial worker over an explicit contrary actor in another paragraph", () => {
+  expect(grade(account.replace("The verification command failed", "\n\nRiley ran verification and it failed"))).toBe(false);
 });
 
 it("does not give a heading credit for another actor's command", () => {

tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts

@@ -85,7 +85,7 @@ export function truthfulTimelineAccount(input: {
         resolvedPronouns.add(at + lead.length);
         return lead + owner;
       });
-    const paragraph = { text, owner, start: offset, end: offset + text.length };
+    const paragraph = { text, owner, namedWorkerHeading: owners.length > 0, start: offset, end: offset + text.length };
     offset = paragraph.end + 2;
     return paragraph;
   });
@@ -93,7 +93,7 @@ export function truthfulTimelineAccount(input: {
   // Search milestone clauses, not the first occurrence of a worker or outcome.
   // A summary or heading may mention the repair before the ordered account.
   const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))]
-    .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, paragraphStart: paragraph?.start }; });
+    .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, namedWorkerHeading: paragraph?.namedWorkerHeading, paragraphStart: paragraph?.start }; });
   // A command noun can have ordinary modifiers, but never absorb a new event,
   // actor, negation or clause boundary while connecting this pause to failure.
   const commandBoundaries = [
@@ -108,7 +108,7 @@ export function truthfulTimelineAccount(input: {
 
   const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi;
   const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i");
-  const failures = clauses.flatMap(({ text, at, owner }) => (mentions(text, firstWorker) || owner === firstWorker && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker))
+  const failures = clauses.flatMap(({ text, at, owner, namedWorkerHeading }) => (mentions(text, firstWorker) || (owner === firstWorker || !namedWorkerHeading) && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker))
     && new RegExp(`\\b${commandSubject}\\b`, "i").test(text)
     ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []);
   const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index })));

tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts

@@ -236,7 +236,7 @@ it("admits the selected live timeline metadata without invoking fixture callback
   const spies = callbacks.map(key => vi.spyOn(implementation, key));
   try {
     expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 });
-    expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v6");
+    expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v7");
     expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles);
     expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 });
     for (const spy of spies) expect(spy).not.toHaveBeenCalled();
@@ -383,3 +383,17 @@ it.each(["another agent", "a bot"])("retains explicit nonhuman handoff attributi
   o.reply = o.reply.replace("Then a human reassigned the task to Riley.", `The task was reassigned to Riley by ${actor}.`);
   expect(gradeWorkTimeline(o).passed).toBe(false);
 });
+
+const unnamedAttempt = "The verification command failed with exit code 1 and the task paused. You reassigned the task to Riley. Riley repaired the label and the check passed.";
+it("accepts a recorded failed attempt without requiring every worker's name", () => {
+  const o = fixture(); o.reply = unnamedAttempt;
+  expect(gradeWorkTimeline(o).passed).toBe(true);
+});
+it.each([
+  unnamedAttempt.replace("You reassigned the task to Riley. ", ""),
+  unnamedAttempt.replace("the check passed", "the check failed"),
+  unnamedAttempt.replace("The verification command failed", "Riley ran verification and it failed"),
+])("keeps missing handoff, failed repair and explicit wrong initial actor rejected: %s", reply => {
+  const o = fixture(); o.reply = reply;
+  expect(gradeWorkTimeline(o).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts

@@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase {
     const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name };
     await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state;
   }, gradeWorkTimeline);
-  return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v6", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
+  return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v7", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
     sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"],
     async observe(ctx, state): Promise<Timeline> { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; },
     async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },