From 936c3bbe0e83ca15616254f23d9023b2065fc5fb Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 20:46:11 -0600 Subject: [PATCH] test(evals): bind timeline command notation to worker bullets --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-timeline-narrative.test.ts | 23 +++++++++++++++++++ .../internal-timeline-narrative.ts | 20 ++++++++++++---- ...internal-timeline-native-milestone.test.ts | 2 +- .../internal-work-timeline.ts | 2 +- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 90a29eddd2c..4305f544df2 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -259,7 +259,7 @@ "../../server/src/services/work-timeline.ts", "../../server/src/services/native-runtime/runner-api-catalog.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v11", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v12", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts index dbc2ebed82b..d0f45d669d4 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts @@ -256,3 +256,26 @@ it.each([ ])("keeps completed-edit command results bounded by actor, assertion, paragraph and order: %s", reply => { expect(gradeEditedArtifact(reply)).toBe(false); }); + + +const bulletCommandAccount = `**Timeline (Nov 6, UTC)** +- **02:31:30–02:32:37 — Morgan:** built launch.html without the label per the brief. Ran authorized verify_launch_html on the saved bytes: exitCode 1. Morgan set the task to blocked. +- **02:32:38 — you reassigned RUN-3 to Riley**, cleared the blocker. +- **02:32:38–02:33:04 — Riley:** added , preserved the heading, reran verify_launch_html on the saved bytes: exitCode 0. Run succeeded; committed to done.`; +const gradeBulletCommand = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" }); +it("keeps a completed command bound to the preceding worker-labeled bullet subject", () => { + expect(gradeBulletCommand(bulletCommandAccount)).toBe(true); +}); +it.each([ + bulletCommandAccount.replace("Ran authorized", "Avery ran authorized"), + bulletCommandAccount.replace("brief. Ran", "brief. Avery inspected the page. Ran"), + bulletCommandAccount.replace("brief. Ran", "brief.\n- Ran"), + bulletCommandAccount.replace("brief. Ran", "brief.\n## Another milestone\nRan"), + bulletCommandAccount.replace("Ran authorized", "Planned to run authorized"), + bulletCommandAccount.replace("exitCode 1", "did not fail"), + bulletCommandAccount.replace("exitCode 0", "exitCode 2"), + bulletCommandAccount.replace("Riley:** added", "Riley:** planned to add"), + bulletCommandAccount.replace("- **02:32:38 — you reassigned RUN-3 to Riley**, cleared the blocker.\n", "") + " You reassigned RUN-3 to Riley.", +])("does not borrow omitted command subjects across a material boundary: %s", reply => { + expect(gradeBulletCommand(reply)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index ab2bdfaa69a..90f73c16f58 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -68,7 +68,7 @@ export function truthfulTimelineAccount(input: { }; let offset = 0; let activeHeadingWorkers: string[] = []; - const paragraphs = reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + const paragraphs = reply.replace(/\bexitCode\b/gi, "exit code").replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") .replace(/^([ \t]*(?:#{1,6} +[^\n]+|\*\*[^\n*]+\*\*))[ \t]*\n[ \t]*\n(?=\S)/gm, "$1\n") .replace(/^([ \t]*\*\*[^\n*]+\*\*)[ \t]+(?=\S)/gm, "$1\n") .split(/\n\s*\n|(?=^[ \t]*(?:#{1,6} +|\*\*[^\n*]+\*\*[ \t]*(?:\n|$)))/m).filter(Boolean).map(raw => { @@ -102,7 +102,7 @@ export function truthfulTimelineAccount(input: { // Search milestone clauses, not the first occurrence of a worker or outcome. // A summary or heading may mention the repair before the ordered account. const clauses = [...body.matchAll(new RegExp(`${withinClause}+(?:[!?]|\\.(?=\\s|$)|(?=\\n|$))`, "g"))] - .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, contraryWorkerHeading: paragraph?.contraryWorkerHeading, paragraphStart: paragraph?.start }; }); + .map(match => { const paragraph = paragraphs.find(row => row.start <= match.index && row.end > match.index); return { text: match[0], at: match.index, owner: paragraph?.owner, contraryWorkerHeading: paragraph?.contraryWorkerHeading, paragraphStart: paragraph?.start, lineStart: body.lastIndexOf("\n", match.index) + 1 }; }); // A command noun can have ordinary modifiers, but never absorb a new event, // actor, negation or clause boundary while connecting this pause to failure. const commandBoundaries = [ @@ -117,9 +117,19 @@ export function truthfulTimelineAccount(input: { const failureOutcome = /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/gi; const commandIsSubject = new RegExp(`^\\s*(?:(?:then|next|later|finally)\\s+)?(?:(?:the|a|an)\\s+)?${commandSubject}\\b`, "i"); - const failures = clauses.flatMap(({ text, at, contraryWorkerHeading }) => (mentions(text, firstWorker) || !contraryWorkerHeading && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker)) - && new RegExp(`\\b${commandSubject}\\b`, "i").test(text) - ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []); + const firstWorkerBullet = new RegExp(`^[\\s\\d:–—-]*${escape(firstWorker)}\\s*:`, "i"); + const continuedCommand = /^\s*(?:then\s+)?(?:re-?ran|ran|executed|performed)\s+/i; + const failures = clauses.flatMap(({ text, at, contraryWorkerHeading, lineStart }, index) => { + const previous = clauses[index - 1]; + const sameWorkerBullet = previous?.lineStart === lineStart && firstWorkerBullet.test(previous.text) + && !namesOtherActor(previous.text, firstWorker) && !mentions(previous.text, repairWorker) + && continuedCommand.test(text) && !namesOtherActor(text.replace(continuedCommand, ""), firstWorker) + && !/\b(?:not|never|no|will|would|could|should|planned|if|unless)\b|n['’]t\b/i.test(text); + return (mentions(text, firstWorker) || sameWorkerBullet + || !contraryWorkerHeading && commandIsSubject.test(text) && !namesOtherActor(text.replace(commandIsSubject, ""), firstWorker)) + && new RegExp(`\\b${commandSubject}\\b`, "i").test(text) + ? [...text.matchAll(failureOutcome)].map(match => ({ at: at + match.index, end: at + match.index + match[0].length, clauseAt: at })) : []; + }); const handoffs = clauses.flatMap(({ text, at }) => handoffMatches(text).map(match => ({ at: at + match.index }))); const repairGap = "(?:\\s+|\\s*[,:—–-]\\s*)"; const repairModifier = "(?:then|later|now|personally|explicitly|actually|had|has|have|also|who|successfully|finally|subsequently|after\\s+the\\s+handoff)"; diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index a027714f00e..ed5f0f1cbbd 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -269,7 +269,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v11"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v12"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index f1e06a54979..6fe60bef9f0 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v11", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v12", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts","../../server/src/services/native-runtime/runner-api-catalog.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },