From 62300ee07d58abf97b3668a62937103315f5a82e Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 18:22:07 -0600 Subject: [PATCH] test(evals): bind completed artifact edits to attached results --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-timeline-narrative.test.ts | 29 +++++++++++++++++++ .../internal-timeline-narrative.ts | 5 ++-- ...internal-timeline-native-milestone.test.ts | 2 +- .../internal-work-timeline.ts | 2 +- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index c38dccd72b7..6440831963c 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v10", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v11", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts index da223316e7d..dbc2ebed82b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.test.ts @@ -227,3 +227,32 @@ it.each([ it("stops repair subject continuity when a coordinated clause introduces another worker", () => { expect(grade("Morgan ran verification which failed and work paused. You reassigned the task to Riley. Riley added the label, and Avery inspected the saved bytes and then re-ran verify_launch_html. It passed with exit code 0 on the saved bytes.")).toBe(false); }); + +const editedArtifactAccount = `Morgan ran verification which failed and work paused. + +You reassigned RUN-3 from Morgan to Riley. + +**Repair and passing verification** +Riley edited \`launch.html\` to add \`\` while preserving the \`

\`, then re-ran \`verify_launch_html\` on the saved bytes: +- Result: exit code 0 (passed), checked 00:09:08.`; +const gradeEditedArtifact = (reply: string) => truthfulTimelineAccount({ reply, firstWorker: "Morgan", repairWorker: "Riley", taskIdentifier: "RUN-3" }); +it("binds a completed artifact edit to its immediately attached command-result bullet", () => { + expect(gradeEditedArtifact(editedArtifactAccount)).toBe(true); +}); +it.each([ + editedArtifactAccount.replace("Riley edited", "Morgan edited"), + editedArtifactAccount.replace("Riley edited", "Riley planned to edit"), + editedArtifactAccount.replace("Riley edited", "Riley never edited"), + editedArtifactAccount.replace("then re-ran", "then Avery re-ran"), + editedArtifactAccount.replace("Result: exit code 0 (passed)", "Result: Avery ran the check which passed"), + editedArtifactAccount.replace("- Result:", "- Avery inspected the page.\n- Result:"), + editedArtifactAccount.replace("- Result:", "\n- Result:"), + editedArtifactAccount.replace("- Result:", "## Another milestone\n- Result:"), + editedArtifactAccount.replace("exit code 0 (passed)", "an unrelated check passed"), + editedArtifactAccount.replace("exit code 0 (passed)", "did not pass"), + editedArtifactAccount.replace("exit code 0 (passed)", "exit code 1 (failed)"), + editedArtifactAccount.replace("You reassigned RUN-3 from Morgan to Riley.", "") + " You reassigned RUN-3 from Morgan to Riley.", + editedArtifactAccount + " A native run failed.", +])("keeps completed-edit command results bounded by actor, assertion, paragraph and order: %s", reply => { + expect(gradeEditedArtifact(reply)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index ce1855d91d3..ab2bdfaa69a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -125,6 +125,7 @@ export function truthfulTimelineAccount(input: { const repairModifier = "(?:then|later|now|personally|explicitly|actually|had|has|have|also|who|successfully|finally|subsequently|after\\s+the\\s+handoff)"; const repairVerb = [ "repair(?:ed)?|fix(?:ed)?", + "edited\\s+(?:[a-z0-9_.-]+|(?:the\\s+)?(?:saved\\s+)?(?:file|page|artifact))\\s+to\\s+add\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label", "(?:completed|finished|performed|applied|implemented)\\s+(?:(?:a|the|needed|required|label|accessibility)\\s+){0,4}(?:repairs?|fix(?:es)?)", "(?:added|inserted|supplied|provided|restored)\\s+(?:(?:a|the|missing|real|email|input|proper|accessible)\\s+){0,5}label", ].join("|"); @@ -155,7 +156,7 @@ export function truthfulTimelineAccount(input: { // repair account. No intervening sentence or earlier/unrelated result joins. const adjacentVerification = new RegExp(`^\\s*(?:(?:the|a|an)\\s+|${escape(repairWorker)}['’]s\\s+)?${commandSubject}\\s+${successModifiers}(?:${repairSuccess.source})(?:\\s+with\\s+exit(?:\\s+(?:code|status))?\\s*[:=]?\\s*0)?\\s*[.!]?\\s*$`, "i"); const contextualCommandLead = new RegExp(`^\\s*(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s+${successModifiers}$`, "i"); - const coordinatedCommand = new RegExp(`(?:,\\s*(?:and\\s+)?|\\band\\s+)(?:${repairModifier}\\s+){0,3}(?:re-?ran|ran|executed|performed)\\s+(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s*[.!]?\\s*$`, "i"); + const coordinatedCommand = new RegExp(`(?:,\\s*(?:and\\s+)?|\\band\\s+)(?:${repairModifier}\\s+){0,3}(?:re-?ran|ran|executed|performed)\\s+(?:(?:the|a|an)\\s+)?${commandSubject}(?:[ \\t]+${commandModifier}){0,5}\\s*[.!:]?\\s*$`, "i"); const repairObjectTail = new RegExp(`^(?:[ \\t]+${commandModifier}){0,6}\\s*[.!]?\\s*$`, "i"); const passiveRepairSubject = new RegExp(`^\\s*(?:${commandModifier}[ \\t]+){1,6}$`, "i"); const deniedRepairObject = /^\s*(?:[a-z]+ly\s+){0,2}(?:(?:no|none|neither|nothing)\b|zero(?=\s|[.!?]|$))/i; @@ -179,7 +180,7 @@ export function truthfulTimelineAccount(input: { const sameWorkerParagraph = paragraphStart === nextClause?.paragraphStart && (owner === repairWorker && nextClause.owner === repairWorker || explicitRepairSubject); const nextCommand = next && sameWorkerParagraph && coordinatedCommand.test(text.slice(end)) - ? next.replace(/^\s*it\b/i, "check") : next; + ? next.replace(/^(?:\s*it\b\s*|\s*result:\s*(?=[^\n]))/i, match => /result:/i.test(match) && !/:\s*$/.test(text) ? match : "check ") : next; const contextualOutcome = nextCommand && repairSuccess.exec(nextCommand); const contextualResult = sameWorkerParagraph && !!contextualOutcome && contextualCommandLead.test(nextCommand.slice(0, contextualOutcome.index)) diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index c7e3e64e0d4..a027714f00e 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -269,7 +269,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v10"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v11"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 443e4898425..0e71f4e841a 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -92,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v10", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v11", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },