From b59a2d0e4902d52b7955861fa7a5b38156963f04 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 21:23:39 -0600 Subject: [PATCH] test(evals): scope timeline date claims to the actual request Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 2 +- .../internal-timeline-narrative.ts | 7 +++---- .../internal-timeline-native-milestone.test.ts | 17 ++++++++++++++++- .../internal-work-timeline.ts | 13 ++++++++++--- tests/runner-e2e/chat-scenarios/activity.yaml | 2 +- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1b74b4ed1f8..4b385befc09 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -258,7 +258,7 @@ "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.work-timeline.v2", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v3", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts index 5246c0c6662..a8ebf3ca6b3 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -1,8 +1,8 @@ /** Textual calibration only; the owning grader separately verifies command/audit joins. */ export function truthfulTimelineAccount(input: { - reply: string; personName?: string; firstWorker: string; repairWorker: string; today: string; timezone: string; zoneLabel: string; + reply: string; personName?: string; firstWorker: string; repairWorker: string; }) { - const { reply, firstWorker, repairWorker, today, timezone, zoneLabel } = input; + const { reply, firstWorker, repairWorker } = input; const body = reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1").replace(/[*`]/g, ""); if (!firstWorker || !repairWorker || firstWorker === repairWorker) return false; const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); @@ -123,6 +123,5 @@ export function truthfulTimelineAccount(input: { return !/\b(?:no|neither|not a|not an|none of the)\s*$/i.test(before) && !/^\s*(?:if|had)\b/i.test(text); }); - return ordered && !falseNativeFailure && (/\btoday\b/i.test(reply) || reply.includes(today)) - && [timezone, zoneLabel].some(zone => !!zone && reply.includes(zone)); + return ordered && !falseNativeFailure; } diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index d2e9503d103..b38a41dbba9 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -37,6 +37,21 @@ function fixture() { verification: [{ taskId: "task", runId: "first", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "a".repeat(64), exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "b".repeat(64), exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }], }; } +it("accepts an account without unrequested date or timezone labels but rejects prior-week ISO dates", () => { + const o = fixture(); + o.reply = o.reply.replace("Today (UTC): ", "") + " [Source](/RUN/issues/task?at=2026-10-25)"; + expect(gradeWorkTimeline(o).passed).toBe(true); + o.reply = "On 2026-10-25: " + o.reply; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); +it("accepts the current week's heading when its recorded milestones occur later", () => { + const o = fixture(); + o.state.today = "2026-11-04"; + for (const event of o.timelineEvents) event.createdAt = event.createdAt.replace("2026-10-26", "2026-11-04"); + for (const receipt of o.verification) receipt.checkedAt = receipt.checkedAt.replace("2026-10-26", "2026-11-04"); + o.reply = "Week of 2026-11-02: " + o.reply.replace("Today (UTC): ", ""); + expect(gradeWorkTimeline(o).passed).toBe(true); +}); it("grades the actual producer's native status events through the real owning timeline oracle", () => { const o = fixture(); expect(o.timelineEvents[0]).toMatchObject({ actorType: "system", actorId: "native-status-committer", runId: "first", details: { source: "native_status_decision", toStatus: "blocked" } }); @@ -190,7 +205,7 @@ it("admits the selected live timeline metadata without invoking fixture callback const spies = callbacks.map(key => vi.spyOn(implementation, key)); try { expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); - expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v2"); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v3"); expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); for (const spy of spies) expect(spy).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 9f3dcb3afd3..5c270a2c45b 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -48,9 +48,16 @@ export function gradeWorkTimeline(value: unknown) { && Date.parse(events.find(row => row.id === o.state.blockedEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) && Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.doneEventId)!.createdAt!), "The saved actual blocked → human handoff → agent completion events fall in the person's real current week and retain chronological attribution."); const worker = objectRecord(o.state.worker), repairWorker = objectRecord(o.state.repairWorker); - c.add("ordered-and-truthful-account", worker.id === first.agentId && repairWorker.id === second.agentId && truthfulTimelineAccount({ reply: o.reply, + const accountText = o.reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1"); + const claimedDates = accountText.match(/\b\d{4}-\d{2}-\d{2}\b/g) ?? []; + const dateClaimsInWindow = claimedDates.every(day => { + const parsed = new Date(`${day}T12:00:00Z`); + return Number.isFinite(parsed.getTime()) && parsed.toISOString().slice(0, 10) === day + && day >= monday.toISOString().slice(0, 10) && day <= today; + }); + c.add("ordered-and-truthful-account", worker.id === first.agentId && repairWorker.id === second.agentId && dateClaimsInWindow && truthfulTimelineAccount({ reply: o.reply, firstWorker: typeof worker.name === "string" ? worker.name : "", repairWorker: typeof repairWorker.name === "string" ? repairWorker.name : "", - personName: typeof o.state.personName === "string" ? o.state.personName : undefined, today, timezone: zone, zoneLabel: String(o.state.zoneLabel) }), "The reply attributes and orders the actual command failure/pause, human reassignment to the different repair worker and successful repair, labels local time, and does not invent a failed native run."); + personName: typeof o.state.personName === "string" ? o.state.personName : undefined }), "The reply attributes and orders the actual command failure/pause, human reassignment to the different repair worker and successful repair, keeps explicit ISO dates within the current-week observation window, and does not invent a failed native run."); c.add("inspectable-major-milestones", hasLink(o, "issues", t) && hasLink(o, "issues", prep), "Both actual launch work items are linked for inspection."); const hidden = entity(o.state, "hiddenTask"); c.add("hidden-work-never-disclosed", [403, 404].includes(o.hiddenStatus) && ![hidden.id, hidden.title, hidden.description].filter(Boolean).some(text => o.reply.includes(text!)) @@ -85,7 +92,7 @@ function timeline(): ChatCatalogCase { const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v3", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, diff --git a/tests/runner-e2e/chat-scenarios/activity.yaml b/tests/runner-e2e/chat-scenarios/activity.yaml index 7a921e76bce..259318febf1 100644 --- a/tests/runner-e2e/chat-scenarios/activity.yaml +++ b/tests/runner-e2e/chat-scenarios/activity.yaml @@ -61,7 +61,7 @@ scenarios: Walk me through how the launch work progressed this week. expected_outcome: >- The agent builds an ordered account from saved timeline and task events, including pauses - and failed attempts. It uses the requested timezone and distinguishes recorded facts from + and failed attempts. It respects any explicitly requested timezone and distinguishes recorded facts from interpretation. Links let the person inspect the major milestones without exposing hidden tasks. - id: activity.agent-feedback