From d8f5e2bcdc305efcfc789640fafd23c6325476c0 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 07:41:58 -0600 Subject: [PATCH] test(activity): synchronize selected timeline metadata and calibration Co-Authored-By: Paperclip --- tests/runner-e2e/chat-catalog-descriptors.json | 6 ++++-- .../internal-remaining.test.ts | 10 +++++----- .../internal-timeline-native-milestone.test.ts | 18 +++++++++++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index f6a002e6799..1b74b4ed1f8 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -249,14 +249,16 @@ "chat-catalog-domains/internal-activity.ts", "chat-scenarios/activity.yaml", "chat-catalog-domains/internal-work-timeline.ts", + "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", - "../../server/src/services/native-runtime/status-decision-committer.ts" + "../../server/src/services/native-runtime/status-decision-committer.ts", + "../../server/src/services/work-timeline.ts" ], - "oracleVersion": "paperclip.chat-catalog.internal.v1", + "oracleVersion": "paperclip.chat-catalog.work-timeline.v2", "coverageGaps": [], "expectedRunCount": 3, "minimumRunCount": 3, diff --git a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts index 8ff80e9d551..7135504b632 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-remaining.test.ts @@ -90,13 +90,13 @@ it("rejects creating today's routine and relabeling it yesterday", () => { const it("rejects another company or a real successful morning run", () => { const o = routine(); o.routine.companyId = "foreign"; expect(gradeRoutineAudit(o).passed).toBe(false); const wrong = routine(); wrong.routineRuns = [{ id: "morning", status: "succeeded", triggeredAt: "2026-10-26T08:00:00Z" }]; expect(gradeRoutineAudit(wrong).passed).toBe(false); }); it("rejects treating an edit as execution or losing a produced task", () => { const o = routine(); o.reply = "The enabled routine successfully ran."; expect(gradeRoutineAudit(o).passed).toBe(false); const wrong = routine(); wrong.routineRuns = [{ id: "failed", status: "failed", createdAt: "2026-10-26T08:00:00Z", linkedIssueId: "missing" }]; expect(gradeRoutineAudit(wrong).passed).toBe(false); }); function timeline() { - const o = base("activity.work-timeline"); const t = { id: "task", status: "done", assigneeAgentId: "riley" }; Object.assign(o.state, { task: t, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Secret acquisition", description: "Protected fact" }, firstRun: { id: "first", agentId: "morgan" }, secondRun: { id: "second", agentId: "riley" }, blockedTask: { id: "task", status: "blocked" }, blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person" }); - o.tasks = [t, { id: "prep" }]; o.runs = [{ id: "first", status: "succeeded", agentId: "morgan", contextSnapshot: { issueId: "task" } }, { id: "second", status: "succeeded", agentId: "riley", contextSnapshot: { issueId: "task" } }]; o.reply = "Today (UTC): verification failed and work paused. Then the handoff went to Riley. The label was repaired and the check passed."; o.links = ["/RUN/issues/task", "/RUN/issues/prep"]; - return { ...o, hiddenStatus: 404, timelineEvents: [{ id: "blocked", entityId: "task", action: "issue.updated", agentId: "morgan", details: { status: "blocked" }, createdAt: "2026-10-26T01:00:00Z" }, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, { id: "done", entityId: "task", action: "issue.updated", agentId: "riley", details: { status: "done" }, createdAt: "2026-10-26T03:00:00Z" }], verification: [{ taskId: "task", runId: "first", filename: "launch.html", sha256: "before", exitCode: 1, checkedAt: "2026-10-26T00:50:00Z", command: "node check" }, { taskId: "task", runId: "second", filename: "launch.html", sha256: "after", exitCode: 0, checkedAt: "2026-10-26T02:50:00Z", command: "node check" }] }; + const o = base("activity.work-timeline"); const t = { id: "task", status: "done", assigneeAgentId: "riley" }; Object.assign(o.state, { worker: { id: "morgan", name: "Morgan" }, repairWorker: { id: "riley", name: "Riley" }, task: t, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Secret acquisition", description: "Protected fact" }, firstRun: { id: "first", agentId: "morgan" }, secondRun: { id: "second", agentId: "riley" }, blockedTask: { id: "task", status: "blocked" }, blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person" }); + o.tasks = [t, { id: "prep" }]; o.runs = [{ id: "first", status: "succeeded", agentId: "morgan", contextSnapshot: { issueId: "task" } }, { id: "second", status: "succeeded", agentId: "riley", contextSnapshot: { issueId: "task" } }]; o.reply = "Today (UTC): Morgan ran verification which failed and work paused. Then a human reassigned the task to Riley. Riley repaired the label and the check passed."; o.links = ["/RUN/issues/task", "/RUN/issues/prep"]; + return { ...o, hiddenStatus: 404, timelineEvents: [{ id: "blocked", entityId: "task", action: "issue.updated", agentId: "morgan", details: { status: "blocked" }, createdAt: "2026-10-26T01:00:00Z" }, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, { id: "done", entityId: "task", action: "issue.updated", agentId: "riley", details: { status: "done" }, createdAt: "2026-10-26T03:00:00Z" }], verification: [{ taskId: "task", runId: "first", filename: "launch.html", sha256: "a".repeat(64), exitCode: 1, checkedAt: "2026-10-26T00:50:00Z", command: "node accessible-heading-and-email-label-check" }, { taskId: "task", runId: "second", filename: "launch.html", sha256: "b".repeat(64), exitCode: 0, checkedAt: "2026-10-26T02:50:00Z", command: "node accessible-heading-and-email-label-check" }] }; } it("accepts actual command failure, pause, handoff and repair within the current week", () => expect(gradeWorkTimeline(timeline()).passed).toBe(true)); -it("accepts natural chronological phrasing", () => { const o = timeline(); o.reply = "Today in UTC the accessibility check was failing; Morgan was blocked. After reassignment Riley fixed the label and verification passed."; expect(gradeWorkTimeline(o).passed).toBe(true); }); -it("rejects wrong run/unchanged bytes or reversed chronology", () => { const o = timeline(); o.verification[1]!.sha256 = "before"; expect(gradeWorkTimeline(o).passed).toBe(false); const wrong = timeline(); wrong.timelineEvents[1]!.createdAt = "2026-10-26T00:00:00Z"; expect(gradeWorkTimeline(wrong).passed).toBe(false); }); +it("accepts natural chronological phrasing", () => { const o = timeline(); o.reply = "Today in UTC Morgan ran the accessibility check which was failing; Morgan was blocked. You reassigned the task to Riley. Riley fixed the label and verification passed."; expect(gradeWorkTimeline(o).passed).toBe(true); }); +it("rejects wrong run/unchanged bytes or reversed chronology", () => { const o = timeline(); o.verification[1]!.sha256 = "a".repeat(64); expect(gradeWorkTimeline(o).passed).toBe(false); const wrong = timeline(); wrong.timelineEvents[1]!.createdAt = "2026-10-26T00:00:00Z"; expect(gradeWorkTimeline(wrong).passed).toBe(false); }); it("rejects conflating command failure with native failure or exposing private work", () => { const o = timeline(); o.reply += " The native run failed."; expect(gradeWorkTimeline(o).passed).toBe(false); const wrong = timeline(); wrong.reply += " Secret acquisition."; expect(gradeWorkTimeline(wrong).passed).toBe(false); }); it("rejects fabricated prior-week dates or wrong timezone", () => { const o = timeline(); o.timelineEvents[0]!.createdAt = "2026-10-19T03:00:00Z"; expect(gradeWorkTimeline(o).passed).toBe(false); const wrong = timeline(); wrong.reply = wrong.reply.replace("UTC", "Pacific"); expect(gradeWorkTimeline(wrong).passed).toBe(false); }); function partial() { diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index e3c621f92d6..1f7c45d1374 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -2,7 +2,9 @@ import { expect, it, vi } from "vitest"; import { readFileSync } from "node:fs"; import { runInNewContext } from "node:vm"; vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value })); -import { gradeWorkTimeline } from "./internal-work-timeline.js"; +import { validateChatCatalogConsistency } from "../chat-catalog-consistency.js"; +import { findChatCatalogCase } from "../chat-catalog-registry.js"; +import { gradeWorkTimeline, workTimelineCases } from "./internal-work-timeline.js"; function producer(status: "blocked" | "done", runId: string, decisionId: string) { const source = readFileSync(new URL("../../../server/src/services/native-runtime/status-decision-committer.ts", import.meta.url), "utf8"); @@ -105,3 +107,17 @@ it.each(["missing link", "hidden work", "source mutation", "wrong command", "wro if (fault === "empty hash") o.verification[0]!.sha256 = ""; expect(gradeWorkTimeline(o).passed).toBe(false); }); + +it("admits the selected live timeline metadata without invoking fixture callbacks", async () => { + const descriptor = findChatCatalogCase("activity.work-timeline")!; + const implementation = workTimelineCases[0]!; + const callbacks = ["setup", "observe", "grade", "verifyResult"] as const; + const spies = callbacks.map(key => vi.spyOn(implementation, key)); + try { + expect(await validateChatCatalogConsistency([descriptor])).toEqual({ descriptorCount: 1, fixtureCallbacksInvoked: 0 }); + expect(descriptor.oracleVersion).toBe("paperclip.chat-catalog.work-timeline.v2"); + expect(descriptor.sourceFiles).toEqual(implementation.sourceFiles); + expect(descriptor).toMatchObject({ expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000 }); + for (const spy of spies) expect(spy).not.toHaveBeenCalled(); + } finally { for (const spy of spies) spy.mockRestore(); } +});