test(evals): join timeline milestones to native status decisions

Preserve raw system activity and prove its exact accepted worker outcome instead of requiring the legacy agent PATCH audit shape.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts

@@ -0,0 +1,68 @@
+import { expect, it, vi } from "vitest";
+import { readFileSync } from "node:fs";
+import { runInNewContext } from "node:vm";
+vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value }));
+import { gradeWorkTimeline } from "./internal-work-timeline.js";
+
+function producer(status: "blocked" | "done", runId: string, decisionId: string) {
+  const source = readFileSync(new URL("../../../server/src/services/native-runtime/status-decision-committer.ts", import.meta.url), "utf8");
+  const marker = "const { publication } = await persistActivity(tx as unknown as Db, ";
+  const anchor = source.lastIndexOf('actorId: "native-status-committer"');
+  expect(anchor).toBeGreaterThan(0);
+  const start = source.lastIndexOf(marker, anchor) + marker.length;
+  const end = source.indexOf("\n    });", anchor);
+  expect(end).toBeGreaterThan(start);
+  // Execute only the actual committed event-object projection, no server/DB import.
+  return runInNewContext("(" + source.slice(start, end) + "\n})", {
+    input: { companyId: "company", issueId: "task", runId, assessmentId: "assessment-" + runId, decision: { statusAction: "transition", toStatus: status } },
+    issue: { status: "in_progress" }, decisionRow: { id: decisionId }, reasonCode: "work_report", materialized: [],
+  }, { timeout: 100 }) as Record<string, unknown>;
+}
+function fixture() {
+  const first = { id: "first", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" }, resultJson: { decisionId: "block-decision", assessmentId: "assessment-first", issueStatusBefore: "in_progress", issueStatusAfter: "blocked" } };
+  const second = { ...first, id: "second", agentId: "riley", resultJson: { decisionId: "done-decision", assessmentId: "assessment-second", issueStatusBefore: "in_progress", issueStatusAfter: "done" } };
+  const blocked = { id: "task", companyId: "company", status: "blocked", assigneeAgentId: "morgan", lastStatusDecisionId: "block-decision" };
+  const done = { ...blocked, status: "done", assigneeAgentId: "riley", lastStatusDecisionId: "done-decision" };
+  const blockedEvent = { ...producer("blocked", "first", "block-decision"), id: "blocked", createdAt: "2026-10-26T01:00:00Z" };
+  const doneEvent = { ...producer("done", "second", "done-decision"), id: "done", createdAt: "2026-10-26T03:00:00Z" };
+  return { scenarioId: "activity.work-timeline", origin: "https://fixture.test", companyId: "company", leadId: "ceo",
+    state: { task: done, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Protected acquisition" }, firstRun: first, secondRun: second, blockedTask: blocked,
+      blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person" },
+    tasks: [done, { id: "prep" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [],
+    runs: [first, second], links: ["/RUN/issues/task", "/RUN/issues/prep"],
+    reply: "Today (UTC): verification failed and work paused. Then the handoff went to Riley. The label was repaired and the check passed.",
+    hiddenStatus: 404, timelineEvents: [blockedEvent, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, doneEvent],
+    verification: [{ taskId: "task", runId: "first", filename: "launch.html", sha256: "before", exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", sha256: "after", exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }],
+  };
+}
+it("grades the actual producer's native status events through the real owning timeline oracle", () => {
+  const o = fixture();
+  expect(o.timelineEvents[0]).toMatchObject({ actorType: "system", actorId: "native-status-committer", runId: "first", details: { source: "native_status_decision", toStatus: "blocked" } });
+  expect(gradeWorkTimeline(o).passed).toBe(true);
+});
+it.each(["companyId", "entityId", "runId", "actorId", "actorType", "source", "toStatus", "decisionId", "assessmentId"] as const)("rejects a wrong native event %s rather than accepting any system status event", field => {
+  const o = fixture(), event = o.timelineEvents[0] as Record<string, unknown>;
+  if (["source", "toStatus", "decisionId", "assessmentId"].includes(field)) (event.details as Record<string, unknown>)[field] = "foreign";
+  else event[field] = "foreign";
+  expect(gradeWorkTimeline(o).passed).toBe(false);
+});
+it.each(["worker", "missingWorker", "source", "contradictorySource", "status", "decision", "assessment", "resultStatus", "taskDecision", "assignee", "currentStatus"] as const)("rejects missing or unrelated accepted run/task proof %s", fault => {
+  const o = fixture();
+  if (fault === "worker") o.state.firstRun.agentId = "foreign";
+  if (fault === "missingWorker") Reflect.deleteProperty(o.state.firstRun, "agentId");
+  if (fault === "contradictorySource") Object.assign(o.state.firstRun.contextSnapshot, { taskId: "foreign" });
+  if (fault === "source") o.state.firstRun.contextSnapshot.issueId = "foreign";
+  if (fault === "status") o.state.firstRun.status = "running";
+  if (fault === "decision") o.state.firstRun.resultJson.decisionId = "foreign";
+  if (fault === "assessment") o.state.firstRun.resultJson.assessmentId = "foreign";
+  if (fault === "resultStatus") o.state.firstRun.resultJson.issueStatusAfter = "done";
+  if (fault === "taskDecision") o.state.blockedTask.lastStatusDecisionId = "foreign";
+  if (fault === "assignee") o.state.blockedTask.assigneeAgentId = "foreign";
+  if (fault === "currentStatus") o.state.blockedTask.status = "done";
+  expect(gradeWorkTimeline(o).passed).toBe(false);
+});
+it("still rejects fake command exits, unchanged bytes and unobserved human handoff", () => {
+  const wrongExit = fixture(); wrongExit.verification[0]!.exitCode = 0; expect(gradeWorkTimeline(wrongExit).passed).toBe(false);
+  const sameBytes = fixture(); sameBytes.verification[1]!.sha256 = "before"; expect(gradeWorkTimeline(sameBytes).passed).toBe(false);
+  const wrongHuman = fixture(); (wrongHuman.timelineEvents[1] as Record<string, unknown>).actorId = "foreign"; expect(gradeWorkTimeline(wrongHuman).passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts

@@ -8,6 +8,24 @@ import { startTimelineVerifier, type VerificationReceipt } from "./internal-time
 import { localDay } from "./internal-routine-audit.js";
 import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js";
 interface Timeline extends InternalObservation { verification: VerificationReceipt[]; hiddenStatus: number; timelineEvents: Entity[] }
+const objectRecord = (value: unknown): Record<string, unknown> => value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+/** Native outcomes are projected by the canonical committer, not an agent PATCH. */
+export function timelineStatusMilestone(row: Entity, task: Entity, run: Entity, workerId: string, status: "blocked" | "done") {
+  if (typeof workerId !== "string" || !workerId || !task.id || !run.id || row.action !== "issue.updated" || row.entityId !== task.id) return false;
+  // Preserve genuinely attributed legacy agent updates; system rows need the full native join.
+  if (row.actorType !== "system" && row.agentId === workerId && row.details?.status === status) return true;
+  const event = objectRecord(row), current = objectRecord(task), native = objectRecord(run), context = objectRecord(native.contextSnapshot), result = objectRecord(native.resultJson);
+  return typeof task.companyId === "string" && !!task.companyId && row.companyId === task.companyId && native.companyId === task.companyId
+    && task.status === status && task.assigneeAgentId === workerId && run.agentId === workerId && run.status === "succeeded"
+    && (context.issueId === task.id || context.taskId === task.id)
+    && (context.issueId === undefined || context.issueId === task.id) && (context.taskId === undefined || context.taskId === task.id)
+    && row.actorType === "system" && row.actorId === "native-status-committer" && event.runId === run.id
+    && row.details?.source === "native_status_decision" && row.details.toStatus === status
+    && typeof current.lastStatusDecisionId === "string" && !!current.lastStatusDecisionId
+    && row.details.decisionId === current.lastStatusDecisionId && result.decisionId === current.lastStatusDecisionId
+    && result.issueStatusAfter === status && result.issueStatusBefore === row.details.fromStatus
+    && typeof result.assessmentId === "string" && !!result.assessmentId && row.details.assessmentId === result.assessmentId;
+}
 export function gradeWorkTimeline(value: unknown) {
   const c = checks(); const o = observation(value) as Timeline | undefined;
   c.add("real-timeline-and-command-receipts", !!o && Array.isArray(o.verification) && Array.isArray(o.timelineEvents) && typeof o.hiddenStatus === "number", "Real command exits, current-week state events and independent private access are required.");
@@ -21,9 +39,9 @@ export function gradeWorkTimeline(value: unknown) {
   const today = String(o.state.today); const zone = String(o.state.timezone); const monday = new Date(`${today}T12:00:00Z`); monday.setUTCDate(monday.getUTCDate() - (monday.getUTCDay() + 6) % 7);
   c.add("ordered-current-week-native-milestones", events.length === 3 && [o.state.blockedEventId, o.state.handoffEventId, o.state.doneEventId].every(id => events.some(row => row.id === id))
     && events.every(row => row.entityId === t.id && row.action === "issue.updated")
-    && events.some(row => row.id === o.state.blockedEventId && row.agentId === first.agentId && row.details?.status === "blocked")
+    && events.some(row => row.id === o.state.blockedEventId && timelineStatusMilestone(row, entity(o.state, "blockedTask"), first, first.agentId!, "blocked"))
     && events.some(row => row.id === o.state.handoffEventId && row.actorType === "user" && row.actorId === o.state.personId && row.details?.assigneeAgentId === second.agentId)
-    && events.some(row => row.id === o.state.doneEventId && row.agentId === second.agentId && row.details?.status === "done")
+    && events.some(row => row.id === o.state.doneEventId && timelineStatusMilestone(row, t, second, second.agentId!, "done"))
     && events.every(row => !!row.createdAt && localDay(row.createdAt, zone) >= monday.toISOString().slice(0, 10) && localDay(row.createdAt, zone) <= today)
     && Date.parse(events.find(row => row.id === o.state.blockedEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!)
     && Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.doneEventId)!.createdAt!), "The saved actual blocked → human handoff → agent completion events fall in the person's real current week and retain chronological attribution.");
@@ -55,17 +73,19 @@ function timeline(): ChatCatalogCase {
     verifier = await startTimelineVerifier(ctx, t.id, [worker.id, repair.id]);
     await ctx.input.api.patch(`/api/issues/${t.id}`, { status: "todo", assigneeAgentId: worker.id, description: "Build the initial launch.html prototype with a semantic h1 and an email input id=\"email\", deliberately leave the label pending for the later accessibility review. Run the authorized Launch verification app verify_launch_html on the actual saved bytes. If it fails, record its actual exit code and pause this task as blocked awaiting a label repair; do not pretend the command passed, fix the pending label yet, or mark it done. Do not create other tasks." });
     const first = await pollUntil({ label: "Real launch verification failure and saved pause", deadlineAt: Date.now() + 180000, load: async () => ({ t: await ctx.input.api.get<Entity>(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), accept: row => row.t.status === "blocked" && row.runs.length === 1 && row.runs[0]?.status === "succeeded" && verifier!.receipts.some(r => r.runId === row.runs[0]?.id && r.exitCode === 1) });
-    const blockedEvent = (await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`)).find(row => row.action === "issue.updated" && row.agentId === worker.id && row.details?.status === "blocked"); if (!blockedEvent) throw new Error("Missing actual blocked milestone");
+    const firstRun = await ctx.input.api.get<Entity>(`/api/heartbeat-runs/${first.runs[0]!.id}`);
+    const blockedEvent = (await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`)).find(row => timelineStatusMilestone(row, first.t, firstRun, worker.id, "blocked")); if (!blockedEvent) throw new Error("Missing actual blocked milestone");
     await ctx.input.api.patch(`/api/issues/${t.id}`, { status: "todo", assigneeAgentId: repair.id, description: "Repair the existing launch.html prototype by adding a real label for the email input. Preserve its semantic heading. Run Launch verification verify_launch_html on the saved bytes, report its actual result, and complete only if the real command succeeds. The earlier failed check and pause remain history; do not rewrite them." });
     const second = await pollUntil({ label: "Real human handoff and verified native repair", deadlineAt: Date.now() + 180000, load: async () => ({ t: await ctx.input.api.get<Entity>(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), accept: row => row.t.status === "done" && row.runs.length === 2 && row.runs.every(r => r.status === "succeeded") && verifier!.receipts.some(r => r.runId !== first.runs[0]?.id && r.exitCode === 0) });
-    const events = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`); const handoff = events.find(row => row.actorType === "user" && row.details?.assigneeAgentId === repair.id); const done = events.find(row => row.agentId === repair.id && row.details?.status === "done"); if (!handoff || !done) throw new Error("Missing actual handoff/completion milestone");
+    const secondRun = await ctx.input.api.get<Entity>(`/api/heartbeat-runs/${second.runs.find(row => row.id !== firstRun.id)!.id}`);
+    const events = await ctx.input.api.get<Entity[]>(`/api/issues/${t.id}/activity`); const handoff = events.find(row => row.actorType === "user" && row.details?.assigneeAgentId === repair.id); const done = events.find(row => timelineStatusMilestone(row, second.t, secondRun, repair.id, "done")); if (!handoff || !done) throw new Error("Missing actual handoff/completion milestone");
     const timezone = await ctx.input.page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone); const now = new Date(); const today = localDay(now.toISOString(), timezone); const zoneLabel = new Intl.DateTimeFormat("en-US", { timeZone: timezone, timeZoneName: "short" }).formatToParts(now).find(part => part.type === "timeZoneName")!.value;
     const session = await ctx.input.api.get<{ user?: { id: string } }>("/api/auth/get-session"); if (!session.user?.id) throw new Error("Timeline requires actual human handoff identity");
-    const state = { project: p, task: second.t, prepTask, hiddenTask, firstRun: first.runs[0], secondRun: second.runs.find(row => row.id !== first.runs[0]!.id), blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id };
+    const state = { project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id };
     await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state;
   }, gradeWorkTimeline);
   return { ...base, authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000,
-    sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts"],
+    sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts"],
     async observe(ctx, state): Promise<Timeline> { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; },
     async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } },
     async cleanup(ctx) { try { await verifier?.close(); } finally { verifier = undefined; await cleanupPrivacyProbes(ctx); await owner?.dispose(); owner = undefined; } },