From 4dcd2a5c1e4d202e97a67b55ddd2a729986aa2af Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 00:38:19 -0600 Subject: [PATCH] test(evals): join timeline milestones to native status decisions Preserve raw system activity and prove its exact accepted worker outcome instead of requiring the legacy agent PATCH audit shape. Co-Authored-By: Paperclip --- ...internal-timeline-native-milestone.test.ts | 68 +++++++++++++++++++ .../internal-work-timeline.ts | 32 +++++++-- 2 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts new file mode 100644 index 00000000000..95aead94075 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -0,0 +1,68 @@ +import { expect, it, vi } from "vitest"; +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; +vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value })); +import { gradeWorkTimeline } from "./internal-work-timeline.js"; + +function producer(status: "blocked" | "done", runId: string, decisionId: string) { + const source = readFileSync(new URL("../../../server/src/services/native-runtime/status-decision-committer.ts", import.meta.url), "utf8"); + const marker = "const { publication } = await persistActivity(tx as unknown as Db, "; + const anchor = source.lastIndexOf('actorId: "native-status-committer"'); + expect(anchor).toBeGreaterThan(0); + const start = source.lastIndexOf(marker, anchor) + marker.length; + const end = source.indexOf("\n });", anchor); + expect(end).toBeGreaterThan(start); + // Execute only the actual committed event-object projection, no server/DB import. + return runInNewContext("(" + source.slice(start, end) + "\n})", { + input: { companyId: "company", issueId: "task", runId, assessmentId: "assessment-" + runId, decision: { statusAction: "transition", toStatus: status } }, + issue: { status: "in_progress" }, decisionRow: { id: decisionId }, reasonCode: "work_report", materialized: [], + }, { timeout: 100 }) as Record; +} +function fixture() { + const first = { id: "first", companyId: "company", agentId: "morgan", status: "succeeded", contextSnapshot: { issueId: "task" }, resultJson: { decisionId: "block-decision", assessmentId: "assessment-first", issueStatusBefore: "in_progress", issueStatusAfter: "blocked" } }; + const second = { ...first, id: "second", agentId: "riley", resultJson: { decisionId: "done-decision", assessmentId: "assessment-second", issueStatusBefore: "in_progress", issueStatusAfter: "done" } }; + const blocked = { id: "task", companyId: "company", status: "blocked", assigneeAgentId: "morgan", lastStatusDecisionId: "block-decision" }; + const done = { ...blocked, status: "done", assigneeAgentId: "riley", lastStatusDecisionId: "done-decision" }; + const blockedEvent = { ...producer("blocked", "first", "block-decision"), id: "blocked", createdAt: "2026-10-26T01:00:00Z" }; + const doneEvent = { ...producer("done", "second", "done-decision"), id: "done", createdAt: "2026-10-26T03:00:00Z" }; + return { scenarioId: "activity.work-timeline", origin: "https://fixture.test", companyId: "company", leadId: "ceo", + state: { task: done, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Protected acquisition" }, firstRun: first, secondRun: second, blockedTask: blocked, + blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person" }, + tasks: [done, { id: "prep" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], + runs: [first, second], links: ["/RUN/issues/task", "/RUN/issues/prep"], + reply: "Today (UTC): verification failed and work paused. Then the handoff went to Riley. The label was repaired and the check passed.", + hiddenStatus: 404, timelineEvents: [blockedEvent, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, doneEvent], + verification: [{ taskId: "task", runId: "first", filename: "launch.html", sha256: "before", exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", sha256: "after", exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }], + }; +} +it("grades the actual producer's native status events through the real owning timeline oracle", () => { + const o = fixture(); + expect(o.timelineEvents[0]).toMatchObject({ actorType: "system", actorId: "native-status-committer", runId: "first", details: { source: "native_status_decision", toStatus: "blocked" } }); + expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each(["companyId", "entityId", "runId", "actorId", "actorType", "source", "toStatus", "decisionId", "assessmentId"] as const)("rejects a wrong native event %s rather than accepting any system status event", field => { + const o = fixture(), event = o.timelineEvents[0] as Record; + if (["source", "toStatus", "decisionId", "assessmentId"].includes(field)) (event.details as Record)[field] = "foreign"; + else event[field] = "foreign"; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); +it.each(["worker", "missingWorker", "source", "contradictorySource", "status", "decision", "assessment", "resultStatus", "taskDecision", "assignee", "currentStatus"] as const)("rejects missing or unrelated accepted run/task proof %s", fault => { + const o = fixture(); + if (fault === "worker") o.state.firstRun.agentId = "foreign"; + if (fault === "missingWorker") Reflect.deleteProperty(o.state.firstRun, "agentId"); + if (fault === "contradictorySource") Object.assign(o.state.firstRun.contextSnapshot, { taskId: "foreign" }); + if (fault === "source") o.state.firstRun.contextSnapshot.issueId = "foreign"; + if (fault === "status") o.state.firstRun.status = "running"; + if (fault === "decision") o.state.firstRun.resultJson.decisionId = "foreign"; + if (fault === "assessment") o.state.firstRun.resultJson.assessmentId = "foreign"; + if (fault === "resultStatus") o.state.firstRun.resultJson.issueStatusAfter = "done"; + if (fault === "taskDecision") o.state.blockedTask.lastStatusDecisionId = "foreign"; + if (fault === "assignee") o.state.blockedTask.assigneeAgentId = "foreign"; + if (fault === "currentStatus") o.state.blockedTask.status = "done"; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); +it("still rejects fake command exits, unchanged bytes and unobserved human handoff", () => { + const wrongExit = fixture(); wrongExit.verification[0]!.exitCode = 0; expect(gradeWorkTimeline(wrongExit).passed).toBe(false); + const sameBytes = fixture(); sameBytes.verification[1]!.sha256 = "before"; expect(gradeWorkTimeline(sameBytes).passed).toBe(false); + const wrongHuman = fixture(); (wrongHuman.timelineEvents[1] as Record).actorId = "foreign"; expect(gradeWorkTimeline(wrongHuman).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 0b35ff1b54e..453b33b3cc1 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -8,6 +8,24 @@ import { startTimelineVerifier, type VerificationReceipt } from "./internal-time import { localDay } from "./internal-routine-audit.js"; import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js"; interface Timeline extends InternalObservation { verification: VerificationReceipt[]; hiddenStatus: number; timelineEvents: Entity[] } +const objectRecord = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +/** Native outcomes are projected by the canonical committer, not an agent PATCH. */ +export function timelineStatusMilestone(row: Entity, task: Entity, run: Entity, workerId: string, status: "blocked" | "done") { + if (typeof workerId !== "string" || !workerId || !task.id || !run.id || row.action !== "issue.updated" || row.entityId !== task.id) return false; + // Preserve genuinely attributed legacy agent updates; system rows need the full native join. + if (row.actorType !== "system" && row.agentId === workerId && row.details?.status === status) return true; + const event = objectRecord(row), current = objectRecord(task), native = objectRecord(run), context = objectRecord(native.contextSnapshot), result = objectRecord(native.resultJson); + return typeof task.companyId === "string" && !!task.companyId && row.companyId === task.companyId && native.companyId === task.companyId + && task.status === status && task.assigneeAgentId === workerId && run.agentId === workerId && run.status === "succeeded" + && (context.issueId === task.id || context.taskId === task.id) + && (context.issueId === undefined || context.issueId === task.id) && (context.taskId === undefined || context.taskId === task.id) + && row.actorType === "system" && row.actorId === "native-status-committer" && event.runId === run.id + && row.details?.source === "native_status_decision" && row.details.toStatus === status + && typeof current.lastStatusDecisionId === "string" && !!current.lastStatusDecisionId + && row.details.decisionId === current.lastStatusDecisionId && result.decisionId === current.lastStatusDecisionId + && result.issueStatusAfter === status && result.issueStatusBefore === row.details.fromStatus + && typeof result.assessmentId === "string" && !!result.assessmentId && row.details.assessmentId === result.assessmentId; +} export function gradeWorkTimeline(value: unknown) { const c = checks(); const o = observation(value) as Timeline | undefined; c.add("real-timeline-and-command-receipts", !!o && Array.isArray(o.verification) && Array.isArray(o.timelineEvents) && typeof o.hiddenStatus === "number", "Real command exits, current-week state events and independent private access are required."); @@ -21,9 +39,9 @@ export function gradeWorkTimeline(value: unknown) { const today = String(o.state.today); const zone = String(o.state.timezone); const monday = new Date(`${today}T12:00:00Z`); monday.setUTCDate(monday.getUTCDate() - (monday.getUTCDay() + 6) % 7); c.add("ordered-current-week-native-milestones", events.length === 3 && [o.state.blockedEventId, o.state.handoffEventId, o.state.doneEventId].every(id => events.some(row => row.id === id)) && events.every(row => row.entityId === t.id && row.action === "issue.updated") - && events.some(row => row.id === o.state.blockedEventId && row.agentId === first.agentId && row.details?.status === "blocked") + && events.some(row => row.id === o.state.blockedEventId && timelineStatusMilestone(row, entity(o.state, "blockedTask"), first, first.agentId!, "blocked")) && events.some(row => row.id === o.state.handoffEventId && row.actorType === "user" && row.actorId === o.state.personId && row.details?.assigneeAgentId === second.agentId) - && events.some(row => row.id === o.state.doneEventId && row.agentId === second.agentId && row.details?.status === "done") + && events.some(row => row.id === o.state.doneEventId && timelineStatusMilestone(row, t, second, second.agentId!, "done")) && events.every(row => !!row.createdAt && localDay(row.createdAt, zone) >= monday.toISOString().slice(0, 10) && localDay(row.createdAt, zone) <= today) && Date.parse(events.find(row => row.id === o.state.blockedEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) && Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.doneEventId)!.createdAt!), "The saved actual blocked → human handoff → agent completion events fall in the person's real current week and retain chronological attribution."); @@ -55,17 +73,19 @@ function timeline(): ChatCatalogCase { verifier = await startTimelineVerifier(ctx, t.id, [worker.id, repair.id]); await ctx.input.api.patch(`/api/issues/${t.id}`, { status: "todo", assigneeAgentId: worker.id, description: "Build the initial launch.html prototype with a semantic h1 and an email input id=\"email\", deliberately leave the label pending for the later accessibility review. Run the authorized Launch verification app verify_launch_html on the actual saved bytes. If it fails, record its actual exit code and pause this task as blocked awaiting a label repair; do not pretend the command passed, fix the pending label yet, or mark it done. Do not create other tasks." }); const first = await pollUntil({ label: "Real launch verification failure and saved pause", deadlineAt: Date.now() + 180000, load: async () => ({ t: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), accept: row => row.t.status === "blocked" && row.runs.length === 1 && row.runs[0]?.status === "succeeded" && verifier!.receipts.some(r => r.runId === row.runs[0]?.id && r.exitCode === 1) }); - const blockedEvent = (await ctx.input.api.get(`/api/issues/${t.id}/activity`)).find(row => row.action === "issue.updated" && row.agentId === worker.id && row.details?.status === "blocked"); if (!blockedEvent) throw new Error("Missing actual blocked milestone"); + const firstRun = await ctx.input.api.get(`/api/heartbeat-runs/${first.runs[0]!.id}`); + const blockedEvent = (await ctx.input.api.get(`/api/issues/${t.id}/activity`)).find(row => timelineStatusMilestone(row, first.t, firstRun, worker.id, "blocked")); if (!blockedEvent) throw new Error("Missing actual blocked milestone"); await ctx.input.api.patch(`/api/issues/${t.id}`, { status: "todo", assigneeAgentId: repair.id, description: "Repair the existing launch.html prototype by adding a real label for the email input. Preserve its semantic heading. Run Launch verification verify_launch_html on the saved bytes, report its actual result, and complete only if the real command succeeds. The earlier failed check and pause remain history; do not rewrite them." }); const second = await pollUntil({ label: "Real human handoff and verified native repair", deadlineAt: Date.now() + 180000, load: async () => ({ t: await ctx.input.api.get(`/api/issues/${t.id}`), runs: await ctx.allRuns() }), accept: row => row.t.status === "done" && row.runs.length === 2 && row.runs.every(r => r.status === "succeeded") && verifier!.receipts.some(r => r.runId !== first.runs[0]?.id && r.exitCode === 0) }); - const events = await ctx.input.api.get(`/api/issues/${t.id}/activity`); const handoff = events.find(row => row.actorType === "user" && row.details?.assigneeAgentId === repair.id); const done = events.find(row => row.agentId === repair.id && row.details?.status === "done"); if (!handoff || !done) throw new Error("Missing actual handoff/completion milestone"); + const secondRun = await ctx.input.api.get(`/api/heartbeat-runs/${second.runs.find(row => row.id !== firstRun.id)!.id}`); + const events = await ctx.input.api.get(`/api/issues/${t.id}/activity`); const handoff = events.find(row => row.actorType === "user" && row.details?.assigneeAgentId === repair.id); const done = events.find(row => timelineStatusMilestone(row, second.t, secondRun, repair.id, "done")); if (!handoff || !done) throw new Error("Missing actual handoff/completion milestone"); const timezone = await ctx.input.page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone); const now = new Date(); const today = localDay(now.toISOString(), timezone); const zoneLabel = new Intl.DateTimeFormat("en-US", { timeZone: timezone, timeZoneName: "short" }).formatToParts(now).find(part => part.type === "timeZoneName")!.value; const session = await ctx.input.api.get<{ user?: { id: string } }>("/api/auth/get-session"); if (!session.user?.id) throw new Error("Timeline requires actual human handoff identity"); - const state = { project: p, task: second.t, prepTask, hiddenTask, firstRun: first.runs[0], secondRun: second.runs.find(row => row.id !== first.runs[0]!.id), blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id }; + const state = { project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); return { ...base, authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts"], + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, async cleanup(ctx) { try { await verifier?.close(); } finally { verifier = undefined; await cleanupPrivacyProbes(ctx); await owner?.dispose(); owner = undefined; } },