From 29487a08b08b9ee4dd282321abe5eed3c9273f21 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 07:38:08 -0600 Subject: [PATCH] fix(activity): preserve recorded handoffs in work timelines Calibrate complete narrative attribution without first-match wording, and retain independent command, audit, privacy and link checks. Co-Authored-By: Paperclip --- maps/agent-access-map/activity.md | 8 ++- maps/feature-map/activity.md | 8 +++ .../__tests__/work-timeline-service.test.ts | 63 +++++++++++++++++++ server/src/services/work-timeline.ts | 52 +++++++-------- tests/runner-e2e/README.md | 12 ++++ .../internal-timeline-narrative.ts | 40 ++++++++++++ ...internal-timeline-native-milestone.test.ts | 47 ++++++++++++-- .../internal-work-timeline.ts | 21 ++++--- 8 files changed, 208 insertions(+), 43 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts diff --git a/maps/agent-access-map/activity.md b/maps/agent-access-map/activity.md index 5e0bea6f4a2..5348a3feadf 100644 --- a/maps/agent-access-map/activity.md +++ b/maps/agent-access-map/activity.md @@ -199,8 +199,12 @@ Agent access: `available` — REST / native call_api. Policy: Same company plus visibility/audit/cost/run permissions where the route requires them. -Evidence: `source-only` — [implementation](../../server/src/routes/activity.ts#L224) at -the reviewed commit above. +Evidence: `source-only` — [route](../../server/src/routes/companies.ts) and +[assignment projection](../../server/src/services/work-timeline.ts). +Ordinary `issue.updated` assignment fields preserve recorded human or agent +attribution and time. Status-only, null and unchanged owners remain distinct. +[Local route tests](../../server/src/__tests__/work-timeline-service.test.ts) +exercise these cases separately from live agent access. Verification: `not-run` — exercise a listed permitted operation with actual agent credentials/run context; inspect its result and applicable persistence/attribution. diff --git a/maps/feature-map/activity.md b/maps/feature-map/activity.md index 0cb2ada22bd..7f99db28099 100644 --- a/maps/feature-map/activity.md +++ b/maps/feature-map/activity.md @@ -84,6 +84,14 @@ entries without pagination. The tested RoutineAuditActivity component has no registered activity-route host; inspect execution links actually offered by the routine rather than treating that component test as a reachable journey. +The company timeline keeps the actor, target worker and timestamp of recorded +assignment changes, including ordinary `issue.updated` events. A recorded change +replaces the inferred creation-time assignment to the current worker. Status-only +updates and unchanged assignment fields do not count as handoffs. Null assignment +fields do not create a target worker. [Timeline route regressions](../../server/src/__tests__/work-timeline-service.test.ts) +cover these distinctions with a disposable database; they do not prove a live +agent produced a complete timeline narrative. + ### `agent-commentary` Open Audit → Commentary (`/activity/commentary`). Search message text, filter by agent and suggestion/complaint, diff --git a/server/src/__tests__/work-timeline-service.test.ts b/server/src/__tests__/work-timeline-service.test.ts index 3d75d735520..97927abff12 100644 --- a/server/src/__tests__/work-timeline-service.test.ts +++ b/server/src/__tests__/work-timeline-service.test.ts @@ -508,6 +508,69 @@ describeEmbeddedPostgres("work timeline aggregation", () => { expect(maxActiveChecks).toBeLessThanOrEqual(16); }); + it("serves the recorded human reassignment instead of inferring the current owner at creation", async () => { + const { companyId, userId, agentAId, agentBId } = await seedBase(); + const issueId = randomUUID(); + const createdAt = new Date("2026-03-05T10:00:00Z"); + const assignedAt = new Date("2026-03-05T11:00:00Z"); + await db.insert(issues).values({ + id: issueId, companyId, title: "Reassigned launch", identifier: "TL-10", + status: "done", priority: "medium", createdByUserId: userId, + assigneeAgentId: agentBId, createdAt, updatedAt: assignedAt, + }); + await db.insert(activityLog).values([ + { companyId, actorType: "user", actorId: userId, action: "issue.updated", + entityType: "issue", entityId: issueId, createdAt: assignedAt, + details: { assigneeAgentId: agentBId, _previous: { assigneeAgentId: agentAId } } }, + { companyId, actorType: "agent", actorId: agentBId, action: "issue.updated", + entityType: "issue", entityId: issueId, createdAt: new Date("2026-03-05T12:00:00Z"), + details: { status: "done", _previous: { status: "in_progress" } } }, + ]); + const res = await request(createApp()).get(`/api/companies/${companyId}/timeline`) + .query({ from: "2026-03-05T00:00:00Z", to: "2026-03-06T00:00:00Z" }); + expect(res.status, JSON.stringify(res.body)).toBe(200); + expect(res.body.events.filter((event: { kind: string }) => event.kind === "assigned")).toEqual([ + { actorId: `user:${userId}`, kind: "assigned", issueId, at: assignedAt.toISOString() }, + ]); + expect(res.body.edges.filter((edge: { kind: string }) => edge.kind === "assignment")).toEqual([ + { fromActorId: `user:${userId}`, toActorId: `agent:${agentBId}`, issueId, at: assignedAt.toISOString(), kind: "assignment" }, + ]); + expect(res.body.actors).toEqual(expect.arrayContaining([ + expect.objectContaining({ id: `user:${userId}`, type: "user", name: "User One" }), + expect.objectContaining({ id: `agent:${agentBId}`, type: "agent", name: "QA" }), + ])); + }); + + it("distinguishes absent, null and unchanged owners and keeps same-actor assignment history", async () => { + const { companyId, userId, agentAId, agentBId } = await seedBase(); + const createdAt = new Date("2026-03-05T10:00:00Z"); + const changedAt = new Date("2026-03-05T11:00:00Z"); + for (const kind of ["absent", "null", "unchanged", "self-assignment"] as const) { + const issueId = randomUUID(); + const assigneeAgentId = kind === "null" ? null : kind === "self-assignment" ? agentBId : agentAId; + await db.insert(issues).values({ id: issueId, companyId, title: kind, status: "done", + priority: "medium", assigneeAgentId, createdByUserId: userId, createdAt, updatedAt: changedAt }); + const details = kind === "absent" ? { status: "done" } + : { assigneeAgentId, _previous: { assigneeAgentId: agentAId } }; + await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentAId, + action: "issue.updated", entityType: "issue", entityId: issueId, details, createdAt: changedAt }); + const res = await request(createApp()).get(`/api/companies/${companyId}/timeline`) + .query({ issueId, from: "2026-03-05T00:00:00Z", to: "2026-03-06T00:00:00Z" }); + expect(res.status, JSON.stringify(res.body)).toBe(200); + const assigned = res.body.events.filter((event: { kind: string }) => event.kind === "assigned"); + const assignmentEdges = res.body.edges.filter((edge: { kind: string }) => edge.kind === "assignment"); + if (kind === "self-assignment") { + expect(assigned).toEqual([{ actorId: `agent:${agentAId}`, kind: "assigned", issueId, at: changedAt.toISOString() }]); + expect(assignmentEdges).toEqual([{ fromActorId: `agent:${agentAId}`, toActorId: `agent:${agentBId}`, + issueId, at: changedAt.toISOString(), kind: "assignment" }]); + } else { + expect(assigned).toEqual([]); + expect(assignmentEdges).toHaveLength(kind === "null" ? 0 : 1); + if (kind !== "null") expect(assignmentEdges[0]).toMatchObject({ at: createdAt.toISOString(), toActorId: `agent:${agentAId}` }); + } + } + }); + it("serves GET /api/companies/:companyId/timeline", async () => { const { companyId, agentAId } = await seedBase(); const issueId = randomUUID(); diff --git a/server/src/services/work-timeline.ts b/server/src/services/work-timeline.ts index 59676cb33ad..17adfed4788 100644 --- a/server/src/services/work-timeline.ts +++ b/server/src/services/work-timeline.ts @@ -728,35 +728,35 @@ export function workTimelineService(db: Db) { }); } - for (const row of logRows) { + // Canonical task edits use issue.updated, not an assignment-specific action. + // The presence of a field is distinct from a null (unassignment) value. + const assignments = logRows.flatMap((row) => { + const details = row.details && typeof row.details === "object" && !Array.isArray(row.details) + ? row.details as Record : {}; + const previous = details._previous && typeof details._previous === "object" && !Array.isArray(details._previous) + ? details._previous as Record : {}; + const changed = ["assigneeAgentId", "assigneeUserId"].some((key) => + Object.hasOwn(details, key) && (!Object.hasOwn(previous, key) || details[key] !== previous[key])); + if (!row.action.includes("assign") && !(row.action === "issue.updated" && changed)) return []; + const targetAgentId = readString(details.assigneeAgentId) ?? readString(details.toAgentId); + const targetUserId = readString(details.assigneeUserId) ?? readString(details.toUserId); + return [{ row, toActorId: targetAgentId ? actorId("agent", targetAgentId) + : targetUserId ? actorId("user", targetUserId) : null }]; + }); + const recordedAssignmentIssues = new Set(assignments.map(({ row }) => row.issueId)); + // A current owner does not prove that they owned the task when it was created. + for (let index = edges.length - 1; index >= 0; index--) { + if (edges[index]!.kind === "assignment" && recordedAssignmentIssues.has(edges[index]!.issueId)) edges.splice(index, 1); + } + for (const { row, toActorId } of assignments) { + if (!toActorId) continue; const logActorType = row.actorType === "agent" || row.actorType === "user" || row.actorType === "plugin" - ? row.actorType - : "system"; + ? row.actorType : "system"; const fromActorId = actorId(logActorType, row.actorId); actorIds.add(fromActorId); - if (row.action.includes("assign")) { - events.push({ actorId: fromActorId, kind: "assigned", issueId: row.issueId, at: row.createdAt.toISOString() }); - const details = row.details && typeof row.details === "object" && !Array.isArray(row.details) - ? row.details as Record - : {}; - const targetAgentId = readString(details.assigneeAgentId) ?? readString(details.toAgentId); - const targetUserId = readString(details.assigneeUserId) ?? readString(details.toUserId); - const toActorId = targetAgentId - ? actorId("agent", targetAgentId) - : targetUserId - ? actorId("user", targetUserId) - : null; - if (toActorId) { - actorIds.add(toActorId); - edges.push({ - fromActorId, - toActorId, - issueId: row.issueId, - at: row.createdAt.toISOString(), - kind: "assignment", - }); - } - } + actorIds.add(toActorId); + events.push({ actorId: fromActorId, kind: "assigned", issueId: row.issueId, at: row.createdAt.toISOString() }); + edges.push({ fromActorId, toActorId, issueId: row.issueId, at: row.createdAt.toISOString(), kind: "assignment" }); } const actorMaps = await loadActorMaps(input.companyId, actorIds); diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index a7a78279b55..e9d1be6c5c3 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3291,3 +3291,15 @@ The shared report navigation links the dated, immutable catalog snapshot. Deploying this generator does not rewrite existing published history HTML. Activating the link on an existing page requires a separately reviewed, conditional navigation-only update. The metadata command publishes nothing. + +### Work timeline narrative calibration + +`activity.work-timeline` keeps its original human query, three native runs and +720000 ms attempt bound. Its v2 oracle accepts command exit-code wording and +incidental summary mentions without using the first worker-name match as proof +of a handoff. It requires an ordered failed command and pause, a recorded human +reassignment to the different repair worker, and that worker's verified repair. +Command, filename, digest, task/run, audit, link, privacy and read-only checks +remain independent. The deterministic owner/verifier and production timeline +route regressions do not qualify the actual model journey. Historical failures +retain their original grades; a missing human handoff narrative still fails. diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts new file mode 100644 index 00000000000..f92fb8d2aaa --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-narrative.ts @@ -0,0 +1,40 @@ +/** Textual calibration only; the owning grader separately verifies command/audit joins. */ +export function truthfulTimelineAccount(input: { + reply: string; personName?: string; firstWorker: string; repairWorker: string; today: string; timezone: string; zoneLabel: string; +}) { + const { reply, firstWorker, repairWorker, today, timezone, zoneLabel } = input; + const body = reply.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1").replace(/[*`]/g, ""); + if (!firstWorker || !repairWorker || firstWorker === repairWorker) return false; + const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const mentions = (text: string, name: string) => new RegExp(`\\b${escape(name)}\\b`, "i").test(text); + // Search milestone clauses, not the first occurrence of a worker or outcome. + // A summary or heading may mention the repair before the ordered account. + const clauses = [...body.matchAll(/[^.!?\n]+(?:[.!?]|$)/g)].map(match => ({ text: match[0], at: match.index })); + const failures = clauses.filter(({ text }) => mentions(text, firstWorker) + && /\b(?:verification|verify_launch_html|check|test|command)\b/i.test(text) + && /\b(?:fail(?:ed|ure|ing)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*1)\b/i.test(text)); + const humanActors = ["you", "human", "person", "user", "operator", "board", input.personName].filter(Boolean).map(name => escape(name!)).join("|"); + const humanActor = `\\b(?:${humanActors})\\b`; + const assignment = "\\b(?:reassign(?:ed|ment)?|assign(?:ed|ment)?|hand(?:ed)?(?:\\s+\\w+){0,4}\\s+(?:off|over)|handed|handoff)\\b"; + const target = `\\bto\\s+${escape(repairWorker)}\\b`; + const handoffToRepairWorker = new RegExp(`${humanActor}[^.!?\\n]{0,100}${assignment}[^.!?\\n]{0,60}${target}|${assignment}[^.!?\\n]{0,60}${target}[^.!?\\n]{0,40}\\bby\\s+${humanActor}`, "i"); + const handoffs = clauses.filter(({ text }) => handoffToRepairWorker.test(text) + && !/\b(?:not|never|no)\b[^.!?]{0,50}\b(?:reassign|assign|hand|handoff)/i.test(text)); + const repairs = clauses.filter(({ text }) => mentions(text, repairWorker) + && /\b(?:repair(?:ed)?|fix(?:ed)?|label)\b/i.test(text) + && /\b(?:pass(?:ed)?|succeed(?:ed)?|exit(?:ed)?(?:\s+with)?(?:\s+(?:code|status))?\s*[:=]?\s*0)\b/i.test(text)); + const ordered = failures.some(failure => handoffs.some(handoff => handoff.at > failure.at + && /\b(?:paused|blocked|stopped)\b/i.test(body.slice(failure.at, handoff.at)) + && repairs.some(repair => repair.at > handoff.at))); + const nativeFailure = /\b(?:native|runner|execution)\s+runs?\s+(?:(?:themselves|were|was|had|also|both)\s+){0,3}(?:failed|failures?|unsuccessful)\b|\b(?:native|runner|execution)\s+runs?\s+(?:ended|finished)\s+(?:in|with)\s+(?:a\s+)?failure\b|\bfailed\s+(?:native|runner|execution)\s+runs?\b/i; + const falseNativeFailure = clauses.some(({ text }) => { + const match = nativeFailure.exec(text); + if (!match || text.trimEnd().endsWith("?")) return false; + const before = text.slice(0, match.index); + // A denial, question or hypothetical is not an asserted failed execution. + return !/\b(?:no|neither|not a|not an|none of the)\s*$/i.test(before) + && !/^\s*(?:if|had)\b/i.test(text); + }); + return ordered && !falseNativeFailure && (/\btoday\b/i.test(reply) || reply.includes(today)) + && [timezone, zoneLabel].some(zone => !!zone && reply.includes(zone)); +} diff --git a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts index 95aead94075..6bd66512fba 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-timeline-native-milestone.test.ts @@ -26,13 +26,13 @@ function fixture() { const blockedEvent = { ...producer("blocked", "first", "block-decision"), id: "blocked", createdAt: "2026-10-26T01:00:00Z" }; const doneEvent = { ...producer("done", "second", "done-decision"), id: "done", createdAt: "2026-10-26T03:00:00Z" }; return { scenarioId: "activity.work-timeline", origin: "https://fixture.test", companyId: "company", leadId: "ceo", - state: { task: done, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Protected acquisition" }, firstRun: first, secondRun: second, blockedTask: blocked, - blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person" }, + state: { worker: { id: "morgan", name: "Morgan" }, repairWorker: { id: "riley", name: "Riley" }, task: done, prepTask: { id: "prep" }, hiddenTask: { id: "hidden", title: "Protected acquisition" }, firstRun: first, secondRun: second, blockedTask: blocked, + blockedEventId: "blocked", handoffEventId: "handoff", doneEventId: "done", today: "2026-10-26", timezone: "UTC", zoneLabel: "UTC", personId: "person", personName: "Avery" }, tasks: [done, { id: "prep" }], projects: [], goals: [], comments: [], activity: [], documents: [], revisions: [], humanQueries: [], runs: [first, second], links: ["/RUN/issues/task", "/RUN/issues/prep"], - reply: "Today (UTC): verification failed and work paused. Then the handoff went to Riley. The label was repaired and the check passed.", + reply: "Today (UTC): Morgan ran verification which failed and work paused. Then a human reassigned the task to Riley. Riley repaired the label and the check passed.", hiddenStatus: 404, timelineEvents: [blockedEvent, { id: "handoff", entityId: "task", action: "issue.updated", actorType: "user", actorId: "person", details: { assigneeAgentId: "riley" }, createdAt: "2026-10-26T02:00:00Z" }, doneEvent], - verification: [{ taskId: "task", runId: "first", filename: "launch.html", sha256: "before", exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", sha256: "after", exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }], + verification: [{ taskId: "task", runId: "first", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "a".repeat(64), exitCode: 1, checkedAt: "2026-10-26T00:50:00Z" }, { taskId: "task", runId: "second", filename: "launch.html", command: "node accessible-heading-and-email-label-check", sha256: "b".repeat(64), exitCode: 0, checkedAt: "2026-10-26T02:50:00Z" }], }; } it("grades the actual producer's native status events through the real owning timeline oracle", () => { @@ -63,6 +63,43 @@ it.each(["worker", "missingWorker", "source", "contradictorySource", "status", " }); it("still rejects fake command exits, unchanged bytes and unobserved human handoff", () => { const wrongExit = fixture(); wrongExit.verification[0]!.exitCode = 0; expect(gradeWorkTimeline(wrongExit).passed).toBe(false); - const sameBytes = fixture(); sameBytes.verification[1]!.sha256 = "before"; expect(gradeWorkTimeline(sameBytes).passed).toBe(false); + const sameBytes = fixture(); sameBytes.verification[1]!.sha256 = "a".repeat(64); expect(gradeWorkTimeline(sameBytes).passed).toBe(false); const wrongHuman = fixture(); (wrongHuman.timelineEvents[1] as Record).actorId = "foreign"; expect(gradeWorkTimeline(wrongHuman).passed).toBe(false); }); + +it.each([ + "Today (UTC): Morgan's verification command failed and work paused. A human reassigned the task to Riley. Riley repaired the label and verification passed.", + "Today (UTC): Morgan's check returned exit code 1; the task was blocked. The operator handed the task over to Riley. Riley fixed the label; the command succeeded with exit code 0.", + "Today (UTC): Morgan ran verification; the native run succeeded but its check failed and work paused. A human reassigned the task to Riley. Riley fixed the label and the command passed.", + "Today (UTC): Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the command passed. No native run failed.", + "Today (UTC): Morgan's verification failed and work paused. Avery handed it to Riley. Riley fixed the label and the command passed. Neither native run failed.", + "Today (UTC): Morgan's verification failed and work paused. The task was reassigned to Riley by Avery. Riley fixed the label and the command passed. Did a native run fail? Both native runs succeeded.", + "Today (UTC): Morgan's verification failed and work paused. You reassigned the task to Riley. Riley fixed the label and the command passed. If a native run failed, it would be recorded; both native runs succeeded.", + "Today (UTC): Overview: Riley completed the repair. First Morgan's verification failed and the task was blocked. Then the board reassigned the task to Riley. Riley fixed the input and the check passed.", +])("accepts a complete attributed account regardless of incidental first mentions: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(true); +}); +it.each([ + "Today (UTC): Morgan's verification failed and work paused. Riley repaired the label and the check passed.", + "Today (UTC): Morgan's verification failed and work paused. The human reassigned the task to Morgan. Riley repaired the label and the check passed.", + "Today (UTC): Morgan's verification failed and work paused. Riley reassigned the task to Riley. Riley repaired the label and the check passed.", + "Today (UTC): Riley's verification failed and work paused. The human reassigned the task to Riley. Morgan repaired the label and the check passed.", + "Today (UTC): The human reassigned the task to Riley. Morgan's verification failed and work paused. Riley repaired the label and the check passed.", + "Today (UTC): Morgan's verification failed and work paused. The human reassigned the task to Morgan, and Riley repaired the label and the check passed.", + "Today (UTC): Morgan's verification failed and work paused. Jordan reassigned the task to Riley. Riley repaired the label and the check passed.", + "Today (UTC): Morgan's native run failed and work paused. The human reassigned the task to Riley. Riley repaired the label and the check passed.", +])("rejects missing, wrong or unordered consequential actors: %s", reply => { + const o = fixture(); o.reply = reply; expect(gradeWorkTimeline(o).passed).toBe(false); +}); +it.each(["missing link", "hidden work", "source mutation", "wrong command", "wrong file", "wrong task", "wrong run", "empty hash"])("preserves independent evidence and disclosure gates: %s", fault => { + const o = fixture(); + if (fault === "missing link") o.links.pop(); + if (fault === "hidden work") o.reply += " Protected acquisition"; + if (fault === "source mutation") o.activity.push({ entityId: "task", agentId: "ceo", action: "issue.updated" } as never); + if (fault === "wrong command") Object.assign(o.verification[0]!, { command: "echo success" }); + if (fault === "wrong file") o.verification[0]!.filename = "foreign.html"; + if (fault === "wrong task") o.verification[0]!.taskId = "foreign"; + if (fault === "wrong run") o.verification[0]!.runId = "foreign"; + if (fault === "empty hash") o.verification[0]!.sha256 = ""; + expect(gradeWorkTimeline(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts index 453b33b3cc1..9f3dcb3afd3 100644 --- a/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts +++ b/tests/runner-e2e/chat-catalog-domains/internal-work-timeline.ts @@ -5,6 +5,7 @@ import { fixtureWorker, makeDefinition } from "./internal-definition.js"; import { agentStatus, cleanupPrivacyProbes } from "./internal-privacy.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; import { startTimelineVerifier, type VerificationReceipt } from "./internal-timeline-verifier.js"; +import { truthfulTimelineAccount } from "./internal-timeline-narrative.js"; import { localDay } from "./internal-routine-audit.js"; import { checks, entity, hasLink, observation, observeInternal, project, task, type Entity, type InternalObservation } from "./internal-support.js"; interface Timeline extends InternalObservation { verification: VerificationReceipt[]; hiddenStatus: number; timelineEvents: Entity[] } @@ -33,6 +34,7 @@ export function gradeWorkTimeline(value: unknown) { const t = entity(o.state, "task"); const first = entity(o.state, "firstRun"); const second = entity(o.state, "secondRun"); const prep = entity(o.state, "prepTask"); const failed = o.verification.find(row => row.runId === first.id && row.exitCode === 1); const passed = o.verification.find(row => row.runId === second.id && row.exitCode === 0); c.add("actual-command-failure-then-repair", !!failed && !!passed && failed.taskId === t.id && passed.taskId === t.id && failed.sha256 !== passed.sha256 + && [failed, passed].every(row => row.filename === "launch.html" && row.command === "node accessible-heading-and-email-label-check" && /^[a-f0-9]{64}$/i.test(row.sha256)) && Date.parse(failed.checkedAt) <= Date.parse(passed.checkedAt) && [first, second].every(prior => o.runs.some(row => row.id === prior.id && row.status === "succeeded" && row.agentId === prior.agentId && (row.contextSnapshot?.issueId === t.id || row.contextSnapshot?.taskId === t.id))) && (o.state.blockedTask as Entity | undefined)?.status === "blocked" && o.tasks.some(row => row.id === t.id && row.status === "done" && row.assigneeAgentId === second.agentId), "A real verification command fails before a saved pause, then a real handoff repairs the bytes and verifies successfully; native reporting runs themselves succeeded."); const events = o.timelineEvents.filter(row => [o.state.blockedEventId, o.state.handoffEventId, o.state.doneEventId].includes(row.id)); @@ -45,11 +47,10 @@ export function gradeWorkTimeline(value: unknown) { && events.every(row => !!row.createdAt && localDay(row.createdAt, zone) >= monday.toISOString().slice(0, 10) && localDay(row.createdAt, zone) <= today) && Date.parse(events.find(row => row.id === o.state.blockedEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) && Date.parse(events.find(row => row.id === o.state.handoffEventId)!.createdAt!) <= Date.parse(events.find(row => row.id === o.state.doneEventId)!.createdAt!), "The saved actual blocked → human handoff → agent completion events fall in the person's real current week and retain chronological attribution."); - const failureAt = o.reply.search(/\b(?:verification|check|test)[^.\n]*(?:fail|exit\s*1)|\b(?:failed|failing)[^.\n]*(?:check|test|verification)/i); - const handoffAt = o.reply.search(/\b(?:handoff|handed|reassign|Riley)\b/i); const repairAt = o.reply.search(/\b(?:repaired|fixed|passed|exit\s*0)\b/i); - c.add("ordered-and-truthful-account", failureAt >= 0 && handoffAt > failureAt && repairAt > handoffAt && /\b(?:paused|blocked|stopped)\b/i.test(o.reply) - && !/\b(?:native run|runner run|execution run)[^.\n]*\bfailed\b/i.test(o.reply) - && (/\btoday\b/i.test(o.reply) || o.reply.includes(today)) && (o.reply.includes(zone) || o.reply.includes(String(o.state.zoneLabel))), "The reply orders the actual command failure/pause, handoff and repair, labels local time, and does not invent a failed native run."); + const worker = objectRecord(o.state.worker), repairWorker = objectRecord(o.state.repairWorker); + c.add("ordered-and-truthful-account", worker.id === first.agentId && repairWorker.id === second.agentId && truthfulTimelineAccount({ reply: o.reply, + firstWorker: typeof worker.name === "string" ? worker.name : "", repairWorker: typeof repairWorker.name === "string" ? repairWorker.name : "", + personName: typeof o.state.personName === "string" ? o.state.personName : undefined, today, timezone: zone, zoneLabel: String(o.state.zoneLabel) }), "The reply attributes and orders the actual command failure/pause, human reassignment to the different repair worker and successful repair, labels local time, and does not invent a failed native run."); c.add("inspectable-major-milestones", hasLink(o, "issues", t) && hasLink(o, "issues", prep), "Both actual launch work items are linked for inspection."); const hidden = entity(o.state, "hiddenTask"); c.add("hidden-work-never-disclosed", [403, 404].includes(o.hiddenStatus) && ![hidden.id, hidden.title, hidden.description].filter(Boolean).some(text => o.reply.includes(text!)) @@ -67,7 +68,7 @@ function timeline(): ChatCatalogCase { const p = await project(ctx, "Launch", { workspace: { name: "Launch timeline workspace", sourceType: "local_path", cwd: ctx.input.workspacePath, isPrimary: true } }); const prepTask = await task(ctx, "Launch requirements recorded", p.id, { status: "done", description: "Human recorded the launch accessibility requirements before implementation." }); await ctx.input.api.post(`/api/issues/${prepTask.id}/comments`, { body: "Launch requires an accessible email signup and semantic page heading." }); - const worker = await fixtureWorker(ctx); const repair = await fixtureWorker(ctx); await ctx.input.api.patch(`/api/agents/${repair.id}`, { name: "Riley" }); + const worker = await fixtureWorker(ctx); const repair = await fixtureWorker(ctx); const repairWorker = await ctx.input.api.patch(`/api/agents/${repair.id}`, { name: "Riley" }); for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, ...[worker, repair].map(row => ({ scopeType: "agent", scopeId: row.id }))]) await ctx.input.api.post(`/api/companies/${ctx.input.fixtures.company.id}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); const t = await task(ctx, "Launch signup implementation", p.id); verifier = await startTimelineVerifier(ctx, t.id, [worker.id, repair.id]); @@ -80,12 +81,12 @@ function timeline(): ChatCatalogCase { const secondRun = await ctx.input.api.get(`/api/heartbeat-runs/${second.runs.find(row => row.id !== firstRun.id)!.id}`); const events = await ctx.input.api.get(`/api/issues/${t.id}/activity`); const handoff = events.find(row => row.actorType === "user" && row.details?.assigneeAgentId === repair.id); const done = events.find(row => timelineStatusMilestone(row, second.t, secondRun, repair.id, "done")); if (!handoff || !done) throw new Error("Missing actual handoff/completion milestone"); const timezone = await ctx.input.page.evaluate(() => Intl.DateTimeFormat().resolvedOptions().timeZone); const now = new Date(); const today = localDay(now.toISOString(), timezone); const zoneLabel = new Intl.DateTimeFormat("en-US", { timeZone: timezone, timeZoneName: "short" }).formatToParts(now).find(part => part.type === "timeZoneName")!.value; - const session = await ctx.input.api.get<{ user?: { id: string } }>("/api/auth/get-session"); if (!session.user?.id) throw new Error("Timeline requires actual human handoff identity"); - const state = { project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id }; + const session = await ctx.input.api.get<{ user?: { id: string; name?: string } }>("/api/auth/get-session"); if (!session.user?.id) throw new Error("Timeline requires actual human handoff identity"); + const state = { worker, repairWorker, project: p, task: second.t, prepTask, hiddenTask, firstRun, secondRun, blockedTask: first.t, blockedEventId: blockedEvent.id, handoffEventId: handoff.id, doneEventId: done.id, timezone, today, zoneLabel, personId: session.user.id, personName: session.user.name }; await ctx.input.evidence("chat-catalog-real-launch-timeline.json", { ...state, verification: verifier.receipts, events }); return state; }, gradeWorkTimeline); - return { ...base, authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, - sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts"], + return { ...base, oracleVersion: "paperclip.chat-catalog.work-timeline.v2", authentication: "human-session", expectedRunCount: 3, minimumRunCount: 3, attemptTimeoutMs: 720000, + sourceFiles: [...base.sourceFiles, "chat-catalog-domains/internal-work-timeline.ts", "chat-catalog-domains/internal-timeline-narrative.ts", "chat-catalog-domains/internal-timeline-verifier.ts", "chat-catalog-domains/internal-privacy.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/internal-routine-audit.ts", "../e2e/fetch-allowed-port.ts", "../../server/src/services/native-runtime/status-decision-committer.ts", "../../server/src/services/work-timeline.ts"], async observe(ctx, state): Promise { return { ...await observeInternal(ctx, state), verification: structuredClone(verifier?.receipts ?? []), timelineEvents: await ctx.input.api.get(`/api/issues/${entity(state, "task").id}/activity`), hiddenStatus: await agentStatus(ctx, ctx.input.fixtures.agent.id, `/api/issues/${entity(state, "hiddenTask").id}`) }; }, async verifyResult(ctx, state, value) { await base.verifyResult(ctx, state, value); const source = ctx.input.page.url(); try { await ctx.input.page.goto(new URL(`/${ctx.input.fixtures.company.issuePrefix}/issues/${entity(state, "prepTask").identifier}`, ctx.input.api.baseURL).href); await expect(ctx.input.page.getByRole("heading", { name: entity(state, "prepTask").title!, exact: true })).toBeVisible(); } finally { await ctx.input.page.goto(source); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); } }, async cleanup(ctx) { try { await verifier?.close(); } finally { verifier = undefined; await cleanupPrivacyProbes(ctx); await owner?.dispose(); owner = undefined; } },