diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index ac4025b0874..3db04081944 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1555,6 +1555,21 @@ worked when the clicked task meets the same completion rule. The banner list fol `GET /api/companies/:id/assigned-work-invariant` at the end of the wave: `violationCount`, `violations` by reason, `agentsWithStalledSlots` and `openTasks`. +The summary also lists, by start time, every run that started after the task completed, under "Runs started after +the task completed" (`extraTurns`, with counts by wake reason in `extraTurnsByReason`). A run queued after completion +that never started is listed on its own (`queuedAfterDone`). + +The completion time comes from the product's own record, in this order: + +1. the earliest accepted run result whose `reportedWorkDisposition` is `done`; +2. the issue's `completedAt`; +3. the first bare `DONE-` comment, only when neither exists. + +Each line prints the DONE comment time too when it differs, and gives the run's wake reason, source, trigger and +requesting actor from the saved wake evidence. A bare DONE comment posted by a run whose accepted result was not `done` +(for example `blocked` while a parent waits on its subtasks) is reported on its own, under "DONE comment posted before +the task completed" (`doneBeforeCompletion`). + **Grade.** The usual R1–R6, U1–U5 and T rules apply. A needed Retry still fails R3. The T-oracle checks the result, not the status: @@ -1565,7 +1580,7 @@ not the status: | `rounds_once` | a round line appears twice (a tool ran again) | | `done_comment_posted` | no comment's whole body is `DONE-`; a summary that quotes the marker does not count | | `done_comment_once` | that comment was posted more than once | -| `completion_once` | more than one run posted a comment that mentions `DONE-` (a duplicate completion), or the comments could not be read | +| `completion_once` | more than one run completed the task (a duplicate completion), or the comments could not be read. A run completes the task when its accepted result has disposition `done`, or when it posts a comment that mentions `DONE-` at or after the completion time. Without accepted results in the wave evidence, the bare DONE comment counts instead. A progress or blocked comment that says it will post the marker later is not a completion | | `follow_up_posted` | a planned follow-up was never posted (the detail lists the post errors) | | `follow_up_served` | a follow-up was posted during a run and the ledger lacks its line | | `subtask__done`, `parent_read_subtask_` | a subtask is missing, not done or its ledger fails, or the parent did not report its last hash | @@ -1978,7 +1993,8 @@ re-arms the task's own triggered faults instead of using their offsets. - each run's `context_snapshot`, invocation source and trigger; - the `agent_wakeup_requests` row behind each run and every wake for the issue: reason, source, trigger detail, requesting actor, idempotency key, payload and times; - - the issue's status changes from the activity log. + - the issue's status changes from the activity log: only `issue.*` entries whose status is an issue status, so a + `runner.api_called` entry with an HTTP status is not mistaken for one. Each wake and run also names the server generations alive when it was created (`servingAtCreate`): the pid, boot id, epoch and process space from `controller_instances`, and the harness's own primary or secondary role during an diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 3d52bfd1a09..8329327fd12 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -1,9 +1,12 @@ import { describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; -import { collectStagingMixArtifacts, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, type StagingMixTask } from "./staging-mix.js"; import { - bannerSink, issueView, readAssignedWorkInvariant, retryAfterSettleStage, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, - type RetryStageObserver, + collectStagingMixArtifacts, completedAtAnchor, doneCommentsBeforeCompletion, doneMarker, expectedRoundHash, gradeStagingMix, planStagingMixWave, ROUND_SECONDS, + stagingMixDescription, type CompletionEvidence, type StagingMixTask, +} from "./staging-mix.js"; +import { + bannerSink, commentEvidence, evidenceCompletion, extraTurnLine, extraTurnsByReason, issueView, readAssignedWorkInvariant, retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, + stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; @@ -158,9 +161,10 @@ describe("staging-mix Retry policy", () => { describe("staging-mix completion comments", () => { const nonce = "b3d70e77"; - const base = (comments: Array<{ body: string; runId: string | null }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, + const base = (comments: Array<{ body: string; runId: string | null; at?: string }>): OracleArtifacts => ({ issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: comments.map(comment => comment.body), files: {}, commands: {}, workspacePath: null, - stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId }))) }); + stagingMix: collectStagingMixArtifacts({} as TaskRecord, comments.map(comment => ({ body: comment.body, authorAgentId: "a", createdByRunId: comment.runId, + ...(comment.at ? { createdAt: comment.at } : {}) }))) }); const check = (artifacts: OracleArtifacts, name: string) => gradeStagingMix(nonce, task(), artifacts).checks.find(entry => entry.name === name)!; it("counts only the DONE comment itself, not a closing summary that quotes it", () => { @@ -182,13 +186,114 @@ describe("staging-mix completion comments", () => { expect(check(unread, "done_comment_posted").ok).toBe(true); }); + it("does not count a blocked run's promise to post DONE later as a completion (w-20261112T223803-s7101 t001)", () => { + const comments = [ + { body: "Prepared Parent ledger log for this response.", runId: "37724b59-4e6a", at: "2026-11-12T23:12:54.696Z" }, + { body: doneMarker(nonce), runId: "37724b59-4e6a", at: "2026-11-12T23:12:44.587Z" }, + { body: "Blocked report accepted. Summary of progress so far:\n\n- Completed all 4 parent rounds.\n\nParent now waits on both subtasks to finish. " + + `When they complete, I'll read each subtask's ledger hash, save this task's \`ledger\` document, post \`${doneMarker(nonce)}\`, and mark the task done.`, + runId: "fcc55827-ab78", at: "2026-11-12T22:56:04.145Z" }, + ]; + expect(check(base(comments), "completion_once")).toMatchObject({ ok: true, detail: "runs=1" }); + expect(check(base(comments), "done_comment_once")).toMatchObject({ ok: true, detail: "comments=1" }); + expect(check(base(comments), "done_comment_posted").ok).toBe(true); + }); + it("flags a completion summary repeated by a different run as a duplicate completion", () => { const comments = [ - { body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce" }, - { body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13" }, - { body: doneMarker(nonce), runId: "13de31be-fb13" }, + { body: `CHA-1 complete. Posted exactly one comment ${doneMarker(nonce)} and marked the task done.`, runId: "413ddcf2-ffce", at: "2026-11-12T18:10:00.000Z" }, + { body: `CHA-1 complete. Posted \`${doneMarker(nonce)}\`.`, runId: "13de31be-fb13", at: "2026-11-12T18:05:01.000Z" }, + { body: doneMarker(nonce), runId: "13de31be-fb13", at: "2026-11-12T18:05:00.000Z" }, ]; expect(check(base(comments), "done_comment_once").ok).toBe(true); - expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 413ddcf2, 13de31be each reported completion" }); + expect(check(base(comments), "completion_once")).toMatchObject({ ok: false, detail: "duplicate completion: runs 13de31be, 413ddcf2 each reported completion" }); + expect(check(base([...comments.slice(1), { body: doneMarker(nonce), runId: "run-z", at: "2026-11-12T18:20:00.000Z" }]), "completion_once").ok).toBe(false); + }); +}); + +describe("staging-mix runs after the task completed", () => { + const evidence = { + completedAt: 8_000, + runs: [{ runId: "6b825e48-aaaa", createdAt: 1_000, startedAt: 1_100, invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "w1", acceptedDisposition: "blocked", acceptedAt: 2_000 }, + { runId: "37724b59-bbbb", createdAt: 5_000, startedAt: 5_100, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "w2", acceptedDisposition: "done", acceptedAt: 7_900 }, + { runId: "5e5e5e5e-eeee", createdAt: 7_000, startedAt: 8_500, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "w2", acceptedDisposition: null, acceptedAt: null }, + { runId: "91f0c3d2-cccc", createdAt: 9_000, startedAt: 9_100, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w3", acceptedDisposition: null, acceptedAt: null }, + { runId: "aa11bb22-dddd", createdAt: 9_500, startedAt: null, invocationSource: "on_demand", triggerDetail: "manual", wakeupRequestId: null }, + { runId: "0f0f0f0f-ffff", createdAt: 7_500, startedAt: null, invocationSource: "on_demand", triggerDetail: null, wakeupRequestId: null }], + wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: null, requestedByActorType: "user", requestedByActorId: "u1" }, + { id: "w2", reason: "issue_children_completed", source: "automation", triggerDetail: null, requestedByActorType: "system", requestedByActorId: null }, + { id: "w3", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: "sweep" }], + }; + const comments = [{ body: doneMarker("n1"), runId: "6b825e48-aaaa", createdAt: 3_000 }]; + + it("anchors on the accepted done result, then completedAt, then the DONE comment", () => { + expect(completedAtAnchor(evidenceCompletion(evidence), comments, "n1")).toEqual({ at: 7_900, source: "accepted_result" }); + expect(completedAtAnchor({ completedAt: 8_000, runs: [] }, comments, "n1")).toEqual({ at: 8_000, source: "completed_at" }); + expect(completedAtAnchor(null, comments, "n1")).toEqual({ at: 3_000, source: "done_comment" }); + expect(completedAtAnchor(evidenceCompletion(null, "2026-11-13T00:06:13.120Z"), [], "n1")).toEqual({ at: Date.parse("2026-11-13T00:06:13.120Z"), source: "completed_at" }); + }); + + it("lists runs that started after completion, by start time, and prints the DONE comment time when it differs", () => { + const { started, neverStarted } = runsAfterDone("t001", evidence, completedAtAnchor(evidenceCompletion(evidence), comments, "n1"), 3_000); + expect(started.map(turn => [turn.runId, turn.wakeReason, turn.actorType])).toEqual([["5e5e5e5e-eeee", "issue_children_completed", "system"], + ["91f0c3d2-cccc", "issue_recovery_action_restored", "system"]]); + expect(extraTurnLine(started[1]!)).toBe(`- t001: run 91f0c3d2 started ${new Date(9_100).toISOString()} after completion ${new Date(7_900).toISOString()}, ` + + `DONE comment ${new Date(3_000).toISOString()}; wake issue_recovery_action_restored (automation/system) by system sweep`); + expect(neverStarted.map(turn => turn.runId)).toEqual(["aa11bb22-dddd"]); + expect(extraTurnLine(neverStarted[0]!)).toContain("never started"); + expect(extraTurnsByReason(started)).toEqual({ issue_children_completed: 1, issue_recovery_action_restored: 1 }); + expect(runsAfterDone("t001", evidence, { at: null, source: null })).toEqual({ started: [], neverStarted: [] }); + expect(runsAfterDone("t001", null, { at: 1, source: "completed_at" })).toEqual({ started: [], neverStarted: [] }); + }); + + it("does not count the run that completed the task as an extra turn (w-20261112T233630-s7201 t005)", () => { + const t005 = { completedAt: Date.parse("2026-11-13T00:06:13.120Z"), + runs: [{ runId: "2936e0b3-6a40", createdAt: 1_794_526_637_260, startedAt: 1_794_526_637_356, invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "f61202d1", + acceptedDisposition: "blocked", acceptedAt: Date.parse("2026-11-12T23:53:29.905Z") }, + { runId: "94066b4e-aacd", createdAt: 1_794_527_610_556, startedAt: 1_794_528_342_051, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "0645a2dc", + acceptedDisposition: "done", acceptedAt: Date.parse("2026-11-13T00:06:12.706Z") }], + wakes: [{ id: "0645a2dc", reason: "issue_execution_promoted", source: "automation", triggerDetail: null, requestedByActorType: "user", requestedByActorId: "local-board" }] }; + const posted = commentEvidence([{ body: doneMarker("c9b1da0f"), authorAgentId: "a", createdByRunId: "94066b4e-aacd", createdAt: "2026-11-13T00:06:07.168Z" }, + { body: `Blocked on child completion. I will post ${doneMarker("c9b1da0f")} later.`, authorAgentId: "a", createdByRunId: "2936e0b3-6a40", createdAt: "2026-11-12T23:53:30.499Z" }]); + const completion = evidenceCompletion(t005); + expect(runsAfterDone("t005", t005, completedAtAnchor(completion, posted, "c9b1da0f")).started).toEqual([]); + const grade = gradeStagingMix("c9b1da0f", { turnMinutes: 15, rounds: 10, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger("c9b1da0f", 10) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: completion! } }); + expect(grade.checks.find(check => check.name === "completion_once")).toMatchObject({ ok: true, detail: "runs=1" }); + }); +}); + +describe("staging-mix completion from accepted results", () => { + const nonce = "abcd1234"; + const completion = (runs: CompletionEvidence["runs"]): CompletionEvidence => ({ completedAt: 10_000, runs }); + + it("counts runs whose accepted result was done, plus a completion summary posted after completion by another run", () => { + const runs = completion([{ runId: "r1", startedAt: 1, acceptedDisposition: "blocked", acceptedAt: 5_000 }, { runId: "r2", startedAt: 6_000, acceptedDisposition: "done", acceptedAt: 9_000 }]); + const posted = [{ body: doneMarker(nonce), runId: "r2", createdAt: 8_900 }]; + const grade = (extra: Array<{ body: string; runId: string; createdAt: number }>) => gradeStagingMix(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: [...posted, ...extra], completion: runs } }).checks.find(check => check.name === "completion_once")!; + expect(grade([])).toMatchObject({ ok: true, detail: "runs=1" }); + expect(grade([{ body: `Will post ${doneMarker(nonce)} when the subtasks finish.`, runId: "r1", createdAt: 4_000 }]).ok).toBe(true); + expect(grade([{ body: `Task complete; posted ${doneMarker(nonce)}.`, runId: "r3", createdAt: 9_500 }])).toMatchObject({ ok: false }); + const twice = completion([{ runId: "r2", startedAt: 6_000, acceptedDisposition: "done", acceptedAt: 9_000 }, { runId: "r4", startedAt: 9_600, acceptedDisposition: "done", acceptedAt: 9_900 }]); + expect(gradeStagingMix(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments: posted, completion: twice } }).checks.find(check => check.name === "completion_once")!.ok).toBe(false); + }); + + it("reports a DONE comment from a run whose accepted result was not done as its own finding", () => { + const runs = completion([{ runId: "r1", startedAt: 1, acceptedDisposition: "blocked", acceptedAt: 5_000 }, { runId: "r2", startedAt: 6_000, acceptedDisposition: "done", acceptedAt: 9_000 }]); + expect(doneCommentsBeforeCompletion([{ body: doneMarker(nonce), runId: "r1", createdAt: 4_900 }, { body: doneMarker(nonce), runId: "r2", createdAt: 8_900 }], nonce, runs)) + .toEqual([{ runId: "r1", disposition: "blocked" }]); + expect(doneCommentsBeforeCompletion([{ body: doneMarker(nonce), runId: "r1", createdAt: 4_900 }], nonce, { completedAt: 1, runs: [] })).toEqual([]); + }); + + it("tells a parent not to post DONE until both subtasks are done and their hashes are recorded", () => { + const text = stagingMixDescription(nonce, { turnMinutes: 5, rounds: 4, subtasks: 2, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }); + expect(text).toContain(`end your turn now: do not save the \`ledger\` document, do not post the \`${doneMarker(nonce)}\` comment and do not mark this task done`); + expect(text).toContain("Only after both subtasks are done and both CHILD lines are in the `ledger` document, post exactly one comment"); + expect(stagingMixDescription(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 })).toContain("Post exactly one comment"); }); }); diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 78d51694ee5..cea5753242c 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -2,7 +2,9 @@ import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import type { ChaosApi } from "./api.js"; -import { bannerShownMs, DUPLICATE_CHECKS, noteRetryBanner } from "./staging-mix.js"; +import { + bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence, +} from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; type Json = Record; @@ -170,6 +172,81 @@ export function stagingMixHeadline(result: StagingMixResult): string[] { ]; } +export interface ExtraTurn { + taskId: string; + runId: string | null; + createdAt: number | null; + startedAt: number | null; + completedAt: number | null; + completedAtSource: string | null; + doneCommentAt: number | null; + invocationSource: string | null; + wakeReason: string | null; + wakeSource: string | null; + triggerDetail: string | null; + actorType: string | null; + actorId: string | null; +} + +export interface WakeEvidenceFile { + completedAt?: number | null; + runs?: Array<{ runId: string | null; createdAt: number | null; startedAt?: number | null; invocationSource: string | null; triggerDetail: string | null; + wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null }>; + wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null }>; +} + +export type CommentRow = { body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown }; + +export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ body: string; runId: string | null; createdAt: number | null }> { + return (rows ?? []).filter(row => row.authorAgentId).map(row => { + const at = typeof row.createdAt === "string" ? Date.parse(row.createdAt) : NaN; + return { body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null, createdAt: Number.isFinite(at) ? at : null }; + }); +} + +export function evidenceCompletion(evidence: WakeEvidenceFile | null, issueCompletedAt?: unknown): CompletionEvidence | null { + const fromIssue = typeof issueCompletedAt === "string" && Number.isFinite(Date.parse(issueCompletedAt)) ? Date.parse(issueCompletedAt) : null; + if (!evidence && fromIssue === null) return null; + return { completedAt: evidence?.completedAt ?? fromIssue, runs: (evidence?.runs ?? []).map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, + acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) }; +} + +export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, anchor: { at: number | null; source: string | null }, doneCommentAt: number | null = null): + { started: ExtraTurn[]; neverStarted: ExtraTurn[] } { + const doneAt = anchor.at; + if (!evidence || doneAt === null) return { started: [], neverStarted: [] }; + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + const turn = (run: NonNullable[number]): ExtraTurn => { + const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; + return { taskId, runId: run.runId, createdAt: run.createdAt, startedAt: run.startedAt ?? null, completedAt: doneAt, completedAtSource: anchor.source, doneCommentAt, + invocationSource: run.invocationSource, wakeReason: wake?.reason ?? null, wakeSource: wake?.source ?? null, triggerDetail: wake?.triggerDetail ?? run.triggerDetail, + actorType: wake?.requestedByActorType ?? null, actorId: wake?.requestedByActorId ?? null }; + }; + const runs = evidence.runs ?? []; + return { + started: runs.filter(run => run.startedAt != null && run.startedAt > doneAt).map(turn), + neverStarted: runs.filter(run => run.startedAt == null && run.createdAt !== null && run.createdAt > doneAt).map(turn), + }; +} + +const iso = (at: number | null) => at === null ? "?" : new Date(at).toISOString(); + +export function extraTurnLine(turn: ExtraTurn): string { + const reference = `completion ${iso(turn.completedAt)}${turn.completedAtSource && turn.completedAtSource !== "accepted_result" ? ` (${turn.completedAtSource})` : ""}` + + `${turn.doneCommentAt !== null && turn.doneCommentAt !== turn.completedAt ? `, DONE comment ${iso(turn.doneCommentAt)}` : ""}`; + const when = turn.startedAt !== null ? `started ${iso(turn.startedAt)} after ${reference}` + : `queued ${iso(turn.createdAt)} after ${reference}, never started`; + return `- ${turn.taskId}: run ${turn.runId?.slice(0, 8) ?? "?"} ${when}; ` + + `wake ${turn.wakeReason ?? "unknown"} (${turn.wakeSource ?? turn.invocationSource ?? "?"}${turn.triggerDetail ? `/${turn.triggerDetail}` : ""}) ` + + `by ${turn.actorType ?? "unknown"}${turn.actorId ? ` ${turn.actorId}` : ""}`; +} + +export function extraTurnsByReason(turns: readonly ExtraTurn[]): Record { + const counts: Record = {}; + for (const turn of turns) counts[turn.wakeReason ?? "unknown"] = (counts[turn.wakeReason ?? "unknown"] ?? 0) + 1; + return counts; +} + export async function readAssignedWorkInvariant(api: Pick, companyId: string): Promise { try { const report = await api.get(`/api/companies/${companyId}/assigned-work-invariant`, 30_000); @@ -185,8 +262,20 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick tasks: ReadonlyArray<{ plan: TaskPlan; record: TaskRecord }>; timeline: ReadonlyArray<{ kind: string; [key: string]: unknown }> }): Promise { const extraClicks = new Set(input.timeline.filter(entry => entry.kind === "release_swap_retry_clicked" && typeof entry.issueId === "string").map(entry => String(entry.issueId))); const tasks: ResultTask[] = []; + const extraTurns: ExtraTurn[] = []; + const queuedAfterDone: ExtraTurn[] = []; + const doneBeforeCompletion: Array<{ taskId: string; runId: string; disposition: string }> = []; for (const task of input.tasks) { const issue = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}`, 10_000).catch(() => null) : null; + const comments = task.record.issueId ? await input.api.get(`/api/issues/${task.record.issueId}/comments`, 10_000).catch(() => null) : null; + const evidence = await readFile(path.join(input.waveDir, "wake-evidence", `${task.plan.taskId}.json`), "utf8") + .then(text => JSON.parse(text) as WakeEvidenceFile).catch(() => null); + const posted = commentEvidence(comments); + const completion = evidenceCompletion(evidence, issue?.completedAt); + const afterDone = runsAfterDone(task.plan.taskId, evidence, completedAtAnchor(completion, posted, task.plan.nonce), firstDoneCommentAt(posted, task.plan.nonce)); + doneBeforeCompletion.push(...doneCommentsBeforeCompletion(posted, task.plan.nonce, completion).map(entry => ({ taskId: task.plan.taskId, ...entry }))); + extraTurns.push(...afterDone.started); + queuedAfterDone.push(...afterDone.neverStarted); const grade = await readFile(path.join(input.waveDir, `${task.plan.taskId}.grade.json`), "utf8") .then(text => JSON.parse(text) as { grade?: { T?: OracleResult } }).catch(() => null); tasks.push({ taskId: task.plan.taskId, issueId: task.record.issueId, record: task.record, finalStatus: issue?.status ?? task.record.finalStatus, @@ -194,13 +283,17 @@ export async function writeStagingMixResults(input: { waveDir: string; api: Pick } const result = summarizeStagingMix(tasks, extraClicks, await readAssignedWorkInvariant(input.api, input.companyId)); await writeFile(path.join(input.waveDir, "staging-mix-results.json"), `${JSON.stringify({ schema: "paperclip.restart-chaos.staging-mix/v1", - headline: stagingMixHeadline(result), ...result }, null, 2)}\n`, { mode: 0o600 }); + headline: stagingMixHeadline(result), ...result, extraTurns, extraTurnsByReason: extraTurnsByReason(extraTurns), queuedAfterDone, doneBeforeCompletion }, null, 2)}\n`, { mode: 0o600 }); const invariant = result.invariant.violationCount === null ? `unreadable${result.invariant.error ? ` (${result.invariant.error})` : ""}` : `${result.invariant.violationCount} violations, ${result.invariant.agentsWithStalledSlots} agents with stalled slots, ${result.invariant.openTasks} open tasks`; await writeFile(path.join(input.waveDir, "staging-mix-summary.md"), [...stagingMixHeadline(result), `Assigned-work invariant: ${invariant}`, "", ...result.duplicatedSideEffects.map(entry => `- ${entry.taskId}: duplicated ${entry.checks.join(", ")}`), ...result.evidenceIncomplete.map(entry => `- ${entry.taskId}: evidence incomplete for ${entry.checks.join(", ")}`), ...result.bannerShown.map(entry => `- ${entry.taskId}: Retry banner from ${entry.firstShownAt} for ${Math.round(entry.shownMs / 1000)} s` - + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), ""].join("\n"), { mode: 0o600 }); + + `${entry.errorCode ? ` (${entry.errorCode})` : ""}${entry.completedAnyway ? ", completed without a click" : ""}`), + ...(extraTurns.length ? ["", `Runs started after the task completed: ${extraTurns.length}`, ...extraTurns.map(extraTurnLine)] : []), + ...(queuedAfterDone.length ? ["", `Runs queued after the task completed, never started: ${queuedAfterDone.length}`, ...queuedAfterDone.map(extraTurnLine)] : []), + ...(doneBeforeCompletion.length ? ["", `DONE comment posted before the task completed: ${doneBeforeCompletion.length}`, + ...doneBeforeCompletion.map(entry => `- ${entry.taskId}: run ${entry.runId.slice(0, 8)} posted DONE but its accepted result was ${entry.disposition}`)] : []), ""].join("\n"), { mode: 0o600 }); return result; } diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 9e622047fda..cff36dadc35 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -54,6 +54,12 @@ export interface StagingMixRecord { retries?: Array<{ at: number; inOverlap: boolean }>; retryBanner?: RetryBanner; retryAfterSettle?: { at: number; accepted: boolean | null; issueStatusAtClick: string | null }; + completion?: CompletionEvidence; +} + +export interface CompletionEvidence { + completedAt: number | null; + runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null }>; } export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null } @@ -62,8 +68,9 @@ export interface StagingMixArtifacts { followUp: StagingMixRecord["followUp"] | null; followUpErrors: string[]; retries: NonNullable; - comments?: Array<{ body: string; runId: string | null }>; + comments?: Array<{ body: string; runId: string | null; createdAt?: number | null }>; commentsUnreadable?: boolean; + completion?: CompletionEvidence; } export const DUPLICATE_CHECKS: ReadonlySet = new Set(["rounds_once", "done_comment_once", "completion_once", "subtasks_once", "subtask_rounds_once"]); @@ -362,17 +369,58 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent lines.push(`${step++}. ${roundsSteps(nonce, task.rounds).join("\n ")}`); lines.push(`${step++}. If the board comments while you work and asks you to include a line, keep working and include that line on its own line in the document below.`); if (parent) { - lines.push(`${step++}. Wait until both subtasks are done. If they are not done yet, end your turn without marking this task done; you will be woken when they finish.`, + lines.push(`${step++}. Wait until both subtasks are done. If they are not done yet, end your turn now: do not save the \`ledger\` document, do not post the` + + ` \`${doneMarker(nonce)}\` comment and do not mark this task done. You will be woken when they finish.`, ` Then read each subtask's \`ledger\` document and take the hash at the end of its last ROUND line.`); } lines.push(`${step++}. Save this issue's document with key \`ledger\` (title "Ledger"): the full contents of \`${ledgerFile(nonce)}\` exactly as the file reads,` + ` then every line the board asked for${parent ? `, then one line \`${childLine(1, "")}\` per subtask (CHILD-2 for subtask 2), with from that subtask's ledger` : ""}.`); - lines.push(`${step++}. Post exactly one comment on this task with the body \`${doneMarker(nonce)}\`. Then mark this task done.`); + lines.push(`${step++}. ${parent ? "Only after both subtasks are done and both CHILD lines are in the `ledger` document, post" : "Post"} exactly one comment on this task` + + ` with the body \`${doneMarker(nonce)}\`. Then mark this task done.`); return lines.join("\n"); } export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`); +type CommentEvidence = { body: string; runId: string | null; createdAt?: number | null }; + +export function firstDoneCommentAt(comments: readonly CommentEvidence[], nonce: string): number | null { + const times = comments.filter(comment => isDoneComment(comment.body, nonce) && typeof comment.createdAt === "number").map(comment => comment.createdAt!); + return times.length ? Math.min(...times) : null; +} + +export function hasDispositions(completion: CompletionEvidence | null | undefined): completion is CompletionEvidence { + return Boolean(completion?.runs.some(run => run.acceptedDisposition !== null)); +} + +export function completedAtAnchor(completion: CompletionEvidence | null | undefined, comments: readonly CommentEvidence[], nonce: string): + { at: number | null; source: "accepted_result" | "completed_at" | "done_comment" | null } { + const accepted = (completion?.runs ?? []).filter(run => run.acceptedDisposition === "done" && run.acceptedAt !== null).map(run => run.acceptedAt!); + if (accepted.length) return { at: Math.min(...accepted), source: "accepted_result" }; + if (completion?.completedAt != null) return { at: completion.completedAt, source: "completed_at" }; + const comment = firstDoneCommentAt(comments, nonce); + return { at: comment, source: comment === null ? null : "done_comment" }; +} + +export function completionRunIds(comments: readonly CommentEvidence[], nonce: string, completion?: CompletionEvidence | null): string[] { + const anchor = completedAtAnchor(completion, comments, nonce).at; + const summaries = comments.filter(comment => anchor !== null && typeof comment.createdAt === "number" && comment.createdAt >= anchor + && comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!); + if (hasDispositions(completion)) { + const accepted = completion.runs.filter(run => run.acceptedDisposition === "done" && run.runId).map(run => run.runId!); + return [...new Set([...accepted, ...summaries])]; + } + return [...new Set([...comments.filter(comment => isDoneComment(comment.body, nonce) && comment.runId).map(comment => comment.runId!), ...summaries])]; +} + +export function doneCommentsBeforeCompletion(comments: readonly CommentEvidence[], nonce: string, completion?: CompletionEvidence | null): + Array<{ runId: string; disposition: string }> { + if (!hasDispositions(completion)) return []; + const dispositions = new Map(completion.runs.flatMap(run => run.runId ? [[run.runId, run.acceptedDisposition] as const] : [])); + return [...new Set(comments.filter(comment => isDoneComment(comment.body, nonce) && comment.runId).map(comment => comment.runId!))] + .flatMap(runId => { const disposition = dispositions.get(runId); return disposition && disposition !== "done" ? [{ runId, disposition }] : []; }); +} + export function isDoneComment(body: string, nonce: string): boolean { return body.trim().replace(/\.$/, "").replace(/^`+|`+$/g, "").trim() === doneMarker(nonce); } @@ -412,7 +460,7 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: const own = verifyLedger(nonce, task.rounds, ledger); const comments = artifacts.stagingMix?.comments ?? artifacts.agentComments.map(body => ({ body, runId: null })); const doneComments = comments.filter(comment => isDoneComment(comment.body, nonce)).length; - const completionRuns = [...new Set(comments.filter(comment => comment.body.includes(doneMarker(nonce)) && comment.runId).map(comment => comment.runId!))]; + const completionRuns = completionRunIds(comments, nonce, artifacts.stagingMix?.completion); const followUp = artifacts.stagingMix?.followUp ?? null; const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(nonce)); const checks: OracleResult["checks"] = [ @@ -573,8 +621,12 @@ export function bannerShownMs(banner: RetryBanner, now: number): number { return banner.shownMs + (banner.visibleSince === null ? 0 : now - banner.visibleSince); } -export function collectStagingMixArtifacts(record: TaskRecord, rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown }> | null): StagingMixArtifacts { +export function collectStagingMixArtifacts(record: TaskRecord, + rows: ReadonlyArray<{ body?: unknown; authorAgentId?: unknown; createdByRunId?: unknown; createdAt?: unknown }> | null): StagingMixArtifacts { + const time = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)) ? Date.parse(value) : null; return { followUp: record.stagingMix?.followUp ?? null, followUpErrors: record.stagingMix?.followUpErrors ?? [], retries: record.stagingMix?.retries ?? [], - ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null })) } + ...(record.stagingMix?.completion ? { completion: record.stagingMix.completion } : {}), + ...(rows ? { comments: rows.filter(row => row.authorAgentId).map(row => ({ body: String(row.body ?? ""), + runId: typeof row.createdByRunId === "string" ? row.createdByRunId : null, createdAt: time(row.createdAt) })) } : { commentsUnreadable: true }) }; } diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts index 58dc56d1990..2ff7ba29eba 100644 --- a/tests/restart-chaos/wake-evidence.test.ts +++ b/tests/restart-chaos/wake-evidence.test.ts @@ -62,6 +62,16 @@ describe("wake evidence", () => { expect(notes).toEqual([["wake_evidence_failed", { error: "Error: database is gone" }]]); }); + it("keeps only issue status transitions, not other actions' status fields (w-20261112T223803-s7101 t001)", () => { + const evidence = wakeEvidenceForIssue("i1", { runs: [], wakes: [], controllers: [], windows: [], statusChanges: [ + { issueId: "i1", action: "issue.created", actorType: "user", actorId: "local-board", fromStatus: null, toStatus: "todo", createdAt: 1_794_523_115_500 }, + { issueId: "i1", action: "runner.api_called", actorType: "agent", runId: "37724b59", fromStatus: null, toStatus: "201", createdAt: 1_794_525_164_631 }, + { issueId: "i1", action: "issue.updated", actorType: "agent", runId: "37724b59", fromStatus: "in_progress", toStatus: "done", createdAt: 1_794_525_165_000 }, + { issueId: "i1", action: "issue.updated", actorType: "agent", fromStatus: null, toStatus: "200", createdAt: 1_794_525_166_000 }, + ] }); + expect(evidence.statusChanges.map(change => [change.action, change.from, change.to])).toEqual([["issue.created", null, "todo"], ["issue.updated", "in_progress", "done"]]); + }); + it("joins each run to its wake, owner and status changes for one issue", () => { const evidence = wakeEvidenceForIssue("i1", { runs: [{ id: "r1", issueId: "i1", status: "succeeded", wakeupRequestId: "w2", controllerBootId: "boot-b", createdAt: 550_000, contextSnapshot: { wakeReason: "retry" } }, diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts index ad1d4651a97..7bf4eaf0a51 100644 --- a/tests/restart-chaos/wake-evidence.ts +++ b/tests/restart-chaos/wake-evidence.ts @@ -2,6 +2,8 @@ import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import type { ServerProcessWindow } from "./multi-server.js"; import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js"; +import { ISSUE_STATUSES } from "../../packages/shared/src/constants.js"; +import type { CompletionEvidence } from "./staging-mix.js"; import { createDb, type Db } from "../../packages/db/src/index.js"; type Row = Record; @@ -14,6 +16,12 @@ const text = (value: unknown) => value === null || value === undefined ? null : export const PROCESS_MATCH_SLACK_MS = 60_000; +const STATUSES: ReadonlySet = new Set(ISSUE_STATUSES); + +export function isIssueStatusTransition(row: Row): boolean { + return String(row.action ?? "").startsWith("issue.") && STATUSES.has(String(row.toStatus ?? "")) && (row.fromStatus == null || STATUSES.has(String(row.fromStatus))); +} + export function boundByProcessExit(controllers: readonly ControllerRow[], windows: readonly ServerProcessWindow[]): ControllerRow[] { return controllers.map(controller => { const window = windows.filter(candidate => candidate.pid === controller.pid && candidate.spawnedAt <= controller.startedAt + PROCESS_MATCH_SLACK_MS @@ -51,6 +59,7 @@ export interface WakeEvidenceInput { statusChanges: readonly Row[]; controllers: readonly ControllerRow[]; windows: readonly ServerProcessWindow[]; + issues?: readonly Row[]; } export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) { @@ -61,11 +70,12 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) const controllerByBoot = new Map(controllers.map(controller => [controller.bootId, controller])); return { issueId, + completedAt: ms(input.issues?.find(issue => issue.id === issueId)?.completedAt), runs: runs.map(run => { const owner = controllerByBoot.get(String(run.controllerBootId ?? "")); return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail), wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), - contextSnapshot: run.contextSnapshot ?? null, + acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) }; }), @@ -74,7 +84,7 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) status: text(wake.status), runId: text(wake.runId), payload: wake.payload ?? null, error: text(wake.error), createdAt: ms(wake.createdAt), requestedAt: ms(wake.requestedAt), claimedAt: ms(wake.claimedAt), finishedAt: ms(wake.finishedAt), servingAtCreate: servingAt(ms(wake.createdAt), controllers, input.windows) })), - statusChanges: input.statusChanges.filter(change => change.issueId === issueId).map(change => ({ at: ms(change.createdAt), action: text(change.action), + statusChanges: input.statusChanges.filter(change => change.issueId === issueId && isIssueStatusTransition(change)).map(change => ({ at: ms(change.createdAt), action: text(change.action), actorType: text(change.actorType), actorId: text(change.actorId), runId: text(change.runId), from: text(change.fromStatus), to: text(change.toStatus) })), }; } @@ -82,11 +92,18 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> { const client = db.$client; const runs = await client.unsafe( - `select id::text as id, issue_id::text as "issueId", status, invocation_source as "invocationSource", trigger_detail as "triggerDetail", - wakeup_request_id::text as "wakeupRequestId", context_snapshot as "contextSnapshot", controller_boot_id::text as "controllerBootId", - floor(extract(epoch from created_at) * 1000) as "createdAt", floor(extract(epoch from started_at) * 1000) as "startedAt", - floor(extract(epoch from finished_at) * 1000) as "finishedAt" - from heartbeat_runs where company_id::text = $1 order by created_at`, [companyId]) as Row[]; + `select r.id::text as id, r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail", + r.wakeup_request_id::text as "wakeupRequestId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId", + floor(extract(epoch from r.created_at) * 1000) as "createdAt", floor(extract(epoch from r.started_at) * 1000) as "startedAt", + floor(extract(epoch from r.finished_at) * 1000) as "finishedAt", accepted.disposition as "acceptedDisposition", + floor(extract(epoch from accepted.accepted_at) * 1000) as "acceptedAt" + from heartbeat_runs r + left join lateral (select coalesce(e.payload->'prpEvent'->'payload'->'result'->>'reportedWorkDisposition', e.payload->'result'->>'reportedWorkDisposition') as disposition, e.created_at as accepted_at + from heartbeat_run_events e where e.run_id = r.id and e.event_type = 'run.result.accepted' + order by e.seq desc limit 1) accepted on true + where r.company_id::text = $1 order by r.created_at`, [companyId]) as Row[]; + const issues = await client.unsafe( + `select id::text as id, floor(extract(epoch from completed_at) * 1000) as "completedAt" from issues where company_id::text = $1`, [companyId]) as Row[]; const wakes = await client.unsafe( `select id::text as id, payload->>'issueId' as "issueId", source, trigger_detail as "triggerDetail", reason, payload, status, requested_by_actor_type as "requestedByActorType", requested_by_actor_id as "requestedByActorId", idempotency_key as "idempotencyKey", @@ -98,15 +115,16 @@ async function readRows(db: Pick, companyId: string, shifts: read `select entity_id as "issueId", action, actor_type as "actorType", actor_id as "actorId", run_id::text as "runId", coalesce(details->'_previous'->>'status', details->>'previousStatus') as "fromStatus", details->>'status' as "toStatus", floor(extract(epoch from created_at) * 1000) as "createdAt" - from activity_log where company_id::text = $1 and entity_type = 'issue' - and (details ? 'status' or details->'_previous' ? 'status' or details ? 'previousStatus') order by created_at`, [companyId]) as Row[]; + from activity_log where company_id::text = $1 and entity_type = 'issue' and action like 'issue.%' + and details ? 'status' order by created_at`, [companyId]) as Row[]; const controllers = (await client.unsafe( `select boot_id as "bootId", epoch, process_space as "processSpace", pid, floor(extract(epoch from started_at) * 1000) as "startedAt", floor(extract(epoch from coalesce(stopped_at, fenced_at)) * 1000) as "endedAt" from controller_instances order by epoch`) as Row[]) .map(row => ({ bootId: String(row.bootId), epoch: Number(row.epoch), processSpace: String(row.processSpace), pid: Number(row.pid), startedAt: Number(row.startedAt), endedAt: ms(row.endedAt) })); return { - runs: unshiftRows(runs, ["createdAt", "startedAt", "finishedAt"], ["contextSnapshot"], shifts), + runs: unshiftRows(runs, ["createdAt", "startedAt", "finishedAt", "acceptedAt"], ["contextSnapshot"], shifts), + issues: unshiftRows(issues, ["completedAt"], [], shifts), wakes: unshiftRows(wakes, ["createdAt", "requestedAt", "claimedAt", "finishedAt"], ["payload"], shifts), statusChanges: unshiftRows(statusChanges, ["createdAt"], [], shifts), controllers: controllers.map(controller => ({ ...controller, startedAt: unshiftMs(controller.startedAt, shifts), @@ -115,10 +133,10 @@ async function readRows(db: Pick, companyId: string, shifts: read } export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string; - observers: ReadonlyArray<{ plan: { taskId: string }; record: { issueId: string | null } }> | null; windows: readonly ServerProcessWindow[]; + observers: ReadonlyArray | null; windows: readonly ServerProcessWindow[]; shifts?: readonly AgingShift[]; redact: (text: string) => string }): Promise { const rows = await readRows(input.db, input.companyId, input.shifts ?? []); - const observers = input.observers ?? [...new Set(rows.runs.map(run => text(run.issueId)).filter((id): id is string => id !== null))] + const observers: readonly EvidenceObserver[] = input.observers ?? [...new Set(rows.runs.map(run => text(run.issueId)).filter((id): id is string => id !== null))] .map(issueId => ({ plan: { taskId: `issue-${issueId}` }, record: { issueId } })); const directory = path.join(input.waveDir, "wake-evidence"); await mkdir(directory, { recursive: true, mode: 0o700 }); @@ -127,6 +145,7 @@ export async function captureWakeEvidence(input: { db: Pick; comp if (!observer.record.issueId) continue; const evidence = { schema: "paperclip.restart-chaos.wake-evidence/v1", taskId: observer.plan.taskId, ...wakeEvidenceForIssue(observer.record.issueId, { ...rows, windows: input.windows }) }; + if (observer.plan.stagingMix) (observer.record.stagingMix ??= {}).completion = completionEvidence(evidence); await writeFile(path.join(directory, `${observer.plan.taskId}.json`), input.redact(`${JSON.stringify(evidence, null, 2)}\n`), { mode: 0o600 }); written += 1; } @@ -134,8 +153,18 @@ export async function captureWakeEvidence(input: { db: Pick; comp return written; } +export interface EvidenceObserver { + plan: { taskId: string; stagingMix?: unknown }; + record: { issueId: string | null; stagingMix?: { completion?: CompletionEvidence } }; +} + +export function completionEvidence(evidence: ReturnType): CompletionEvidence { + return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt, + acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt })) }; +} + export function wakeEvidenceOnce(input: { databaseUrl: () => string; companyId: string; waveDir: string; - observers: () => ReadonlyArray<{ plan: { taskId: string }; record: { issueId: string | null } }>; windows: readonly ServerProcessWindow[]; + observers: () => ReadonlyArray; windows: readonly ServerProcessWindow[]; shifts: readonly AgingShift[]; redact: (text: string) => string; note: (kind: string, data: Record) => void; open?: (url: string) => Pick }): () => Promise { let done: Promise | null = null;