diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 20fe2ccdd08..785b001ec2e 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1906,10 +1906,28 @@ re-arms the task's own triggered faults instead of using their offsets. - `ledger.jsonl`, `SUMMARY.md` — every graded task, per-cell pass rate, open signatures. - `/plan.json`, `timeline.jsonl`, `server.log`, `faults.json`, `.grade.json`. -- `/evidence//` on failure — run ids and statuses, last 200 events per - run, workspace operations, server log window around each fault, startup phase - timings, fault timeline, seed, git SHA and the replay command. Known provider - key values are redacted. +- `/evidence//` on failure — run ids and statuses, each run's events (all of them, or the first 400 + and the last 200 with an `omittedEvents` marker between, so the wake payload and `turn.submitted` are always + kept), workspace operations, server log window around each fault, startup phase timings, fault timeline, seed, + git SHA and the replay command. Known provider key values are redacted. +- `/wake-evidence/.json` for every task, read from the wave database before the instance is deleted. + It holds: + - each run's `context_snapshot`, invocation source and trigger; + - the `agent_wakeup_requests` row behind each run and every wake for the issue: reason, source, trigger detail, + requesting actor, idempotency key, payload and times; + - the issue's status changes from the activity log. + + Each wake and run also names the server generations alive when it was created (`servingAtCreate`): the pid, boot + id, epoch and process space from `controller_instances`, and the harness's own primary or secondary role during an + overlap. The run's owning controller is listed as `owner`. `wake-evidence/servers.json` lists every controller and + harness server window. + + - **Process exit:** a controller's lifetime also ends at the process exit the harness recorded, so a hard-killed + server is not listed as serving later work. + - **Aged waves:** in TM2/TM3 waves, the database times are put back at their real times before they are compared + with the harness clock. + - **Failed waves:** the capture also runs, once, on the failure path before PostgreSQL stops. If the task list does + not exist yet, it writes one `issue-.json` per issue that has runs. ## Grading notes diff --git a/tests/restart-chaos/evidence.ts b/tests/restart-chaos/evidence.ts index e60053caa54..d1534fda8cb 100644 --- a/tests/restart-chaos/evidence.ts +++ b/tests/restart-chaos/evidence.ts @@ -5,9 +5,12 @@ import { replayCommand } from "./config.js"; import type { TimelineEntry } from "./timeline.js"; import type { FaultReceipt, OracleArtifacts, TaskGrade, TaskRecord, WavePlan } from "./types.js"; +export { captureWakeEvidence, wakeEvidenceOnce } from "./wake-evidence.js"; + const LOG_BEFORE_BYTES = 128 * 1024; const LOG_AFTER_BYTES = 512 * 1024; -const EVENTS_PER_RUN = 200; +export const EVENTS_HEAD = 400; +export const EVENTS_TAIL = 200; const OBSERVER_POLL_LINE = /INFO\S*: \S*(?:GET|HEAD) \/(?:issues\/[0-9a-f-]+(?:\/runs)?|ready|liveness|health) 200/; type Json = Record; @@ -24,19 +27,24 @@ export function knownSecretValues(env: NodeJS.ProcessEnv = process.env): string[ return Object.entries(env).filter(([key, value]) => value && /(?:KEY|TOKEN|SECRET|PASSWORD)$/.test(key)).map(([, value]) => value!); } -async function lastEvents(api: ChaosApi, runId: string): Promise { - const kept: Json[] = []; - let afterSeq = 0; +export async function runEvents(api: ChaosApi, runId: string): Promise { + const head: Json[] = [], tail: Json[] = []; + let total = 0, afterSeq = 0, omittedFrom: unknown = null, omittedTo: unknown = null; for (let page = 0; page < 200; page += 1) { const rows = await api.get(`/api/heartbeat-runs/${runId}/events?afterSeq=${afterSeq}&limit=500`); if (!rows?.length) break; - kept.push(...rows); - if (kept.length > EVENTS_PER_RUN) kept.splice(0, kept.length - EVENTS_PER_RUN); + for (const row of rows) { + total += 1; + if (head.length < EVENTS_HEAD) { head.push(row); continue; } + tail.push(row); + if (tail.length > EVENTS_TAIL) { const dropped = tail.shift()!; omittedFrom ??= dropped.seq ?? null; omittedTo = dropped.seq ?? null; } + } const next = Number(rows[rows.length - 1]?.seq); if (!Number.isFinite(next) || next <= afterSeq || rows.length < 500) break; afterSeq = next; } - return kept; + const omitted = total - head.length - tail.length; + return omitted > 0 ? [...head, { omittedEvents: omitted, fromSeq: omittedFrom, toSeq: omittedTo }, ...tail] : [...head, ...tail]; } async function logWindow(serverLogPath: string, faults: readonly FaultReceipt[]): Promise { @@ -94,7 +102,7 @@ export async function writeEvidenceBundle(input: { await attempt("runs", async () => { runs = await api.get(`/api/issues/${record.issueId}/runs`); await write("runs.json", runs); }); for (const run of runs) { const runId = String(run.runId ?? run.id); - await attempt(`events ${runId}`, async () => write(`run-${runId}-events.json`, await lastEvents(api, runId))); + await attempt(`events ${runId}`, async () => write(`run-${runId}-events.json`, await runEvents(api, runId))); await attempt(`workspace-operations ${runId}`, async () => write(`run-${runId}-workspace-operations.json`, await api.get(`/api/heartbeat-runs/${runId}/workspace-operations`))); } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 5707d54cb2f..a74939db29c 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -28,7 +28,7 @@ import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from " import { assertControllerMatchesImage, assertProviderPackMismatch, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js"; import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js"; import { DaytonaWaveSweeper, leaksForTask, unattributedLeaks, type DaytonaSweepResult } from "./daytona-sweep.js"; -import { knownSecretValues, redactKnownSecrets, writeEvidenceBundle } from "./evidence.js"; +import { knownSecretValues, redactKnownSecrets, wakeEvidenceOnce, writeEvidenceBundle } from "./evidence.js"; import { gradeTask } from "./grader.js"; import { releaseInstance } from "../runner-e2e/instance-retention.js"; import { checkPlanInvariants, readUniversalDbEvidence, summarizeViolations, type InvariantCheck } from "./invariants.js"; @@ -320,8 +320,8 @@ async function main(): Promise { harnessError ??= error; interrupted = true; }); - // Spend is read once per wave: after the tasks, or during cleanup when the wave stopped early. let spendTarget: { api: ChaosApi; companyId: string } | null = null; + let captureEvidence: (() => Promise) | null = null; let spendRecorded = false; const recordWaveSpend = async () => { if (!spendTarget || spendRecorded) return; @@ -463,6 +463,7 @@ async function main(): Promise { }; } }); spendTarget = { api, companyId: lab.companyId }; + captureEvidence = wakeEvidenceOnce({ databaseUrl: waveDatabaseUrl, companyId: lab.companyId, waveDir, observers: () => launched, windows: supervisor.serverWindows, shifts: supervisor.agingShifts, redact: text => redactKnownSecrets(text, knownSecretValues()), note: (kind, data) => timeline.add(kind, data) }); const releaseSwap = plan.options.releaseSwap ? await ReleaseSwapWave.start({ plan, lab, api, timeline, supervisor, databaseUrl: waveDatabaseUrl(), cancelled: () => interrupted }) : null; // --db-pressure: contend with the wave's running run rows from outside the server (contention.ts). if (plan.options.dbPressure) { @@ -782,8 +783,8 @@ async function main(): Promise { } const attributed = new Set(); const secrets = knownSecretValues(); - // U3 and L1 read the wave database once, before the instance is removed. const evidenceDb = createDb(waveDatabaseUrl()); + await captureEvidence?.(); const universalByIssue = await readUniversalDbEvidence(evidenceDb, lab.companyId).catch(error => { timeline.add("universal_db_evidence_failed", { error: String(error) }); return null; }); const allRunMessages = new Map([...(universalByIssue?.values() ?? [])].flatMap(entry => [...entry.runMessages])); @@ -907,11 +908,10 @@ async function main(): Promise { sandboxWatchStopped = true; await mix?.stop().catch(() => undefined); await stopPressure().catch(error => timeline.add("db_pressure_stop_failed", { error: String(error) })); - // Best effort for an interrupted or failed wave; the server may already be down. + await captureEvidence?.(); if (supervisor.serverReady) await recordWaveSpend().catch(() => undefined); await board?.close().catch(() => undefined); await closeJourneyApps(); - // A failure here (for example writing the report) must not skip stopping the controller and Postgres. try { if (sweeper) try { const owned = sweeper as DaytonaWaveSweeper; // An interrupted wave still reports what it leaked before cleanup. diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 37028875903..2a30650b296 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -313,8 +313,11 @@ export function roundCommand(nonce: string): string { function roundsSteps(nonce: string, rounds: number): string[] { return [ `Run this shell command ${rounds} times, as separate tool calls in the foreground, replacing every with the round number 1 to ${rounds}`, - `(each takes about ${ROUND_SECONDS} seconds; do not run rounds in the background or in parallel, and do not skip, repeat or shorten any):`, + `(each takes about ${ROUND_SECONDS} seconds; run them one at a time in the foreground, never in the background or in parallel):`, `\`${roundCommand(nonce)}\``, + `Every round from 1 to ${rounds} must end up in \`${ledgerFile(nonce)}\` exactly once, with the hash the command printed.`, + "Before each round, read the ledger file: if that round's ROUND line is already there, the round is finished, so never run it again.", + "If a round was interrupted (for example the tool call failed or your session restarted) before its ROUND line was written, you must run that round again from the start.", ]; } @@ -355,7 +358,7 @@ export function stagingMixDescription(nonce: string, task: StagingMixTask, agent export const followUpRequestId = (issueId: string, nonce: string) => stableRequestId(`followup-${issueId}-${nonce}`); export function followUpComment(nonce: string): string { - return `Additional requirement: include the line \`${followUpMarker(nonce)}\` on its own line in this task's \`ledger\` document. Keep going with your rounds; do not restart any.`; + return `Additional requirement: include the line \`${followUpMarker(nonce)}\` on its own line in this task's \`ledger\` document. Keep going with your rounds: a round whose ROUND line is already in the ledger file is finished, and a round without one must still be run.`; } export interface LedgerRound { round: number; start: number; end: number; hash: string } diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts new file mode 100644 index 00000000000..58dc56d1990 --- /dev/null +++ b/tests/restart-chaos/wake-evidence.test.ts @@ -0,0 +1,150 @@ +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { activityLog, agentWakeupRequests, agents, companies, controllerInstances, createDb, heartbeatRuns, issues, type Db } from "../../packages/db/src/index.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../server/src/__tests__/helpers/embedded-postgres.js"; +import { boundByProcessExit, captureWakeEvidence, servingAt, unshiftRows, wakeEvidenceForIssue, wakeEvidenceOnce, type ControllerRow } from "./wake-evidence.js"; +import { runEvents } from "./evidence.js"; +import { stagingMixDescription, followUpComment } from "./staging-mix.js"; +import type { ChaosApi } from "./api.js"; + +const support = await getEmbeddedPostgresTestSupport(); + +const controllers: ControllerRow[] = [ + { bootId: "boot-a", epoch: 1, processSpace: "space-a", pid: 100, startedAt: 0, endedAt: 600_000 }, + { bootId: "boot-b", epoch: 2, processSpace: "space-b", pid: 200, startedAt: 500_000, endedAt: null }, +]; + +describe("wake evidence", () => { + it("names every server generation alive when a wake or run was created, overlap included", () => { + const windows = [{ pid: 100, role: "primary" as const, spawnedAt: 0, exitedAt: 600_000, frozen: [] }, + { pid: 200, role: "secondary" as const, spawnedAt: 500_000, exitedAt: null, frozen: [] }, + { pid: 300, role: "primary" as const, spawnedAt: 700_000, exitedAt: null, frozen: [] }]; + expect(servingAt(100_000, controllers, windows).map(server => server.pid)).toEqual([100]); + expect(servingAt(550_000, controllers, windows).map(server => [server.pid, server.processSpace, server.harnessRole])).toEqual([[100, "space-a", "primary"], [200, "space-b", "secondary"]]); + expect(servingAt(800_000, controllers, windows).map(server => [server.pid, server.bootId])).toEqual([[200, "boot-b"], [300, null]]); + expect(servingAt(null, controllers, windows)).toEqual([]); + }); + + it("ends a hard-killed controller at its recorded process exit, so it is not listed as serving a later run", () => { + const killed: ControllerRow[] = [{ bootId: "boot-k", epoch: 1, processSpace: "space-k", pid: 100, startedAt: 1_000, endedAt: null }, + { bootId: "boot-n", epoch: 2, processSpace: "space-n", pid: 200, startedAt: 61_000, endedAt: null }]; + const windows = [{ pid: 100, role: "primary" as const, spawnedAt: 0, exitedAt: 60_000, frozen: [] }, + { pid: 200, role: "primary" as const, spawnedAt: 60_500, exitedAt: null, frozen: [] }]; + expect(boundByProcessExit(killed, windows).map(controller => controller.endedAt)).toEqual([60_000, null]); + expect(servingAt(90_000, boundByProcessExit(killed, windows), windows).map(server => server.pid)).toEqual([200]); + const evidence = wakeEvidenceForIssue("i1", { runs: [{ id: "r1", issueId: "i1", createdAt: 90_000, controllerBootId: "boot-n" }], wakes: [], statusChanges: [], + controllers: killed, windows }); + expect(evidence.runs[0]!.servingAtCreate.map(server => server.pid)).toEqual([200]); + expect(boundByProcessExit(killed, [{ pid: 100, role: "primary", spawnedAt: 500_000, exitedAt: 600_000, frozen: [] }])[0]!.endedAt).toBeNull(); + }); + + it("puts aged database times back at their real times before comparing them with the harness clock", () => { + const shifts = [{ faultId: "f1", ageMs: 26 * 3_600_000, appliedAt: 2_000_000_000_000, waveStartedAt: 1_999_999_000_000 }]; + const aged = 1_999_999_500_000 - 26 * 3_600_000; + const [row] = unshiftRows([{ createdAt: aged, finishedAt: null, contextSnapshot: { at: new Date(aged).toISOString() } }], ["createdAt", "finishedAt"], ["contextSnapshot"], shifts); + expect(row).toEqual({ createdAt: 1_999_999_500_000, finishedAt: null, contextSnapshot: { at: new Date(1_999_999_500_000).toISOString() } }); + expect(unshiftRows([{ createdAt: 1_999_999_600_000 }], ["createdAt"], [], shifts)[0]!.createdAt).toBe(1_999_999_600_000); + expect(unshiftRows([{ createdAt: aged }], ["createdAt"], [], [])[0]!.createdAt).toBe(aged); + }); + + it("captures once, falls back to every issue when the task list is not available yet, and records a failure", async () => { + const notes: Array<[string, Record]> = []; + let opened = 0; + const capture = wakeEvidenceOnce({ databaseUrl: () => "postgres://unused", companyId: "c1", waveDir: "/nonexistent/wave", windows: [], shifts: [], + observers: () => { throw new ReferenceError("launched is not initialized"); }, redact: text => text, note: (kind, data) => notes.push([kind, data]), + open: () => { opened += 1; return { $client: { unsafe: async () => { throw new Error("database is gone"); }, end: async () => undefined } } as never; } }); + await Promise.all([capture(), capture()]); + await capture(); + expect(opened).toBe(1); + expect(notes).toEqual([["wake_evidence_failed", { error: "Error: database is gone" }]]); + }); + + it("joins each run to its wake, owner and status changes for one issue", () => { + const evidence = wakeEvidenceForIssue("i1", { + runs: [{ id: "r1", issueId: "i1", status: "succeeded", wakeupRequestId: "w2", controllerBootId: "boot-b", createdAt: 550_000, contextSnapshot: { wakeReason: "retry" } }, + { id: "r9", issueId: "i9", status: "succeeded", wakeupRequestId: "w9", createdAt: 1 }], + wakes: [{ id: "w1", issueId: "i1", reason: "issue_assigned", source: "assignment", createdAt: 10 }, + { id: "w2", issueId: null, reason: "retry", source: "on_demand", requestedByActorType: "user", idempotencyKey: "k", createdAt: 540_000 }, + { id: "w9", issueId: "i9", createdAt: 1 }], + statusChanges: [{ issueId: "i1", action: "issue.updated", fromStatus: "done", toStatus: "in_progress", createdAt: 560_000 }, { issueId: "i9", toStatus: "done" }], + controllers, windows: [], + }); + expect(evidence.runs).toHaveLength(1); + expect(evidence.runs[0]).toMatchObject({ runId: "r1", wakeupRequestId: "w2", contextSnapshot: { wakeReason: "retry" }, owner: { pid: 200, epoch: 2 } }); + expect(evidence.runs[0]!.servingAtCreate.map(server => server.pid)).toEqual([100, 200]); + expect(evidence.wakes.map(wake => [wake.id, wake.reason, wake.requestedByActorType, wake.idempotencyKey])).toEqual([["w1", "issue_assigned", null, null], ["w2", "retry", "user", "k"]]); + expect(evidence.statusChanges).toEqual([{ at: 560_000, action: "issue.updated", actorType: null, actorId: null, runId: null, from: "done", to: "in_progress" }]); + }); + + it("keeps the first 400 events and the last 200 of a long run, and says what it left out", async () => { + const all = Array.from({ length: 1_291 }, (_, index) => ({ seq: index + 1, eventType: index === 0 ? "turn.submitted" : "log" })); + const api = { get: async (url: string) => { + const afterSeq = Number(/afterSeq=(\d+)/.exec(url)![1]); + return all.filter(event => event.seq > afterSeq).slice(0, 500) as T; + } } as unknown as ChaosApi; + const kept = await runEvents(api, "run-1"); + expect(kept).toHaveLength(601); + expect(kept[0]).toMatchObject({ seq: 1, eventType: "turn.submitted" }); + expect(kept[400]).toEqual({ omittedEvents: 691, fromSeq: 401, toSeq: 1_091 }); + expect(kept.at(-1)).toMatchObject({ seq: 1_291 }); + const short = await runEvents({ get: async (url: string) => (/afterSeq=0/.test(url) ? all.slice(0, 50) : []) as T } as unknown as ChaosApi, "run-2"); + expect(short).toHaveLength(50); + }); +}); + +describe("staging-mix round wording", () => { + it("tells the agent to rerun an interrupted round and never one already in the ledger", () => { + const text = stagingMixDescription("abcd1234", { turnMinutes: 5, rounds: 4, subtasks: 2, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }); + expect(text).not.toMatch(/do not skip, repeat or shorten/); + expect(text).toContain("must end up in `ledger-abcd1234.log` exactly once"); + expect(text).toContain("if that round's ROUND line is already there, the round is finished, so never run it again"); + expect(text).toContain("before its ROUND line was written, you must run that round again from the start"); + expect(followUpComment("abcd1234")).not.toMatch(/do not restart/); + expect(followUpComment("abcd1234")).toContain("a round without one must still be run"); + }); +}); + +describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("restart-chaos-wake-evidence-"); + db = createDb(database.connectionString); + }, 120_000); + afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); + + it("writes one file per task with the wake row, context snapshot, status changes and serving controller", async () => { + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), wakeId = randomUUID(), runId = randomUUID(), bootId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Wake evidence", issuePrefix: companyId.slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Agent", adapterType: "process", status: "active" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Task" }); + await db.insert(controllerInstances).values({ bootId, processSpace: "space-wake", pid: 4242, leaseExpiresAt: new Date(Date.now() + 60_000) }); + await db.insert(agentWakeupRequests).values({ id: wakeId, companyId, agentId, source: "on_demand", reason: "retry", triggerDetail: "manual", + requestedByActorType: "user", requestedByActorId: "u1", idempotencyKey: "idem-1", payload: { issueId }, runId }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, status: "succeeded", invocationSource: "on_demand", wakeupRequestId: wakeId, + contextSnapshot: { wakeReason: "retry", taskId: issueId }, controllerBootId: bootId }); + await db.insert(activityLog).values({ companyId, actorType: "agent", actorId: agentId, action: "issue.updated", entityType: "issue", entityId: issueId, + details: { status: "done", _previous: { status: "in_progress" } } }); + const waveDir = await mkdtemp(path.join(os.tmpdir(), "wake-evidence-")); + try { + const written = await captureWakeEvidence({ db, companyId, waveDir, observers: [{ plan: { taskId: "t01" }, record: { issueId } }, { plan: { taskId: "t02" }, record: { issueId: null } }], + windows: [], redact: text => text.replaceAll("u1", "[REDACTED]") }); + expect(written).toBe(1); + const evidence = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "t01.json"), "utf8")); + expect(evidence.runs[0]).toMatchObject({ runId, invocationSource: "on_demand", wakeupRequestId: wakeId, contextSnapshot: { wakeReason: "retry" }, + owner: { pid: 4242, processSpace: "space-wake" } }); + expect(evidence.wakes[0]).toMatchObject({ id: wakeId, reason: "retry", source: "on_demand", triggerDetail: "manual", requestedByActorType: "user", + requestedByActorId: "[REDACTED]", idempotencyKey: "idem-1", runId }); + expect(evidence.wakes[0].servingAtCreate.map((server: { pid: number }) => server.pid)).toEqual([4242]); + expect(evidence.statusChanges).toMatchObject([{ action: "issue.updated", from: "in_progress", to: "done" }]); + expect(JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "servers.json"), "utf8")).controllers[0]).toMatchObject({ pid: 4242 }); + expect(await captureWakeEvidence({ db, companyId, waveDir, observers: null, windows: [], redact: text => text })).toBe(1); + expect(JSON.parse(await readFile(path.join(waveDir, "wake-evidence", `issue-${issueId}.json`), "utf8")).runs[0]).toMatchObject({ runId }); + } finally { + await rm(waveDir, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts new file mode 100644 index 00000000000..ad1d4651a97 --- /dev/null +++ b/tests/restart-chaos/wake-evidence.ts @@ -0,0 +1,156 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { ServerProcessWindow } from "./multi-server.js"; +import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js"; +import { createDb, type Db } from "../../packages/db/src/index.js"; + +type Row = Record; + +export interface ControllerRow { bootId: string; epoch: number; processSpace: string; pid: number; startedAt: number; endedAt: number | null } +export interface ServingServer { pid: number; bootId: string | null; epoch: number | null; processSpace: string | null; harnessRole: string | null } + +const ms = (value: unknown) => value === null || value === undefined ? null : Number(value); +const text = (value: unknown) => value === null || value === undefined ? null : String(value); + +export const PROCESS_MATCH_SLACK_MS = 60_000; + +export function boundByProcessExit(controllers: readonly ControllerRow[], windows: readonly ServerProcessWindow[]): ControllerRow[] { + return controllers.map(controller => { + const window = windows.filter(candidate => candidate.pid === controller.pid && candidate.spawnedAt <= controller.startedAt + PROCESS_MATCH_SLACK_MS + && (candidate.exitedAt === null || candidate.exitedAt >= controller.startedAt - PROCESS_MATCH_SLACK_MS)) + .sort((a, b) => b.spawnedAt - a.spawnedAt)[0]; + if (!window || window.exitedAt === null) return controller; + return { ...controller, endedAt: controller.endedAt === null ? window.exitedAt : Math.min(controller.endedAt, window.exitedAt) }; + }); +} + +export function unshiftRows(rows: readonly Row[], timeKeys: readonly string[], jsonKeys: readonly string[], shifts: readonly AgingShift[]): Row[] { + if (!shifts.length) return [...rows]; + return rows.map(row => { + const copy: Row = { ...row }; + for (const key of timeKeys) if (copy[key] !== null && copy[key] !== undefined) copy[key] = unshiftMs(Number(copy[key]), shifts); + for (const key of jsonKeys) if (copy[key] !== null && copy[key] !== undefined) copy[key] = unshiftJson(copy[key], shifts); + return copy; + }); +} + +export function servingAt(at: number | null, controllers: readonly ControllerRow[], windows: readonly ServerProcessWindow[]): ServingServer[] { + if (at === null) return []; + const alive = controllers.filter(controller => controller.startedAt <= at && (controller.endedAt === null || controller.endedAt >= at)); + const fromDb = alive.map(controller => ({ pid: controller.pid, bootId: controller.bootId, epoch: controller.epoch, processSpace: controller.processSpace, + harnessRole: windows.find(window => window.pid === controller.pid && window.spawnedAt <= at && (window.exitedAt === null || window.exitedAt >= at))?.role ?? null })); + const known = new Set(fromDb.map(server => server.pid)); + const fromHarness = windows.filter(window => !known.has(window.pid) && window.spawnedAt <= at && (window.exitedAt === null || window.exitedAt >= at)) + .map(window => ({ pid: window.pid, bootId: null, epoch: null, processSpace: null, harnessRole: window.role })); + return [...fromDb, ...fromHarness]; +} + +export interface WakeEvidenceInput { + runs: readonly Row[]; + wakes: readonly Row[]; + statusChanges: readonly Row[]; + controllers: readonly ControllerRow[]; + windows: readonly ServerProcessWindow[]; +} + +export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) { + const runs = input.runs.filter(run => run.issueId === issueId); + const wakeIds = new Set(runs.map(run => text(run.wakeupRequestId)).filter(Boolean)); + const wakes = input.wakes.filter(wake => wake.issueId === issueId || wakeIds.has(text(wake.id))); + const controllers = boundByProcessExit(input.controllers, input.windows); + const controllerByBoot = new Map(controllers.map(controller => [controller.bootId, controller])); + return { + issueId, + runs: runs.map(run => { + const owner = controllerByBoot.get(String(run.controllerBootId ?? "")); + return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail), + wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), + contextSnapshot: run.contextSnapshot ?? null, + owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, + servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) }; + }), + wakes: wakes.map(wake => ({ id: text(wake.id), reason: text(wake.reason), source: text(wake.source), triggerDetail: text(wake.triggerDetail), + requestedByActorType: text(wake.requestedByActorType), requestedByActorId: text(wake.requestedByActorId), idempotencyKey: text(wake.idempotencyKey), + status: text(wake.status), runId: text(wake.runId), payload: wake.payload ?? null, error: text(wake.error), + createdAt: ms(wake.createdAt), requestedAt: ms(wake.requestedAt), claimedAt: ms(wake.claimedAt), finishedAt: ms(wake.finishedAt), + servingAtCreate: servingAt(ms(wake.createdAt), controllers, input.windows) })), + statusChanges: input.statusChanges.filter(change => change.issueId === issueId).map(change => ({ at: ms(change.createdAt), action: text(change.action), + actorType: text(change.actorType), actorId: text(change.actorId), runId: text(change.runId), from: text(change.fromStatus), to: text(change.toStatus) })), + }; +} + +async function readRows(db: Pick, companyId: string, shifts: readonly AgingShift[]): Promise> { + const client = db.$client; + const runs = await client.unsafe( + `select id::text as id, issue_id::text as "issueId", status, invocation_source as "invocationSource", trigger_detail as "triggerDetail", + wakeup_request_id::text as "wakeupRequestId", context_snapshot as "contextSnapshot", controller_boot_id::text as "controllerBootId", + floor(extract(epoch from created_at) * 1000) as "createdAt", floor(extract(epoch from started_at) * 1000) as "startedAt", + floor(extract(epoch from finished_at) * 1000) as "finishedAt" + from heartbeat_runs where company_id::text = $1 order by created_at`, [companyId]) as Row[]; + const wakes = await client.unsafe( + `select id::text as id, payload->>'issueId' as "issueId", source, trigger_detail as "triggerDetail", reason, payload, status, + requested_by_actor_type as "requestedByActorType", requested_by_actor_id as "requestedByActorId", idempotency_key as "idempotencyKey", + run_id::text as "runId", error, floor(extract(epoch from created_at) * 1000) as "createdAt", + floor(extract(epoch from requested_at) * 1000) as "requestedAt", floor(extract(epoch from claimed_at) * 1000) as "claimedAt", + floor(extract(epoch from finished_at) * 1000) as "finishedAt" + from agent_wakeup_requests where company_id::text = $1 order by created_at`, [companyId]) as Row[]; + const statusChanges = await client.unsafe( + `select entity_id as "issueId", action, actor_type as "actorType", actor_id as "actorId", run_id::text as "runId", + coalesce(details->'_previous'->>'status', details->>'previousStatus') as "fromStatus", details->>'status' as "toStatus", + floor(extract(epoch from created_at) * 1000) as "createdAt" + from activity_log where company_id::text = $1 and entity_type = 'issue' + and (details ? 'status' or details->'_previous' ? 'status' or details ? 'previousStatus') order by created_at`, [companyId]) as Row[]; + const controllers = (await client.unsafe( + `select boot_id as "bootId", epoch, process_space as "processSpace", pid, floor(extract(epoch from started_at) * 1000) as "startedAt", + floor(extract(epoch from coalesce(stopped_at, fenced_at)) * 1000) as "endedAt" from controller_instances order by epoch`) as Row[]) + .map(row => ({ bootId: String(row.bootId), epoch: Number(row.epoch), processSpace: String(row.processSpace), pid: Number(row.pid), + startedAt: Number(row.startedAt), endedAt: ms(row.endedAt) })); + return { + runs: unshiftRows(runs, ["createdAt", "startedAt", "finishedAt"], ["contextSnapshot"], shifts), + wakes: unshiftRows(wakes, ["createdAt", "requestedAt", "claimedAt", "finishedAt"], ["payload"], shifts), + statusChanges: unshiftRows(statusChanges, ["createdAt"], [], shifts), + controllers: controllers.map(controller => ({ ...controller, startedAt: unshiftMs(controller.startedAt, shifts), + endedAt: controller.endedAt === null ? null : unshiftMs(controller.endedAt, shifts) })), + }; +} + +export async function captureWakeEvidence(input: { db: Pick; companyId: string; waveDir: string; + observers: ReadonlyArray<{ plan: { taskId: string }; record: { issueId: string | null } }> | null; windows: readonly ServerProcessWindow[]; + shifts?: readonly AgingShift[]; redact: (text: string) => string }): Promise { + const rows = await readRows(input.db, input.companyId, input.shifts ?? []); + const observers = input.observers ?? [...new Set(rows.runs.map(run => text(run.issueId)).filter((id): id is string => id !== null))] + .map(issueId => ({ plan: { taskId: `issue-${issueId}` }, record: { issueId } })); + const directory = path.join(input.waveDir, "wake-evidence"); + await mkdir(directory, { recursive: true, mode: 0o700 }); + let written = 0; + for (const observer of observers) { + if (!observer.record.issueId) continue; + const evidence = { schema: "paperclip.restart-chaos.wake-evidence/v1", taskId: observer.plan.taskId, + ...wakeEvidenceForIssue(observer.record.issueId, { ...rows, windows: input.windows }) }; + await writeFile(path.join(directory, `${observer.plan.taskId}.json`), input.redact(`${JSON.stringify(evidence, null, 2)}\n`), { mode: 0o600 }); + written += 1; + } + await writeFile(path.join(directory, "servers.json"), `${JSON.stringify({ controllers: rows.controllers, harnessWindows: input.windows }, null, 2)}\n`, { mode: 0o600 }); + return written; +} + +export function wakeEvidenceOnce(input: { databaseUrl: () => string; companyId: string; waveDir: string; + observers: () => ReadonlyArray<{ plan: { taskId: string }; record: { issueId: string | null } }>; windows: readonly ServerProcessWindow[]; + shifts: readonly AgingShift[]; redact: (text: string) => string; note: (kind: string, data: Record) => void; + open?: (url: string) => Pick }): () => Promise { + let done: Promise | null = null; + return () => done ??= (async () => { + let observers: ReturnType | null = null; + try { observers = input.observers(); } catch { observers = null; } + const db = (input.open ?? createDb)(input.databaseUrl()); + try { + const written = await captureWakeEvidence({ db, companyId: input.companyId, waveDir: input.waveDir, observers, windows: input.windows, + shifts: input.shifts, redact: input.redact }); + input.note("wake_evidence_captured", { written, byTask: observers !== null }); + } catch (error) { + input.note("wake_evidence_failed", { error: String(error) }); + } finally { + await db.$client.end({ timeout: 5 }).catch(() => undefined); + } + })(); +}