import { spawnSync } from "node:child_process"; import { randomUUID } from "node:crypto"; import { readFile, readdir } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApiError, type ChaosApi } from "./api.js"; import { ACTIVE_RUN_STATUSES, isStalledSample, STALL_LIMIT_MS } from "./grader.js"; import type { ChaosLab } from "./seed.js"; import type { Timeline } from "./timeline.js"; import type { AdapterId, GovernanceRecord, InterruptionSample, OracleArtifacts, RestSweep, RetryAttempt, RunSample, ScheduledFault, TaskPlan, TaskRecord, TaskSample, } from "./types.js"; import { restartProcessStartMatches } from "../runner-e2e/process-tree-owner.js"; import { runOracleCommand } from "./oracle-sandbox.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; const IGNORED_DIRECTORIES = new Set([".git", ".bench", "__pycache__", "node_modules"]); const MAX_SNAPSHOT_FILES = 2_000; /** Relative paths of workspace files, excluding VCS and cache directories. */ export async function listWorkspaceFiles(root: string): Promise { const out: string[] = []; const walk = async (relative: string) => { if (out.length >= MAX_SNAPSHOT_FILES) return; const entries = await readdir(path.join(root, relative), { withFileTypes: true }); for (const entry of entries) { const child = relative ? `${relative}/${entry.name}` : entry.name; if (entry.isDirectory()) { if (!IGNORED_DIRECTORIES.has(entry.name)) await walk(child); } else if (entry.isFile()) out.push(child); if (out.length >= MAX_SNAPSHOT_FILES) return; } }; await walk(""); return out.sort(); } export const QUEUED_TRIGGER_MAX_EMPTY_SAMPLES = 3; function stillQueued(runs: readonly RunSample[]): boolean { return runs.some(run => run.status === "queued" && !run.startedAt) && !runs.some(run => run.startedAt || run.status === "running"); } export const TRIGGER_LATE_MS = 10_000; /** The run settles soon after acceptance; arming on the proposal keeps the window reachable. */ const RESULT_TRIGGER_EVENTS = new Set(["run.result.proposed", "run.result.accepted"]); /** Rest must hold for this long before the observer acts on it. */ export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; export const CHILD_STALL_CHECK_MS = 10_000; export const STOP_CONTINUE_AFTER_MS = 20_000; export function childrenPending(children: ReadonlyArray<{ status: string | null }> | null): boolean { return !children || children.some(child => !["done", "cancelled", "blocked"].includes(child.status ?? "")); } /** Extra time granted after a Retry click before R3 is graded as timed out. */ export const RETRY_BUDGET_MS = 15 * 60_000; /** H1: how long the pause must hold with the server ready before the board resumes. */ export const BUDGET_PAUSE_HOLD_MS = 30_000; /** H1: the raised budget's headroom above observed spend, in cents. */ export const BUDGET_RESUME_HEADROOM_CENTS = 3_000; const DECISION_NOTE = "Restart chaos harness: board decision after a control-plane restart."; type Json = Record; export function toRunSample(run: Json): RunSample { const list = (value: unknown) => (Array.isArray(value) ? value.filter((item): item is string => typeof item === "string") : []); const wake = new Set([...list(run.wakeCommentIds), ...(typeof run.wakeCommentId === "string" ? [run.wakeCommentId] : [])]); return { id: String(run.runId ?? run.id), status: String(run.status), createdAt: run.createdAt ?? null, startedAt: run.startedAt ?? null, finishedAt: run.finishedAt ?? null, errorCode: run.errorCode ?? null, retryOfRunId: run.retryOfRunId ?? null, wakeCommentIds: [...wake], livenessState: run.livenessState ?? null, lastEvidenceAt: latestIso([run.startedAt, run.lastUsefulActionAt, run.execution?.lastConfirmedActivityAt]), }; } interface RunEventCursor { seq: number; lastAt: string | null; /** Tool starts minus tool ends. PRP item ids are not unique per tool call, so this counts. */ openTools: number; /** The run's latest recorded local runner process, until recovery reports it stopped. */ process: { pid: number; startedAt: string } | null; } /** R5 liveness: a tool in flight, or a recorded local runner process. */ export function noteLivenessEvent(cursor: RunEventCursor, event: Json) { const payload = (event.payload ?? {}) as Json; switch (event.eventType) { case "tool.execution.started": cursor.openTools += 1; break; case "tool.execution.completed": cursor.openTools = Math.max(0, cursor.openTools - 1); break; case "native.process_identity_recorded": if (payload.targetKind === "local" && Number.isSafeInteger(payload.processPid) && typeof payload.processStartedAt === "string") { // A replacement runner cannot finish a tool its predecessor started. if (cursor.process && cursor.process.pid !== payload.processPid) cursor.openTools = 0; cursor.process = { pid: payload.processPid, startedAt: payload.processStartedAt }; } break; case "native.local_process_stopped": // A stopped runner's in-flight tools can never send their completion. if (cursor.process && payload.processPid === cursor.process.pid) { cursor.process = null; cursor.openTools = 0; } break; } } /** Alive only when the PID still runs with the recorded start time, so a recycled PID never counts. */ export function localProcessAlive(pid: number, startedAt: string): boolean { if (process.platform === "win32") return false; const result = spawnSync("ps", ["-o", "lstart=", "-p", String(pid)], { encoding: "utf8", env: { PATH: "/usr/bin:/bin", LANG: "C", LC_ALL: "C" }, timeout: 5_000 }); const started = Date.parse(result.stdout?.trim() ?? ""); return Number.isFinite(started) && restartProcessStartMatches(new Date(started).toISOString(), startedAt); } export function latestIso(values: ReadonlyArray): string | null { let best: number | null = null; for (const value of values) { const at = typeof value === "string" ? Date.parse(value) : NaN; if (Number.isFinite(at) && (best === null || at > best)) best = at; } return best === null ? null : new Date(best).toISOString(); } export const RUN_EVENT_PAGE = 500; /** G1: an approval this task asked for (its title names the nonce; a redacted title still counts). */ export function approvalTitleMatches(approval: Json, nonce: string): boolean { const title = approval.payload?.title; return typeof title !== "string" || title.includes(nonce); } /** H1: runs that started at or after `since` and before `until` (epoch ms), across every sample. */ export function runsStartedAfter(samples: readonly TaskSample[], since: number, until = Number.POSITIVE_INFINITY): string[] { const started = new Set(); for (const sample of samples) for (const run of sample.runs ?? []) { const at = run.startedAt ? Date.parse(run.startedAt) : Number.NaN; if (Number.isFinite(at) && at >= since && at < until) started.add(run.id); } return [...started]; } export function toInterruption(value: Json | null | undefined): InterruptionSample | null { if (!value) return null; return { state: String(value.state), canRetry: value.canRetry === true, retryWait: Boolean(value.retryWait), workspaceFailure: value.workspaceFailure?.message ?? null, observedRunId: value.observedRunId ?? null, userAction: value.userAction ?? null }; } /** Rest = no live run and nothing the system itself is still doing. */ export function isResting(sample: TaskSample): boolean { if (!sample.reachable || !sample.serverReady) return false; if ((sample.runs ?? []).some(run => ACTIVE_RUN_STATUSES.has(run.status))) return false; if (sample.scheduledRetry) return false; // A board-owned recovery waits on a person: that is a resting state. if (sample.activeRecoveryAction && sample.recoveryOwner !== "board") return false; const interruption = sample.interruption; if (interruption && (interruption.state === "working" || interruption.state === "retry_scheduled" || interruption.retryWait)) return false; // Unexplained idle work is a possible stall, not a rest: keep watching it so // the task can still resume and R5 can measure the stall. if (isStalledSample(sample)) return false; return true; } /** Stop watching an unexplained idle task once it exceeds the R5 stall limit. */ export const STALL_GIVE_UP_MS = STALL_LIMIT_MS + 15_000; export class TaskObserver { readonly record: TaskRecord; issue: Json | null = null; private restCandidateSince: number | null = null; private childrenSettledAt: number | null = null; private childStallCheckedAt: number | null = null; private stallSince: number | null = null; /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ private readySince: number | null = null; private readonly faultSchedule: ScheduledFault[]; private readonly pendingFaults: ScheduledFault[]; done = false; /** Replaceable in tests. A Daytona run's process is remote, so only local runs are checked. */ processAlive = (pid: number, startedAt: string) => this.lab.environmentKind === "local" && localProcessAlive(pid, startedAt); constructor(readonly plan: TaskPlan, private readonly api: ChaosApi, private readonly lab: ChaosLab, private readonly timeline: Timeline, private readonly supervisor: { serverReady: boolean; faultInProgress: boolean; serverBuild?: "upgrade_from" | "current" }, /** Paths oracle commands may not read: the operator's home, the instance root (secrets key, database) and results. */ private readonly protectedRoots: readonly string[] = [os.homedir()]) { this.record = { plan, issueId: null, identifier: null, createdAt: null, launchError: null, samples: [], retry: null, restAt: null, finalStatus: null, deadlineAt: null, timedOut: false, followUp: null, faultIds: [], sweep: null, preFaultFiles: null, lostFiles: null }; this.faultSchedule = taskFaultSchedule(plan); this.pendingFaults = [...this.faultSchedule]; } get definition() { return workloadDefinitions[this.plan.workload]; } /** * Crash-point suite: create the issue without an assignee, so no wake is * queued yet. `release` assigns it once the batch's point is armed. */ async stage(): Promise { await this.launch({ assign: false }); } /** * Assigns a staged issue, which queues its wake. An earlier task of the same * batch can crash the server meanwhile, so retry until the replacement * answers; setting the same assignee again is idempotent. */ async release(deadlineMs = 10 * 60_000): Promise { if (this.done || !this.record.issueId) return; const until = Date.now() + deadlineMs; for (;;) { try { await this.api.patch(`/api/issues/${this.record.issueId}`, { assigneeAgentId: this.lab.agents[this.plan.adapter].id }); break; } catch (error) { const transient = error instanceof ChaosApiError && (error.status === null || error.status >= 500); if (!transient || Date.now() > until) { this.record.launchError = error instanceof Error ? error.message : String(error); this.done = true; this.timeline.add("task_launch_failed", { taskId: this.plan.taskId, error: this.record.launchError }); return; } await delay(1_000); } } this.record.createdAt = Date.now(); this.record.deadlineAt = this.record.createdAt + this.definition.budgetMs; this.timeline.add("task_released", { taskId: this.plan.taskId, issueId: this.record.issueId }); } async launch(options: { assign?: boolean } = {}): Promise { const definition = this.definition; const assign = options.assign !== false; if (this.plan.workload === "H1") await this.lowerBudgetToSpend(); try { const issue = await this.api.post(`/api/companies/${this.lab.companyId}/issues`, { title: definition.title(this.plan.nonce), description: definition.description(this.plan.nonce, this.workloadAgents()), status: "todo", priority: "medium", ...(assign ? { assigneeAgentId: this.lab.agents[this.plan.adapter].id } : {}), ...(definition.usesRepository ? { projectId: this.lab.projectId } : {}), }); this.record.issueId = String(issue.id); this.record.identifier = issue.identifier ?? null; // K14: which build launched the task, so a replay starts it on the same one. if (this.supervisor.serverBuild) this.record.serverBuildAtLaunch = this.supervisor.serverBuild; this.record.createdAt = Date.now(); this.record.deadlineAt = this.record.createdAt + definition.budgetMs; this.timeline.add("task_launched", { taskId: this.plan.taskId, issueId: issue.id, identifier: issue.identifier, workload: this.plan.workload, adapter: this.plan.adapter, faults: this.faultSchedule }); } catch (error) { this.record.launchError = error instanceof Error ? error.message : String(error); this.done = true; this.timeline.add("task_launch_failed", { taskId: this.plan.taskId, error: this.record.launchError }); } } /** The next scheduled fault once it is due and the task has not rested yet. */ faultDue(now: number): ScheduledFault | null { if (this.done || this.record.createdAt === null) return null; for (const fault of [...this.pendingFaults]) { const armedAt = this.triggerArmedAt(fault); if (armedAt === null) continue; const dueAt = armedAt === "missed" ? 0 : armedAt + (fault.delayMs ?? 0); if (armedAt !== "missed" && now < dueAt) continue; this.pendingFaults.splice(this.pendingFaults.indexOf(fault), 1); const triggered = fault.trigger !== undefined && fault.trigger !== "time"; if (armedAt === "missed" || (triggered && !this.triggerWindowOpen(fault, now, dueAt))) { this.noteTriggerMissed(fault); continue; } return fault; } return null; } private triggerWindowOpen(fault: ScheduledFault, now: number, dueAt: number): boolean { if (now - dueAt > TRIGGER_LATE_MS) return false; if (fault.trigger === "result") return this.resultRunStillRunning(this.latestReachableRuns()); if (fault.trigger !== "queued") return true; const latest = [...this.record.samples].reverse().find(sample => sample.reachable); return stillQueued(latest?.runs ?? []); } async confirmTrigger(fault: ScheduledFault, now: number): Promise { if (!fault.trigger || fault.trigger === "time") return true; const armedAt = this.triggerArmedAt(fault); let open = typeof armedAt === "number" && now - (armedAt + (fault.delayMs ?? 0)) <= TRIGGER_LATE_MS; if (open && fault.trigger === "result") { const runs = await this.api.get(`/api/issues/${this.record.issueId}/runs`, 5_000) .then(rows => (rows ?? []).map(toRunSample)).catch(() => null); open = runs !== null && this.resultRunStillRunning(runs); } if (open && fault.trigger === "queued") { const runs = await this.api.get(`/api/issues/${this.record.issueId}/runs`, 5_000) .then(rows => (rows ?? []).map(toRunSample)).catch(() => null); open = runs !== null && stillQueued(runs); } if (!open) this.noteTriggerMissed(fault); return open; } private latestReachableRuns(): RunSample[] { return [...this.record.samples].reverse().find(sample => sample.reachable)?.runs ?? []; } /** A `result` fault outside the accepted-but-unsettled window would test a different boundary. */ private resultRunStillRunning(runs: readonly RunSample[]): boolean { return runs.some(run => run.id === this.record.resultRunId && run.status === "running"); } /** A due fault the wave decided not to fire (for example, the storm ended during its checks). */ markTriggerMissed(fault: ScheduledFault) { if (fault.trigger && fault.trigger !== "time") this.noteTriggerMissed(fault); } private noteTriggerMissed(fault: ScheduledFault) { const trigger = fault.trigger ?? "time"; (this.record.triggerMissed ??= []).push(trigger); this.timeline.add("trigger_missed", { taskId: this.plan.taskId, trigger, fault: fault.kind }); } private triggerArmedAt(fault: ScheduledFault): number | null | "missed" { const createdAt = this.record.createdAt!; switch (fault.trigger ?? "time") { case "time": return createdAt + fault.atMs; case "comment": { const followUp = this.record.followUp; if (!followUp) return null; return followUp.commentId && followUp.duringRun ? followUp.postedAt ?? followUp.at : "missed"; } case "retry": { const retry = this.record.retry; if (!retry) return null; return retry.accepted ? retry.acceptedAt ?? retry.at : "missed"; } case "queued": { const first = this.record.samples.find(sample => sample.reachable && (sample.runs ?? []).length > 0); if (!first) return this.record.samples.filter(sample => sample.reachable).length >= QUEUED_TRIGGER_MAX_EMPTY_SAMPLES ? "missed" : null; const queued = (first.runs ?? []).some(run => run.status === "queued" && !run.startedAt); const started = (first.runs ?? []).some(run => run.startedAt || run.status === "running"); return queued && !started ? first.at : "missed"; } case "tool": return this.record.benchFirstSeenAt ?? null; case "approval": return this.record.governance?.approvalSeenAt ?? null; case "paused": return this.record.governance?.budgetPausedAt ?? null; case "stop": { const stop = this.record.stop; if (!stop) return null; return stop.error ? "missed" : stop.at; } case "result": { // B8: the run that received the follow-up has accepted its result but // has not settled yet. A follow-up posted at rest cannot cross it. const followUp = this.record.followUp; if (!followUp) return null; if (!followUp.commentId || !followUp.duringRun) return "missed"; return this.record.resultAcceptedAt ?? null; } } } private closeUnfiredTriggers() { for (const fault of this.pendingFaults.splice(0)) if (fault.trigger && fault.trigger !== "time") this.noteTriggerMissed(fault); } /** * B5: read bench.sh's progress file while the task is live. The harness sees * the values change over minutes, which a computed answer cannot produce. */ private async observeBenchProgress(): Promise { if (this.plan.workload !== "B5") return; const cwd = (this.issue?.currentExecutionWorkspace as Json | null | undefined)?.cwd; if (typeof cwd !== "string") return; const value = (await readFile(path.join(cwd, ".bench", "progress"), "utf8").catch(() => "")).trim(); if (!value) return; const seen = this.record.benchProgressSeen ??= []; if (!seen.includes(value)) seen.push(value); this.record.benchFirstSeenAt ??= Date.now(); } /** R2: remember which workspace files existed before the first fault hit this task. */ async snapshotBeforeFault(): Promise { if (this.record.preFaultFiles !== null || this.done) return; // One task's snapshot failure must not end the wave; R2 fails with the reason. try { const workspace = await this.workspacePath(); if (workspace) { this.record.preFaultFiles = await listWorkspaceFiles(workspace); this.record.preFaultWorkspacePath = workspace; } } catch (error) { this.record.preFaultSnapshotError = error instanceof Error ? error.message : String(error); this.timeline.add("pre_fault_snapshot_failed", { taskId: this.plan.taskId, error: this.record.preFaultSnapshotError }); } } noteFault(faultId: string, startedAt?: number) { if (!this.record.faultIds.includes(faultId)) this.record.faultIds.push(faultId); if (startedAt !== undefined) this.record.lastFaultAt = Math.max(this.record.lastFaultAt ?? 0, startedAt); } markUnfinished() { if (this.done) return; const last = this.record.samples.at(-1); this.record.timedOut = true; this.record.finalStatus = last?.issueStatus ?? this.record.finalStatus; this.done = true; this.timeline.add("task_unfinished_at_interrupt", { taskId: this.plan.taskId, status: this.record.finalStatus }); } /** Outages extend the deadline: R1 allows budget plus outage time. */ extendDeadline(ms: number) { if (this.record.deadlineAt !== null) this.record.deadlineAt += ms; } async poll(now = Date.now()): Promise { if (this.done || !this.record.issueId) return; const sample = await this.sample(now); this.record.samples.push(sample); this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); await this.governanceStep(sample, now); if (await this.observeChildStalls(now)) { this.restCandidateSince = null; this.childrenSettledAt = null; } if (await this.maybeStop(sample, now)) { this.restCandidateSince = null; return; } if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; return; } // A task waiting on the board (G1's approval, H1's paused company) is in a named wait, not a stall. this.stallSince = isStalledSample(sample) && !this.awaitingBoard() ? (this.stallSince ?? now) : null; if (this.stallSince !== null && now - this.stallSince > STALL_GIVE_UP_MS) { this.record.restAt = this.stallSince; this.record.finalStatus = sample.issueStatus ?? null; this.done = true; this.timeline.add("task_stalled", { taskId: this.plan.taskId, status: sample.issueStatus, idleMs: now - this.stallSince }); return; } if (!isResting(sample) || this.supervisor.faultInProgress) { this.restCandidateSince = null; } else { this.restCandidateSince ??= now; if (now - this.restCandidateSince >= REST_CONFIRM_MS) { await this.onRest(sample, now); // A task still waiting (for the board, or on a Retry it just got) keeps its deadline. if (this.done) return; } } if (this.record.deadlineAt !== null && now > this.record.deadlineAt && !this.supervisor.faultInProgress) { this.record.timedOut = true; this.record.finalStatus = sample.issueStatus ?? this.record.finalStatus; this.done = true; this.timeline.add("task_timed_out", { taskId: this.plan.taskId, status: sample.issueStatus }); } } private readonly runEventCursors = new Map(); private async runEvidence(run: RunSample): Promise { if (run.status !== "running") return run; const cursor = this.runEventCursors.get(run.id) ?? { seq: 0, lastAt: null, openTools: 0, process: null }; for (let page = 0; page < 3; page += 1) { const events = await this.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${cursor.seq}&limit=${RUN_EVENT_PAGE}`, 5_000).catch(() => null); if (!events?.length) break; for (const event of events) { if (typeof event.seq === "number" && event.seq > cursor.seq) cursor.seq = event.seq; if (RESULT_TRIGGER_EVENTS.has(String(event.eventType)) && this.record.followUp?.duringRun && !this.record.resultAcceptedAt) { this.record.resultAcceptedAt = Date.now(); this.record.resultRunId = run.id; } cursor.lastAt = latestIso([cursor.lastAt, event.createdAt]); noteLivenessEvent(cursor, event); } if (events.length < RUN_EVENT_PAGE) break; } this.runEventCursors.set(run.id, cursor); return { ...run, lastEvidenceAt: latestIso([run.lastEvidenceAt, cursor.lastAt]), ...(cursor.openTools > 0 ? { toolInFlight: true } : {}), ...(cursor.process && this.processAlive(cursor.process.pid, cursor.process.startedAt) ? { processAlive: true } : {}) }; } private async sample(now: number): Promise { const serverReady = this.supervisor.serverReady && !this.supervisor.faultInProgress; try { const [issue, runs] = await Promise.all([ this.api.get(`/api/issues/${this.record.issueId}`, 5_000), this.api.get(`/api/issues/${this.record.issueId}/runs`, 5_000), ]); this.issue = issue; return { at: now, reachable: true, serverReady, issueStatus: String(issue.status), interruption: toInterruption(issue.interruption), activeRecoveryAction: Boolean(issue.activeRecoveryAction), recoveryOwner: issue.activeRecoveryAction?.ownerType ?? null, executionBlocker: Boolean(issue.executionBlocker), scheduledRetry: Boolean(issue.scheduledRetry), runs: await Promise.all((runs ?? []).map(toRunSample).map(run => this.runEvidence(run))) }; } catch { return { at: now, reachable: false, serverReady: false }; } } private async maybeStop(sample: TaskSample, now: number): Promise { if (!this.definition.stopBeforeFollowUp || this.record.stop || this.plan.followUpAfterMs === null || !sample.reachable) return false; if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; const running = (sample.runs ?? []).find(run => run.status === "running"); if (!running) return false; const stop: NonNullable = { at: now, runId: running.id, httpStatus: null, error: null }; try { await this.api.post(`/api/heartbeat-runs/${running.id}/cancel`, { cancellationRequestId: randomUUID() }); stop.httpStatus = 200; } catch (error) { stop.httpStatus = error instanceof ChaosApiError ? error.status : null; stop.error = error instanceof Error ? error.message : String(error); } this.record.stop = stop; this.timeline.add("task_stop_clicked", { taskId: this.plan.taskId, ...stop }); return true; } private async maybeFollowUp(sample: TaskSample, now: number): Promise { const definition = this.definition; if (!definition.followUp || this.record.followUp || this.plan.followUpAfterMs === null || !sample.reachable) return false; if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; if (definition.stopBeforeFollowUp) { const stop = this.record.stop; if (!stop || now - stop.at < STOP_CONTINUE_AFTER_MS || this.supervisor.faultInProgress) return false; if ((sample.runs ?? []).some(run => run.status === "running" || run.status === "queued")) return false; } // C1 targets a live run (B8); wait for one unless the task already rests. const running = (sample.runs ?? []).some(run => run.status === "running"); if (!running && !definition.stopBeforeFollowUp && !isResting(sample)) return false; const sentAt = Math.max(now, Date.now()); try { const comment = await this.api.post(`/api/issues/${this.record.issueId}/comments`, { body: definition.followUp(this.plan.nonce) }); this.record.followUp = { at: now, sentAt, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; } catch (error) { this.record.followUp = { at: now, sentAt, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; } this.timeline.add("task_follow_up", { taskId: this.plan.taskId, ...this.record.followUp }); return true; } private async onRest(sample: TaskSample, now: number) { const status = sample.issueStatus ?? null; // The board has not acted yet: the task waits on it, so it has not rested (and gets no Retry). if (this.awaitingBoard()) { this.restCandidateSince = null; return; } if (this.plan.workload === "H1") { const company = await this.api.get(`/api/companies/${this.lab.companyId}`, 5_000).catch(() => null); (this.record.governance ??= {}).companyActiveAtRest = company?.status === "active"; } // C1: a follow-up posted after rest must be served before the task counts as resting. if (this.definition.followUp && !this.record.followUp) return; const faultSinceRetry = this.record.retry !== null && (this.record.lastFaultAt ?? 0) > this.record.retry.at; if (status !== "done" && this.definition.collect.childDocuments && this.record.issueId) { const children = await this.collectChildren(this.record.issueId).catch(() => null); const pending = childrenPending(children); const blocked = (children ?? []).filter(child => child.status === "blocked").map(child => child.id); if (!pending && blocked.length && this.childrenSettledAt === null) { this.timeline.add("task_child_blocked", { taskId: this.plan.taskId, childIssueIds: blocked }); } this.childrenSettledAt = pending ? null : (this.childrenSettledAt ?? now); if (pending || now - this.childrenSettledAt! < CHILD_WAKE_GRACE_MS) { this.restCandidateSince = null; return; } } if (status !== "done" && (!this.record.retry || faultSinceRetry) && sample.interruption?.canRetry) { if (this.record.retry) (this.record.priorRetries ??= []).push(this.record.retry); this.record.retry = await this.retry(sample, now); this.restCandidateSince = null; if (this.record.retry.accepted && this.record.deadlineAt !== null) this.record.deadlineAt = Math.max(this.record.deadlineAt, now + RETRY_BUDGET_MS); return; } this.record.restAt = this.restCandidateSince ?? now; this.record.finalStatus = status; this.done = true; this.timeline.add("task_rested", { taskId: this.plan.taskId, status, interruption: sample.interruption?.state ?? null, canRetry: sample.interruption?.canRetry ?? false, retried: Boolean(this.record.retry) }); } /** Exactly one click, with the same body ui/src/hooks/useTaskRetry.ts sends. */ private async observeChildStalls(now: number): Promise { if (this.plan.workload !== "M1" || !this.record.issueId) return false; if (this.childStallCheckedAt !== null && now - this.childStallCheckedAt < CHILD_STALL_CHECK_MS) return false; this.childStallCheckedAt = now; const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(this.record.issueId)}&limit=20`) .catch(() => null); const blocked = (Array.isArray(rows) ? rows : []).filter(row => row.parentId === this.record.issueId && row.status === "blocked").map(row => String(row.id)); const parent = await this.api.get(`/api/issues/${this.record.issueId}`).catch(() => null); const stalled = Array.isArray(parent?.stalledDependencies) ? parent.stalledDependencies as Json[] : []; for (const dependency of stalled) { if (dependency.relation !== "child" || typeof dependency.stalledIssueId !== "string") continue; const action = (dependency.recoveryAction ?? null) as Json | null; const childIssueId = dependency.stalledIssueId; const recoveryActionId = typeof action?.id === "string" ? action.id : null; const stalls = this.record.childStalls ??= []; if (!stalls.some(stall => stall.childIssueId === childIssueId && stall.recoveryActionId === recoveryActionId)) { stalls.push({ childIssueId, recoveryActionId }); this.timeline.add("task_child_stalled", { taskId: this.plan.taskId, childIssueId, recoveryActionId }); } } return this.retryBlockedChildren(blocked, now); } private stopArtifacts(stop: NonNullable): NonNullable { const runs = new Map[number]>(); for (const sample of this.record.samples) for (const run of sample.runs ?? []) runs.set(run.id, run); const continueAt = this.record.followUp?.sentAt ?? this.record.followUp?.at ?? null; const continueCommentId = this.record.followUp?.commentId ?? null; const startedBeforeContinue = [...runs.values()].filter(run => run.id !== stop.runId && run.createdAt && !(continueCommentId && (run.wakeCommentIds ?? []).includes(continueCommentId)) && Date.parse(run.createdAt) > stop.at && (continueAt === null || Date.parse(run.createdAt) < continueAt)).map(run => run.id); return { runId: stop.runId, runStatus: runs.get(stop.runId)?.status ?? null, runsStartedBeforeContinue: startedBeforeContinue, continuePosted: Boolean(this.record.followUp?.commentId) }; } private async retryBlockedChildren(childIds: readonly string[], now: number): Promise { let clicked = false; for (const childIssueId of childIds) { if ((this.record.childRetries ?? []).some(attempt => attempt.childIssueId === childIssueId && attempt.accepted)) continue; const child = await this.api.get(`/api/issues/${childIssueId}`).catch(() => null); const interruption = child?.interruption as Json | null | undefined; if (interruption?.canRetry !== true) continue; const observedRunId = typeof interruption.observedRunId === "string" ? interruption.observedRunId : null; const attempt: RetryAttempt & { childIssueId: string } = { childIssueId, at: now, observedRunId, httpStatus: null, accepted: null, error: null }; try { const result = await this.api.post<{ accepted?: boolean }>(`/api/issues/${childIssueId}/retry`, { idempotencyKey: randomUUID(), observedRunId }); attempt.httpStatus = 200; attempt.accepted = result?.accepted === true; attempt.acceptedAt = Math.max(now, Date.now()); } catch (error) { attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; attempt.error = error instanceof Error ? error.message : String(error); } (this.record.childRetries ??= []).push(attempt); this.timeline.add("task_child_retry_clicked", { taskId: this.plan.taskId, ...attempt }); clicked = clicked || attempt.accepted === true; } return clicked; } private async collectStallWakes(issueId: string): Promise> { const rows = await this.api.get(`/api/issues/${issueId}/activity`).catch(() => []); return (rows ?? []).filter(row => row.action === "issue.blocker_stalled_wake").map(row => { const details = (row.details ?? {}) as Json; return { stalledIssueId: typeof details.stalledIssueId === "string" ? details.stalledIssueId : null, recoveryActionId: typeof details.recoveryActionId === "string" ? details.recoveryActionId : null }; }); } private async retry(sample: TaskSample, now: number): Promise { const observedRunId = sample.interruption?.observedRunId ?? (this.issue?.executionBlocker?.runId as string | undefined) ?? null; const attempt: RetryAttempt = { at: now, observedRunId, httpStatus: null, accepted: null, error: null }; try { const result = await this.api.post<{ accepted?: boolean }>(`/api/issues/${this.record.issueId}/retry`, { idempotencyKey: randomUUID(), observedRunId }); attempt.httpStatus = 200; attempt.accepted = result?.accepted === true; attempt.acceptedAt = Math.max(now, Date.now()); } catch (error) { attempt.httpStatus = error instanceof ChaosApiError ? error.status : null; attempt.error = error instanceof Error ? error.message : String(error); } this.timeline.add("task_retry_clicked", { taskId: this.plan.taskId, ...attempt }); return attempt; } async workspacePath(): Promise { if (!this.definition.usesRepository) return null; const current = this.issue?.currentExecutionWorkspace as Json | null | undefined; if (typeof current?.cwd === "string") return current.cwd; const workspaceId = this.issue?.executionWorkspaceId; if (typeof workspaceId === "string") { const workspace = await this.api.get(`/api/execution-workspaces/${workspaceId}`).catch(() => null); if (typeof workspace?.cwd === "string") return workspace.cwd; } return null; } workloadAgents(): WorkloadAgents { const self = this.lab.agents[this.plan.adapter]; const other = Object.values(this.lab.agents).find(agent => agent.id !== self.id) ?? self; return { self, other, companyId: this.lab.companyId }; } /** G1 before the board's decision, or H1 while its company is paused and not yet resumed. */ private awaitingBoard(): boolean { const governance = this.record.governance; if (this.plan.workload === "G1") return Boolean(governance?.approvalSeenAt) && !governance?.decision; if (this.plan.workload === "H1") return Boolean(governance?.budgetPausedAt) && !governance?.resume && !governance?.pauseLostAt; return false; } /** The board acts only after every fault this task planned has fired (or was missed), with the server ready. */ private boardMayAct(sample: TaskSample): boolean { return sample.reachable && sample.serverReady && !this.supervisor.faultInProgress && this.pendingFaults.length === 0; } /** * H1: set the company budget to its observed spend plus one cent through the board's budget * route, so the task's first priced usage crosses the hard stop mid-run. */ private async lowerBudgetToSpend(): Promise { const governance = this.record.governance ??= {}; try { const overview = await this.api.get(`/api/companies/${this.lab.companyId}/budgets/overview`, 10_000); const policy = ((overview?.policies ?? []) as Json[]).find(row => row.scopeType === "company"); const observed = typeof policy?.observedAmount === "number" ? policy.observedAmount : 0; // The budget routes take whole cents; observed spend can be fractional. governance.budgetSetCents = Math.ceil(observed) + 1; await this.api.patch(`/api/companies/${this.lab.companyId}/budgets`, { budgetMonthlyCents: governance.budgetSetCents }); this.timeline.add("governance_budget_set", { taskId: this.plan.taskId, budgetCents: governance.budgetSetCents, observedCents: observed }); } catch (error) { governance.budgetSetError = error instanceof Error ? error.message : String(error); this.timeline.add("governance_budget_set_failed", { taskId: this.plan.taskId, error: governance.budgetSetError }); } } /** G1 and H1: what the board sees and does, through the same routes the board UI uses. */ private async governanceStep(sample: TaskSample, now: number): Promise { if (!sample.reachable || !this.record.issueId) return; if (this.plan.workload === "G1") await this.approvalStep(sample, now); if (this.plan.workload === "H1") await this.budgetStep(sample, now); } private async approvalStep(sample: TaskSample, now: number): Promise { const governance = this.record.governance ??= {}; if (governance.decision) return; const approvals = await this.api.get(`/api/issues/${this.record.issueId}/approvals`, 5_000).catch(() => null); const pending = (approvals ?? []).find(approval => approval.status === "pending" && approvalTitleMatches(approval, this.plan.nonce)); if (!pending) return; if (governance.approvalSeenAt == null) { governance.approvalSeenAt = now; governance.approvalId = String(pending.id); this.timeline.add("governance_approval_pending", { taskId: this.plan.taskId, approvalId: pending.id }); } if (!this.boardMayAct(sample)) return; // The gated action must not exist before the board decides. const before = await this.collectChildren(this.record.issueId!).catch(() => null); governance.childrenBeforeDecision = before === null ? null : before.filter(child => (child.title ?? "").includes(g1ChildTitle(this.plan.nonce))).length; const kind = g1Decision(this.plan.nonce); const decision: NonNullable = { kind, at: now, httpStatus: null, error: null, pendingWhenDecided: true, afterFault: (this.record.lastFaultAt ?? 0) > governance.approvalSeenAt }; try { await this.api.post(`/api/approvals/${pending.id}/${kind}`, { decisionNote: DECISION_NOTE }); // A rejected request also closes the task: the board cancels it. if (kind === "reject") await this.api.patch(`/api/issues/${this.record.issueId}`, { status: "cancelled" }); decision.httpStatus = 200; } catch (error) { decision.httpStatus = error instanceof ChaosApiError ? error.status : null; decision.error = error instanceof Error ? error.message : String(error); } governance.decision = decision; this.restCandidateSince = null; this.timeline.add("governance_approval_decided", { taskId: this.plan.taskId, approvalId: pending.id, ...decision }); } private async budgetStep(sample: TaskSample, now: number): Promise { const governance = this.record.governance ??= {}; if (governance.resume) return; const company = await this.api.get(`/api/companies/${this.lab.companyId}`, 5_000).catch(() => null); if (!company) return; const paused = company.status === "paused" && company.pauseReason === "budget"; // A pause must hold until the board resumes it: one that clears on its own (a restart lost it) // is recorded, and the board does not resume what is no longer paused. if (!paused && governance.budgetPausedAt != null) { governance.pauseLostAt ??= now; this.timeline.add("governance_budget_pause_lost", { taskId: this.plan.taskId, status: company.status ?? null, pauseReason: company.pauseReason ?? null }); return; } if (paused && governance.budgetPausedAt == null) { const recorded = typeof company.pausedAt === "string" ? Date.parse(company.pausedAt) : Number.NaN; governance.budgetPausedAt = Number.isFinite(recorded) ? Math.min(recorded, now) : now; this.timeline.add("governance_budget_paused", { taskId: this.plan.taskId, pausedAt: company.pausedAt ?? null }); } if (governance.budgetPausedAt == null || !this.boardMayAct(sample)) return; if (this.readySince === null || now - this.readySince < BUDGET_PAUSE_HOLD_MS || now - governance.budgetPausedAt < BUDGET_PAUSE_HOLD_MS) return; // While paused nothing may start: every run that started after the pause is a violation. governance.runsStartedWhilePaused = runsStartedAfter(this.record.samples, governance.budgetPausedAt); const resume: NonNullable = { at: now, amount: null, incidentId: null, httpStatus: null, error: null, afterFault: (this.record.lastFaultAt ?? 0) > governance.budgetPausedAt }; try { const overview = await this.api.get(`/api/companies/${this.lab.companyId}/budgets/overview`, 10_000); const incident = ((overview?.activeIncidents ?? []) as Json[]).find(row => row.scopeType === "company"); if (!incident) throw new Error("no active company budget incident to resolve"); const policy = ((overview?.policies ?? []) as Json[]).find(row => row.scopeType === "company"); const observed = Math.max(Number(incident.amountObserved) || 0, Number(policy?.observedAmount) || 0); resume.incidentId = String(incident.id); resume.amount = Math.ceil(observed) + BUDGET_RESUME_HEADROOM_CENTS; await this.api.post(`/api/companies/${this.lab.companyId}/budget-incidents/${incident.id}/resolve`, { action: "raise_budget_and_resume", amount: resume.amount, decisionNote: DECISION_NOTE }); resume.httpStatus = 200; } catch (error) { resume.httpStatus = error instanceof ChaosApiError ? error.status : null; resume.error = error instanceof Error ? error.message : String(error); } // Never before the poll that decided to resume (the pause window cannot end before it starts). resume.doneAt = Math.max(Date.now(), now); governance.resume = resume; this.restCandidateSince = null; this.timeline.add("governance_budget_resumed", { taskId: this.plan.taskId, ...resume, runsStartedWhilePaused: governance.runsStartedWhilePaused }); } async childSpeedTasks(): Promise }>> { if (!this.definition.collect.childDocuments || !this.record.issueId) return []; const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(this.record.issueId)}&limit=20`); const tasks: Array<{ issueId: string; adapter: AdapterId; samples: Array<{ runs?: RunSample[] }> }> = []; for (const row of (rows ?? []).filter(child => child.parentId === this.record.issueId)) { const adapter = (Object.entries(this.lab.agents) as Array<[AdapterId, { id: string }]>) .find(([, agent]) => agent.id === row.assigneeAgentId)?.[0]; if (!adapter) continue; const runs = await this.api.get(`/api/issues/${row.id}/runs`, 15_000); tasks.push({ issueId: String(row.id), adapter, samples: [{ runs: (runs ?? []).map(toRunSample) }] }); } return tasks; } async collectChildren(issueId: string): Promise> { const keys = this.definition.collect.childDocuments; if (!keys) return []; const rows = await this.api.get(`/api/companies/${this.lab.companyId}/issues?view=index&parentIds=${encodeURIComponent(issueId)}&limit=20`) .catch(() => []); const children: NonNullable = []; for (const row of (rows ?? []).filter(child => child.parentId === issueId)) { const documents: Record = {}; for (const key of keys) { const document = await this.api.get(`/api/issues/${row.id}/documents/${key}`).catch(() => null); documents[key] = typeof document?.body === "string" ? document.body : null; } children.push({ id: String(row.id), title: typeof row.title === "string" ? row.title : null, status: typeof row.status === "string" ? row.status : null, assigneeAgentId: typeof row.assigneeAgentId === "string" ? row.assigneeAgentId : null, documents }); } return children; } async collectArtifacts(): Promise { const definition = this.definition; const issueId = this.record.issueId!; const issue = await this.api.get(`/api/issues/${issueId}`).catch(() => this.issue); if (issue) this.issue = issue; const documents: Record = {}; for (const key of definition.collect.documents) { const document = await this.api.get(`/api/issues/${issueId}/documents/${key}`).catch(() => null); documents[key] = typeof document?.body === "string" ? document.body : null; } const comments = await this.api.get(`/api/issues/${issueId}/comments`).catch(() => []); const workspacePath = await this.workspacePath(); const snapshotPath = this.record.preFaultWorkspacePath ?? workspacePath; if (snapshotPath && this.record.preFaultFiles) { if (workspacePath && snapshotPath !== workspacePath) this.record.workspaceMovedTo = workspacePath; try { const present = new Set(await listWorkspaceFiles(snapshotPath)); this.record.lostFiles = this.record.preFaultFiles.filter(file => !present.has(file)); } catch (error) { this.record.restSnapshotError = error instanceof Error ? error.message : String(error); } } const files: Record = {}; const commands: OracleArtifacts["commands"] = {}; if (workspacePath) { for (const relative of definition.collect.files(this.plan.nonce)) { files[relative] = await readFile(path.join(workspacePath, relative), "utf8").catch(() => null); } // Agent-editable code: run it sandboxed, away from the operator's home directory. for (const command of definition.collect.commands) { const argv = typeof command.argv === "function" ? command.argv(this.plan.nonce) : command.argv; commands[command.name] = await runOracleCommand(argv, workspacePath, { protectedRoots: this.protectedRoots, readableFiles: argv[0] === process.execPath ? [process.execPath] : [] }); } } return { issueStatus: issue?.status ?? this.record.finalStatus, documents, agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), ...(this.record.stop ? { stop: this.stopArtifacts(this.record.stop) } : {}), ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId), childStalls: this.record.childStalls ?? [], stallWakes: await this.collectStallWakes(issueId) } : {}), ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), ...(this.record.governance ? { governance: this.record.governance } : {}), }; } /** * H1: the pause window ends when the resume request returned, after the last poll before it. Read * the task's whole run history again, so a run that started in that gap is still caught. */ private async finalPauseCheck(issueId: string): Promise> { const governance = this.record.governance; if (!governance?.budgetPausedAt) return {}; const runs = await this.api.get(`/api/issues/${issueId}/runs`).then(rows => (rows ?? []).map(toRunSample)).catch(() => []); const history: TaskSample[] = [...this.record.samples, { at: Date.now(), reachable: true, serverReady: true, runs }]; governance.runsStartedWhilePaused = runsStartedAfter(history, governance.budgetPausedAt, governance.resume?.doneAt ?? Number.POSITIVE_INFINITY); return {}; } /** G1: every approval linked to the task, whatever its status. */ private async collectApprovals(issueId: string): Promise> { const rows = await this.api.get(`/api/issues/${issueId}/approvals`).catch(() => []); return (rows ?? []).map(row => ({ id: String(row.id), status: String(row.status), title: typeof row.payload?.title === "string" ? row.payload.title : null })); } /** R6 sweep: nothing still pending once the wave is quiet. */ async sweep(): Promise { this.closeUnfiredTriggers(); if (!this.record.issueId) return { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: null }; try { const [issue, runs] = await Promise.all([ this.api.get(`/api/issues/${this.record.issueId}`), this.api.get(`/api/issues/${this.record.issueId}/runs`), ]); const samples = runs.map(toRunSample); const operations: string[] = []; for (const run of samples) { // A failed read must not look like a clean rest: let it reach sweep.error. const rows = await this.api.get(`/api/heartbeat-runs/${run.id}/workspace-operations`); for (const row of rows ?? []) if (row.status === "running") operations.push(String(row.id)); } return { activeRuns: samples.filter(run => ACTIVE_RUN_STATUSES.has(run.status)).map(run => run.id), runningWorkspaceOperations: operations, activeRecoveryAction: Boolean(issue.activeRecoveryAction), error: null }; } catch (error) { return { activeRuns: [], runningWorkspaceOperations: [], activeRecoveryAction: false, error: error instanceof Error ? error.message : String(error) }; } } }