diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 46b5d94723f..99faafee805 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1584,6 +1584,11 @@ tried again on the next poll with the same `clientRequestId`. The overlap window `staging-mix.ts`, which a swap driver registers. Without a probe, the window is any time a fault is in progress and the server still answers (a K15 overlap). The timeline records `staging_mix_follow_up`. +When no run is active once the hold or the 85% point has passed (the agent finished early), the harness posts the +follow-up after the run ended instead. The task then waits for that post before it counts as finished, and the post is +graded like an after-end follow-up: `after_end_follow_ups_served` needs an agent comment after it and the +`FOLLOWUP-` ledger line. The `follow_up_posted` detail says "after the run ended". + To aim a follow-up at the warm session's idle close instead, set `RESTART_CHAOS_MIX_FOLLOW_UP_AFTER_END_MS=-` (for example `298000-306000` for the 300 s idle timeout). The harness then posts the follow-up that long after the task's last run ended, `RESTART_CHAOS_MIX_FOLLOW_UP_COUNT` times (default 2), and waits for every post before the task @@ -1643,7 +1648,11 @@ started from a wake that carries a person's comment posted after the task's firs wake payload's `commentId`) is a required run. This includes the retry of a failed attempt for the same comment. OpenCode, for example, takes no input mid-turn, so a comment posted during its turn is answered by the next run. Such runs are listed under "Follow-ups the running turn did not receive, answered in a later run" (`followUpsAnsweredLater`, -`followUpWaits`). An extra turn is a run that started after completion from a wake that carries no comment. The +`followUpWaits`). An extra turn is a run that started after completion from a wake that carries no comment. A run +counts as having received its comments only when it started a provider turn (a `turn.submitted`, `turn.accepted` or +`turn.started` run event, `providerTurnStarted` in the wake evidence). When a run fails before any provider turn, its +comments carry to the next run that starts one: that run is the required run and may re-complete the task, and the +failed run is not an extra turn. Wake evidence without `providerTurnStarted` counts every started run as a delivery. The server's delivery rule (`task_history.read`, cited evidence, child-result reads) is covered by the server's own tests. `completion_once` stays the independent side-effect check. The first run that reported completion is always the diff --git a/tests/restart-chaos/follow-up-delivery.ts b/tests/restart-chaos/follow-up-delivery.ts index 88b78754a0a..cc4c40e6c30 100644 --- a/tests/restart-chaos/follow-up-delivery.ts +++ b/tests/restart-chaos/follow-up-delivery.ts @@ -6,6 +6,7 @@ export interface DeliveryRun { startedAt?: number | null; wakeupRequestId: string | null; contextSnapshot?: unknown; + providerTurnStarted?: boolean | null; } export interface DeliveryWake { id: string | null; createdAt?: number | null; payload?: unknown } @@ -30,23 +31,44 @@ export function wakeInputCommentIds(run: Pick, w ...strings(payload.commentId), ...strings(wakeContext.wakeCommentIds), ...strings(wakeContext.wakeCommentId)])]; } -export function laterAnswers(evidence: { runs?: readonly DeliveryRun[]; wakes?: readonly DeliveryWake[] } | null): Map { +const earliest = (a: number | null, b: number | null) => a === null ? b : b === null ? a : Math.min(a, b); + +type DeliveryEvidence = { runs?: readonly DeliveryRun[]; wakes?: readonly DeliveryWake[] } | null; + +function followUpDeliveries(evidence: DeliveryEvidence): { answers: Map; undelivered: Set } { const answers = new Map(); - if (!evidence) return answers; + const undelivered = new Set(); + if (!evidence) return { answers, undelivered }; const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); const started = (evidence.runs ?? []).filter((run): run is DeliveryRun & { runId: string; startedAt: number } => Boolean(run.runId) && typeof run.startedAt === "number") .sort((a, b) => a.startedAt - b.startedAt); const firstStart = started[0]?.startedAt ?? null; + let carried: { commentIds: string[]; postedAt: number | null } | null = null; started.forEach((run, index) => { const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; - const owed = wakeInputCommentIds(run, wake); const postedAt = typeof wake?.createdAt === "number" ? wake.createdAt : null; const postedDuringWork = postedAt !== null && firstStart !== null ? postedAt >= firstStart : index > 0; - if (index > 0 && postedDuringWork && owed.length) { - answers.set(run.runId, { runId: run.runId, commentIds: owed, postedAt, queuedAt: run.createdAt, startedAt: run.startedAt }); + const owed = index > 0 && postedDuringWork ? wakeInputCommentIds(run, wake) : []; + const commentIds = [...new Set([...(carried?.commentIds ?? []), ...owed])]; + if (!commentIds.length) return; + const firstPostedAt = owed.length ? earliest(carried?.postedAt ?? null, postedAt) : carried?.postedAt ?? null; + if (run.providerTurnStarted === false) { + carried = { commentIds, postedAt: firstPostedAt }; + undelivered.add(run.runId); + return; } + carried = null; + answers.set(run.runId, { runId: run.runId, commentIds, postedAt: firstPostedAt, queuedAt: run.createdAt, startedAt: run.startedAt }); }); - return answers; + return { answers, undelivered }; +} + +export function laterAnswers(evidence: DeliveryEvidence): Map { + return followUpDeliveries(evidence).answers; +} + +export function undeliveredFollowUpRuns(evidence: DeliveryEvidence): Set { + return followUpDeliveries(evidence).undelivered; } export const COMPLETION_INCOMPLETE_KEY = "native-completion-incomplete"; diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 999dda01c42..c1316211f36 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -84,6 +84,31 @@ describe("TaskObserver", () => { } }); + it("does not finish a staging-mix task whose run ended before its planned follow-up was posted", async () => { + let endedAt = 0; + const { api, posted } = fakeApi({ "/runs": () => [{ runId: "r1", status: "succeeded", createdAt: new Date(endedAt - 1_000).toISOString(), + finishedAt: new Date(endedAt).toISOString() }] }); + const followUpAfterMs = REST_CONFIRM_MS + 500; + const mixPlan: TaskPlan = { ...plan, workload: "SM", fault: "K0", followUpAfterMs: null, + stagingMix: { turnMinutes: 2, rounds: 2, subtasks: 0, followUpAfterMs, arriveAtMs: 0, holdMs: 0 } }; + const observer = new TaskObserver(mixPlan, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + endedAt = start; + const comments = () => posted.filter(route => route.endsWith("/comments")).length; + await observer.poll(start + 100); + await observer.poll(start + 100 + REST_CONFIRM_MS + 1); + expect(comments()).toBe(0); + expect(observer.done).toBe(false); + await observer.poll(start + followUpAfterMs + 200); + expect(comments()).toBe(1); + expect(observer.record.stagingMix?.followUp).toMatchObject({ duringRun: false }); + expect(observer.done).toBe(false); + await observer.poll(start + followUpAfterMs + 300); + await observer.poll(start + followUpAfterMs + 400 + REST_CONFIRM_MS); + expect(observer.done).toBe(true); + }); + it("keeps watching unexplained idle work until the stall limit, then records the stall", async () => { const { api } = fakeApi({}, "in_progress"); const observer = new TaskObserver({ ...plan, workload: "A1", followUpAfterMs: null }, api, lab, new Timeline(null), ready); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index ec339633307..0d10c55eb35 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -881,6 +881,7 @@ export class TaskObserver { if (isRewakeWorkload(this.plan.workload)) return awaitingRewake(this.record); const afterEnd = followUpAfterEndRange(); if (afterEnd && this.plan.stagingMix?.followUpAfterMs != null) return (this.record.stagingMix?.afterEndFollowUps?.length ?? 0) < afterEnd.count; + if (this.plan.stagingMix?.followUpAfterMs != null && !this.record.stagingMix?.followUp) return true; return false; } diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 8e2dd18b5a3..8d99ee8a8c4 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -483,3 +483,74 @@ describe("staging-mix completion from accepted results", () => { expect(stagingMixDescription(nonce, { turnMinutes: 5, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 })).toContain("Post exactly one comment"); }); }); + +describe("a follow-up whose run failed before any provider turn (w-20261114T231451-s7110 t005)", () => { + const nonce = "0c544491"; + const comment = "3f6a9eb9-29d9-47d6-a434-a222769dbc54"; + const first = "60bdd4dc-0dde-4052-9a4d-b81924aa70c1", undelivered = "76f30f70-f42f-4e8f-bef8-71e0ae1b33a3", served = "fdf1f793-95a3-4b90-a196-389ef4416c3d"; + const t005 = (undeliveredTurn: boolean): WakeEvidenceFile => ({ + completedAt: 1_794_699_341_813, + runs: [{ runId: first, createdAt: 1_794_698_171_163, startedAt: 1_794_698_171_226, invocationSource: "assignment", triggerDetail: "system", + wakeupRequestId: "035375c8", acceptedDisposition: "done", acceptedAt: 1_794_699_137_418, finishedAt: 1_794_699_138_705, status: "succeeded", providerTurnStarted: true }, + { runId: undelivered, createdAt: 1_794_699_138_879, startedAt: 1_794_699_139_744, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "7aa52728", acceptedDisposition: null, acceptedAt: null, finishedAt: 1_794_699_184_470, status: "failed", + contextSnapshot: { wakeCommentIds: [comment], wakeCommentId: comment }, providerTurnStarted: undeliveredTurn }, + { runId: served, createdAt: 1_794_699_254_459, startedAt: 1_794_699_303_349, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "fb81665c", acceptedDisposition: "done", acceptedAt: 1_794_699_340_097, finishedAt: 1_794_699_341_889, status: "succeeded", providerTurnStarted: true }], + wakes: [{ id: "035375c8", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_698_171_163 }, + { id: "7aa52728", reason: "issue_execution_promoted", source: "automation", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_698_885_238, payload: { commentId: comment, _paperclipWakeContext: { wakeCommentIds: [comment] } } }, + { id: "fb81665c", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", + requestedByActorId: "execution-recovery", createdAt: 1_794_699_254_459 }], + }); + const posted = commentEvidence([ + { body: "Ledger document now includes the FOLLOWUP line.", authorAgentId: "a", createdByRunId: served, createdAt: "2026-11-14T23:35:42.025Z" }, + { body: "Done. All 10 rounds completed.", authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-14T23:32:18.864Z" }, + { body: doneMarker(nonce), authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-14T23:32:05.368Z" }]); + const completionOnce = (comments: typeof posted, completion: CompletionEvidence) => gradeStagingMix(nonce, + { turnMinutes: 14, rounds: 4, subtasks: 0, followUpAfterMs: 497_251, arriveAtMs: 20_000, holdMs: 600_000 }, + { issueStatus: "done", documents: { ledger: `${ledger(nonce, 4)}\nFOLLOWUP-${nonce}` }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: { at: 1_794_698_884_902, commentId: comment, duringRun: true, inOverlap: true, error: null }, followUpErrors: [], retries: [], comments, + completion } }).checks.find(check => check.name === "completion_once")!; + const both = (evidence: WakeEvidenceFile) => [evidenceCompletion(evidence)!, completionEvidence({ completedAt: evidence.completedAt!, wakes: evidence.wakes, + runs: evidence.runs!.map(run => ({ ...run, acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) })]; + + it("credits the follow-up to the run that served it, with one completer and no extra turn", () => { + const evidence = t005(false); + expect(followUpsAnsweredLater("t005", evidence)).toEqual([{ taskId: "t005", runId: served, commentIds: [comment], postedAt: 1_794_698_885_238, + queuedAt: 1_794_699_254_459, startedAt: 1_794_699_303_349 }]); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(anchor).toEqual({ at: 1_794_699_137_418, source: "accepted_result" }); + expect(completionRunIds(posted, nonce, completion)).toEqual([first]); + const after = runsAfterDone("t005", evidence, anchor); + expect(after.started).toEqual([]); + expect(after.answeredFollowUp.map(turn => turn.runId)).toEqual([served]); + expect(completionOnce(posted, completion)).toMatchObject({ ok: true, detail: "runs=1" }); + } + }); + + it("still fails a recovery re-completion when the run that received the follow-up started its provider turn", () => { + const evidence = t005(true); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t005", evidence, anchor).started.map(turn => [turn.runId, turn.wakeReason])).toEqual([[served, "issue_recovery_action_restored"]]); + expect(completionOnce(posted, completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 60bdd4dc, fdf1f793 each reported completion" }); + } + }); + + it("lets only the serving run re-complete, and still fails a later recovery run that answers nothing", () => { + const extra = "8e8e8e8e-extra"; + const evidence = t005(false); + evidence.runs!.push({ runId: extra, createdAt: 1_794_699_400_000, startedAt: 1_794_699_400_100, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "w-extra", acceptedDisposition: "done", acceptedAt: 1_794_699_410_000, status: "succeeded", providerTurnStarted: true }); + evidence.wakes!.push({ id: "w-extra", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", + requestedByActorId: "execution-recovery", createdAt: 1_794_699_400_000 }); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t005", evidence, anchor).started.map(turn => turn.runId)).toEqual([extra]); + expect(completionOnce(posted, completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 60bdd4dc, 8e8e8e8e each reported completion" }); + } + }); +}); diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 8d286197c19..89c4b329a6f 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -6,7 +6,7 @@ import { bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence, } from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; -import { laterAnswers, laterAnswerWaits, rejectedDoneRuns, type LaterAnswer } from "./follow-up-delivery.js"; +import { laterAnswers, laterAnswerWaits, rejectedDoneRuns, undeliveredFollowUpRuns, type LaterAnswer } from "./follow-up-delivery.js"; type Json = Record; @@ -192,7 +192,8 @@ export interface ExtraTurn { export interface WakeEvidenceFile { completedAt?: number | null; runs?: Array<{ runId: string | null; createdAt: number | null; startedAt?: number | null; invocationSource: string | null; triggerDetail: string | null; - wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null }>; + wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null; + providerTurnStarted?: boolean | null }>; wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null; createdAt?: number | null; payload?: unknown }>; } @@ -232,9 +233,10 @@ export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, }; const runs = evidence.runs ?? []; const afterDone = runs.filter(run => run.startedAt != null && run.startedAt > doneAt); + const undelivered = undeliveredFollowUpRuns(evidence); const answered = (run: (typeof runs)[number]) => Boolean(run.runId && answers.has(run.runId)); return { - started: afterDone.filter(run => !answered(run)).map(turn), + started: afterDone.filter(run => !answered(run) && !(run.runId && undelivered.has(run.runId))).map(turn), neverStarted: runs.filter(run => run.startedAt == null && run.createdAt !== null && run.createdAt > doneAt).map(turn), answeredFollowUp: afterDone.filter(answered).map(turn), }; diff --git a/tests/restart-chaos/staging-mix.test.ts b/tests/restart-chaos/staging-mix.test.ts index c6c82ef5cbc..81f7d9cba1f 100644 --- a/tests/restart-chaos/staging-mix.test.ts +++ b/tests/restart-chaos/staging-mix.test.ts @@ -296,6 +296,46 @@ describe("staging-mix board step", () => { expect(late.stagingMix!.followUp!.inOverlap).toBe(false); }); + it("posts a follow-up the run finished before as an after-end follow-up the product must answer (w-20261114T231451-s7110 t009)", async () => { + const createdAt = 1_794_698_190_262, finishedAt = 1_794_698_875_995, latestAt = createdAt + Math.round(10 * ROUND_SECONDS * 1_000 * 0.85); + const plan = task({ turnMinutes: 15, rounds: 10, followUpAfterMs: 529_234, arriveAtMs: 40_000, holdMs: 600_000 }); + const live: TaskSample = { ...running, runs: [{ ...running.runs![0]!, id: "6dfb753b", createdAt: new Date(createdAt + 26).toISOString() }] }; + const ended: TaskSample = { ...live, issueStatus: "done", runs: [{ ...live.runs![0]!, status: "succeeded", finishedAt: new Date(finishedAt).toISOString() }] }; + const rec = record({ createdAt }); + expect((await stepIn(rec, live, createdAt + 529_234 + 10_000, false, plan)).posted).toEqual([]); + expect((await stepIn(rec, live, finishedAt - 1_000, false, plan)).posted).toEqual([]); + expect((await stepIn(rec, ended, 1_794_698_882_700, true, plan)).posted).toEqual([]); + const after = await stepIn(rec, ended, latestAt + 1_000, false, plan); + expect(after.posted.length).toBe(1); + expect(after.posted[0]).toContain(followUpMarker("n1")); + expect(rec.stagingMix!.followUp).toMatchObject({ commentId: "c1", duringRun: false, at: latestAt + 1_000 }); + expect(rec.stagingMix!.afterEndFollowUps).toEqual([{ afterRunEndAt: finishedAt, at: latestAt + 1_000, delayMs: latestAt + 1_000 - finishedAt, commentId: "c1" }]); + expect((await stepIn(rec, ended, latestAt + 5_000, false, plan)).posted).toEqual([]); + + const nonce = "15b397b0"; + const grade = (comments: Array<{ body: string; runId: string | null; createdAt: number }>, extra: string[]) => gradeStagingMix(nonce, plan, artifacts({ + documents: { ledger: ledgerFor(nonce, 10, extra) }, agentComments: comments.map(comment => comment.body), + stagingMix: { followUp: rec.stagingMix!.followUp!, followUpErrors: [], retries: [], comments, afterEndFollowUps: rec.stagingMix!.afterEndFollowUps } })).checks; + const done = { body: doneMarker(nonce), runId: "6dfb753b", createdAt: finishedAt - 1_000 }; + const unanswered = grade([done], []); + expect(unanswered.find(check => check.name === "follow_up_posted")).toMatchObject({ ok: true, detail: "comment c1 after the run ended" }); + expect(unanswered.find(check => check.name === "after_end_follow_ups_served")!.ok).toBe(false); + expect(unanswered.some(check => check.name === "follow_up_served")).toBe(false); + const answered = grade([done, { body: "Added the FOLLOWUP line.", runId: "r2", createdAt: latestAt + 60_000 }], [followUpMarker(nonce)]); + expect(answered.filter(check => !check.ok)).toEqual([]); + }); + + it("does not post an after-end follow-up between runs before the during-run window closes, or while a run is queued", async () => { + const plan = task({ followUpAfterMs: 60_000 }); + const gap: TaskSample = { ...running, issueStatus: "in_progress", runs: [{ ...running.runs![0]!, status: "failed", finishedAt: new Date(50_000).toISOString() }] }; + const queued: TaskSample = { ...gap, runs: [...gap.runs!, { ...running.runs![0]!, id: "r2", status: "queued" }] }; + const rec = record(); + expect((await stepIn(rec, gap, 61_000, true, plan)).posted).toEqual([]); + expect((await stepIn(rec, queued, 60_000 + 600_000, false, plan)).posted).toEqual([]); + expect((await stepIn(rec, gap, 60_000 + 600_000, false, plan)).posted.length).toBe(1); + expect(rec.stagingMix!.followUp).toMatchObject({ duringRun: false }); + }); + it("tries a failed follow-up again on the next poll and keeps every error for the grade", async () => { const rec = record(); const failing = await step(rec, running, 80_000, true, undefined, () => { throw new Error("HTTP 502"); }); diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index fbadcbc9c46..287309d54a5 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -495,7 +495,7 @@ export function gradeStagingMix(nonce: string, task: StagingMixTask, artifacts: if (task.followUpAfterMs !== null) { const errors = artifacts.stagingMix?.followUpErrors ?? []; checks.push({ name: "follow_up_posted", ok: Boolean(followUp?.commentId), - detail: followUp?.commentId ? `comment ${followUp.commentId}` : `never posted${errors.length ? `; errors: ${errors.slice(-3).join(" | ")}` : ""}` }); + detail: followUp?.commentId ? `comment ${followUp.commentId}${followUp.duringRun ? "" : " after the run ended"}` : `never posted${errors.length ? `; errors: ${errors.slice(-3).join(" | ")}` : ""}` }); } const afterEnd = artifacts.stagingMix?.afterEndFollowUps ?? []; const range = task.followUpAfterMs !== null ? followUpAfterEndRange() : null; @@ -627,11 +627,14 @@ export async function stagingMixStep(input: StagingMixStepInput): Promise if (commentId && !state.followUp) state.followUp = { at: now, commentId, duringRun: false, inOverlap: overlap, error: null }; } } - } else if (task.followUpAfterMs !== null && !state.followUp && running) { + } else if (task.followUpAfterMs !== null && !state.followUp) { const plannedAt = record.createdAt + task.followUpAfterMs; const latestAt = Math.min(plannedAt + task.holdMs, record.createdAt + Math.round(task.rounds * ROUND_SECONDS * 1_000 * 0.85)); - if (now >= plannedAt && (overlap || now >= latestAt)) { - const followUp: NonNullable = { at: now, commentId: null, duringRun: true, inOverlap: overlap, error: null }; + const runs = sample.runs ?? []; + const lastEnd = Math.max(0, ...runs.map(run => Date.parse(run.finishedAt ?? "") || 0)); + const afterRunEnd = !running && !runs.some(run => ACTIVE.has(run.status)) && lastEnd > 0 && now >= Math.max(plannedAt, latestAt); + if (running ? now >= plannedAt && (overlap || now >= latestAt) : afterRunEnd) { + const followUp: NonNullable = { at: now, commentId: null, duringRun: running, inOverlap: overlap, error: null }; try { const comment = await input.api.post<{ id?: string }>(`/api/issues/${record.issueId}/comments`, { body: followUpComment(input.nonce), clientRequestId: followUpRequestId(record.issueId, input.nonce) }); @@ -640,9 +643,11 @@ export async function stagingMixStep(input: StagingMixStepInput): Promise } catch (error) { followUp.error = error instanceof Error ? error.message : String(error); } - if (followUp.commentId) state.followUp = followUp; - else (state.followUpErrors ??= []).push(followUp.error!); - input.note("staging_mix_follow_up", { ...followUp }); + if (followUp.commentId) { + state.followUp = followUp; + if (afterRunEnd) (state.afterEndFollowUps ??= []).push({ afterRunEndAt: lastEnd, at: now, delayMs: now - lastEnd, commentId: followUp.commentId }); + } else (state.followUpErrors ??= []).push(followUp.error!); + input.note("staging_mix_follow_up", { ...followUp, ...(afterRunEnd ? { afterRunEndAt: lastEnd } : {}) }); } } noteRetryBanner(state, retryShown(sample), now, latestFailureCode(sample)); diff --git a/tests/restart-chaos/wake-evidence.test.ts b/tests/restart-chaos/wake-evidence.test.ts index 438a0836b55..686c7d2e799 100644 --- a/tests/restart-chaos/wake-evidence.test.ts +++ b/tests/restart-chaos/wake-evidence.test.ts @@ -3,7 +3,7 @@ import { mkdtemp, readFile, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { activityLog, agentWakeupRequests, agents, companies, controllerInstances, createDb, heartbeatRuns, issues, type Db } from "../../packages/db/src/index.js"; +import { activityLog, agentWakeupRequests, agents, companies, controllerInstances, createDb, heartbeatRunEvents, heartbeatRuns, issues, type Db } from "../../packages/db/src/index.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../server/src/__tests__/helpers/embedded-postgres.js"; import { boundByProcessExit, captureWakeEvidence, servingAt, unshiftRows, wakeEvidenceForIssue, wakeEvidenceOnce, type ControllerRow } from "./wake-evidence.js"; import { runEvents } from "./evidence.js"; @@ -149,7 +149,7 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () expect(written).toBe(1); const evidence = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "t01.json"), "utf8")); expect(evidence.runs[0]).toMatchObject({ runId, invocationSource: "on_demand", wakeupRequestId: wakeId, contextSnapshot: { wakeReason: "retry" }, - owner: { pid: 4242, processSpace: "space-wake" } }); + owner: { pid: 4242, processSpace: "space-wake" }, providerTurnStarted: false }); const slots = JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "agent-slots.json"), "utf8")); expect(slots.agents).toEqual(expect.arrayContaining([{ agentId, maxConcurrentRuns: 3 }, { agentId: wideAgentId, maxConcurrentRuns: 50 }])); expect(slots.runs).toEqual([{ runId, agentId, startedAt: 1_800_000_000_000, finishedAt: 1_800_000_060_000 }]); @@ -158,8 +158,9 @@ describe.skipIf(!support.supported)("wake evidence against real PostgreSQL", () expect(evidence.wakes[0].servingAtCreate.map((server: { pid: number }) => server.pid)).toEqual([4242]); expect(evidence.statusChanges).toMatchObject([{ action: "issue.updated", from: "in_progress", to: "done" }]); expect(JSON.parse(await readFile(path.join(waveDir, "wake-evidence", "servers.json"), "utf8")).controllers[0]).toMatchObject({ pid: 4242 }); + await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 1, eventType: "turn.started" }); expect(await captureWakeEvidence({ db, companyId, waveDir, observers: null, windows: [], redact: text => text })).toBe(1); - expect(JSON.parse(await readFile(path.join(waveDir, "wake-evidence", `issue-${issueId}.json`), "utf8")).runs[0]).toMatchObject({ runId }); + expect(JSON.parse(await readFile(path.join(waveDir, "wake-evidence", `issue-${issueId}.json`), "utf8")).runs[0]).toMatchObject({ runId, providerTurnStarted: true }); } finally { await rm(waveDir, { recursive: true, force: true }); } diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts index 0d8793c36e6..e1634941ec3 100644 --- a/tests/restart-chaos/wake-evidence.ts +++ b/tests/restart-chaos/wake-evidence.ts @@ -79,7 +79,8 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) const owner = controllerByBoot.get(String(run.controllerBootId ?? "")); return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail), wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), - acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, + acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, + providerTurnStarted: run.providerTurnStarted == null ? null : Boolean(run.providerTurnStarted), owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) }; }), wakes: wakes.map(wake => ({ id: text(wake.id), reason: text(wake.reason), source: text(wake.source), triggerDetail: text(wake.triggerDetail), @@ -99,7 +100,9 @@ async function readRows(db: Pick, companyId: string, s r.wakeup_request_id::text as "wakeupRequestId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId", floor(extract(epoch from r.created_at) * 1000) as "createdAt", floor(extract(epoch from r.started_at) * 1000) as "startedAt", floor(extract(epoch from r.finished_at) * 1000) as "finishedAt", accepted.disposition as "acceptedDisposition", - floor(extract(epoch from accepted.accepted_at) * 1000) as "acceptedAt" + floor(extract(epoch from accepted.accepted_at) * 1000) as "acceptedAt", + exists (select 1 from heartbeat_run_events t where t.company_id = r.company_id and t.run_id = r.id + and t.event_type in ('turn.submitted', 'turn.accepted', 'turn.started')) as "providerTurnStarted" from heartbeat_runs r left join lateral (select coalesce(e.payload->'prpEvent'->'payload'->'result'->>'reportedWorkDisposition', e.payload->'result'->>'reportedWorkDisposition') as disposition, e.created_at as accepted_at from heartbeat_run_events e where e.run_id = r.id and e.event_type = 'run.result.accepted'