diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 16a8f0e517d..46b5d94723f 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -1647,11 +1647,15 @@ listed under "Follow-ups the running turn did not receive, answered in a later r server's delivery rule (`task_history.read`, cited evidence, child-result reads) is covered by the server's own tests. `completion_once` stays the independent side-effect check. The first run that reported completion is always the -baseline. A later completion counts as a duplicate unless its run answered a comment as above. +baseline. A later completion counts as a duplicate unless its run answered a comment as above. A `done` result that the +server rejected as incomplete is not a completion: the server shows the rejection by queueing a same-agent continuation +wake with key `native-completion-incomplete` for the issue, and the latest accepted result before that wake is the +`done`. A wake that follows a later result, such as a `yielded` one, does not reject an earlier `done`. Such a run counts only when it posted the +DONE comment itself. Otherwise the continuation run that finishes the work is the baseline. The completion time comes from the product's own record, in this order: -1. the earliest accepted run result whose `reportedWorkDisposition` is `done`; +1. the earliest accepted run result whose `reportedWorkDisposition` is `done`, skipping a rejected `done` as above; 2. the issue's `completedAt`; 3. the first bare `DONE-` comment, only when neither exists. diff --git a/tests/restart-chaos/follow-up-delivery.ts b/tests/restart-chaos/follow-up-delivery.ts index 0e907b116ec..88b78754a0a 100644 --- a/tests/restart-chaos/follow-up-delivery.ts +++ b/tests/restart-chaos/follow-up-delivery.ts @@ -49,6 +49,22 @@ export function laterAnswers(evidence: { runs?: readonly DeliveryRun[]; wakes?: return answers; } +export const COMPLETION_INCOMPLETE_KEY = "native-completion-incomplete"; + +export function rejectedDoneRuns(evidence: { runs?: ReadonlyArray<{ runId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null }>; + wakes?: readonly DeliveryWake[] } | null): Set { + const rejected = new Set(); + const results = (evidence?.runs ?? []).flatMap(run => run.runId && run.acceptedDisposition && typeof run.acceptedAt === "number" + ? [{ runId: run.runId, disposition: run.acceptedDisposition, acceptedAt: run.acceptedAt }] : []); + for (const wake of evidence?.wakes ?? []) { + const at = wake.createdAt; + if (typeof at !== "number" || object(wake.payload).continuationIdempotencyKey !== COMPLETION_INCOMPLETE_KEY) continue; + const cause = results.filter(run => run.acceptedAt <= at).sort((a, b) => b.acceptedAt - a.acceptedAt)[0]; + if (cause?.disposition === "done") rejected.add(cause.runId); + } + return rejected; +} + function percentile(values: readonly number[], fraction: number): number | null { if (!values.length) return null; const sorted = [...values].sort((a, b) => a - b); diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 136314815c2..8e2dd18b5a3 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -9,6 +9,7 @@ import { retryAfterSettleStage, runsAfterDone, shouldClickRetryAfterSettle, stagingMixHeadline, summarizeStagingMix, type RetryStageObserver, type WakeEvidenceFile, } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; +import { completionEvidence } from "./wake-evidence.js"; const wave = (argv: string[]) => { const command = parseCommand(argv); @@ -264,6 +265,93 @@ describe("staging-mix runs after the task completed", () => { }); }); +describe("a done the server rejected as incomplete", () => { + const nonce = "8c82aea3"; + const rejected = "d80d3ab3-e4ea-4959-b65b-329632dfd684", finisher = "a777038c-095d-438d-96fc-e341efd3d23d"; + const incomplete = { continuationKind: "same_agent", continuationIdempotencyKey: "native-completion-incomplete" }; + const t005 = (): WakeEvidenceFile => ({ + completedAt: 1_794_646_221_835, + runs: [{ runId: rejected, createdAt: 1_794_644_311_531, startedAt: 1_794_644_311_621, invocationSource: "assignment", triggerDetail: "system", + wakeupRequestId: "be501b49", acceptedDisposition: "done", acceptedAt: 1_794_646_168_026 }, + { runId: finisher, createdAt: 1_794_646_171_317, startedAt: 1_794_646_171_426, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "68fde0a3", acceptedDisposition: "done", acceptedAt: 1_794_646_218_506 }], + wakes: [{ id: "be501b49", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_644_311_531 }, + { id: "3f3716f3", reason: "issue_status_changed", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: 1_794_646_171_247, payload: incomplete }, + { id: "68fde0a3", reason: "issue_status_changed", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: 1_794_646_171_317, payload: incomplete }], + }); + const posted = commentEvidence([ + { body: "All 17 rounds are complete and verified.", authorAgentId: "a", createdByRunId: finisher, createdAt: "2026-11-14T08:50:22.890Z" }, + { body: doneMarker(nonce), authorAgentId: "a", createdByRunId: finisher, createdAt: "2026-11-14T08:49:58.554Z" }, + { body: "Completed and verified all 17 rounds exactly once in `ledger`.", authorAgentId: "a", createdByRunId: rejected, createdAt: "2026-11-14T08:49:32.345Z" }]); + const completionOnce = (comments: typeof posted, completion: CompletionEvidence) => gradeStagingMix(nonce, + { turnMinutes: 15, rounds: 4, subtasks: 0, followUpAfterMs: null, arriveAtMs: 0, holdMs: 0 }, + { issueStatus: "done", documents: { ledger: ledger(nonce, 4) }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: null, followUpErrors: [], retries: [], comments, completion } }).checks.find(check => check.name === "completion_once")!; + + it("anchors on the run that finished the work, with one completer and no extra turn (w-20261114T081656-s7105 t005)", () => { + const evidence = t005(); + for (const completion of [evidenceCompletion(evidence)!, completionEvidence({ completedAt: evidence.completedAt!, wakes: evidence.wakes, runs: evidence.runs!.map(run => + ({ ...run, acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) })]) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(anchor).toEqual({ at: 1_794_646_218_506, source: "accepted_result" }); + expect(completionRunIds(posted, nonce, completion)).toEqual([finisher]); + expect(runsAfterDone("t005", evidence, anchor).started).toEqual([]); + expect(completionOnce(posted, completion)).toMatchObject({ ok: true, detail: "runs=1" }); + } + }); + + it("still counts a rejected run that posted the DONE comment itself", () => { + const evidence = t005(); + const comments = [...posted, { body: doneMarker(nonce), runId: rejected, createdAt: 1_794_646_160_000 }]; + expect(completionRunIds(comments, nonce, evidenceCompletion(evidence))).toEqual([rejected, finisher]); + }); + + it("keeps a run woken by recovery after a real completion as an extra turn and a duplicate completion (G5c t004)", () => { + const done = "4a4a4a4a-done", woken = "5b5b5b5b-woken"; + const evidence: WakeEvidenceFile = { + completedAt: 10_000, + runs: [{ runId: done, createdAt: 1_000, startedAt: 1_100, invocationSource: "automation", triggerDetail: null, wakeupRequestId: "w1", acceptedDisposition: "done", acceptedAt: 9_900 }, + { runId: woken, createdAt: 20_000, startedAt: 20_100, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w2", acceptedDisposition: "done", acceptedAt: 21_000 }], + wakes: [{ id: "w1", reason: "issue_status_changed", source: "automation", triggerDetail: null, requestedByActorType: "system", requestedByActorId: null, createdAt: 1_000, + payload: incomplete }, + { id: "w2", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: "sweep", + createdAt: 20_000 }], + }; + const comments = [{ body: doneMarker(nonce), runId: done, createdAt: 9_800 }]; + const completion = evidenceCompletion(evidence)!; + const anchor = completedAtAnchor(completion, comments, nonce); + expect(anchor).toEqual({ at: 9_900, source: "accepted_result" }); + expect(runsAfterDone("t004", evidence, anchor).started.map(turn => [turn.runId, turn.wakeReason])).toEqual([[woken, "issue_recovery_action_restored"]]); + expect(completionOnce(comments, completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 4a4a4a4a, 5b5b5b5b each reported completion" }); + }); + + it("keeps a real done when a later recovery run yields with an incomplete wake, and still fails the extra run", () => { + const done = "4a4a4a4a-done", yielded = "6c6c6c6c-yield", extra = "7d7d7d7d-extra"; + const evidence: WakeEvidenceFile = { + completedAt: 10_000, + runs: [{ runId: done, createdAt: 1_000, startedAt: 1_100, invocationSource: "assignment", triggerDetail: null, wakeupRequestId: "w1", acceptedDisposition: "done", acceptedAt: 9_900 }, + { runId: yielded, createdAt: 20_000, startedAt: 20_100, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w2", + acceptedDisposition: "yielded", acceptedAt: 21_000 }, + { runId: extra, createdAt: 21_100, startedAt: 21_200, invocationSource: "automation", triggerDetail: "system", wakeupRequestId: "w3", + acceptedDisposition: "done", acceptedAt: 22_000 }], + wakes: [{ id: "w1", reason: "issue_assigned", source: "assignment", triggerDetail: null, requestedByActorType: "user", requestedByActorId: "u1", createdAt: 1_000 }, + { id: "w2", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: "sweep", + createdAt: 20_000 }, + { id: "w3", reason: "issue_status_changed", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: 21_050, payload: incomplete }], + }; + const comments: typeof posted = []; + const completion = evidenceCompletion(evidence)!; + const anchor = completedAtAnchor(completion, comments, nonce); + expect(anchor).toEqual({ at: 9_900, source: "accepted_result" }); + expect(runsAfterDone("t004", evidence, anchor).started.map(turn => turn.runId)).toEqual([yielded, extra]); + expect(completionOnce(comments, completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 4a4a4a4a, 7d7d7d7d each reported completion" }); + }); +}); + describe("a follow-up the running turn did not receive (w-20261113T052703-s7301 t010)", () => { const nonce = "bd13d3bf"; const comment = "fd278a3a-946b-44df-b62f-0e47c982aea1"; diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 1b52d73dd5f..8d286197c19 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -6,7 +6,7 @@ import { bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence, } from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; -import { laterAnswers, laterAnswerWaits, type LaterAnswer } from "./follow-up-delivery.js"; +import { laterAnswers, laterAnswerWaits, rejectedDoneRuns, type LaterAnswer } from "./follow-up-delivery.js"; type Json = Record; @@ -212,8 +212,10 @@ export function evidenceCompletion(evidence: WakeEvidenceFile | null, issueCompl const fromIssue = typeof issueCompletedAt === "string" && Number.isFinite(Date.parse(issueCompletedAt)) ? Date.parse(issueCompletedAt) : null; if (!evidence && fromIssue === null) return null; const answers = laterAnswers(evidence); + const rejected = rejectedDoneRuns(evidence); return { completedAt: evidence?.completedAt ?? fromIssue, runs: (evidence?.runs ?? []).map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, - acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) }; + acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null, answersFollowUp: Boolean(run.runId && answers.has(run.runId)), + doneRejected: Boolean(run.runId && rejected.has(run.runId)) })) }; } export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, anchor: { at: number | null; source: string | null }, doneCommentAt: number | null = null): diff --git a/tests/restart-chaos/staging-mix.ts b/tests/restart-chaos/staging-mix.ts index 649e9667fb0..fbadcbc9c46 100644 --- a/tests/restart-chaos/staging-mix.ts +++ b/tests/restart-chaos/staging-mix.ts @@ -61,7 +61,8 @@ export interface StagingMixRecord { export interface CompletionEvidence { completedAt: number | null; - runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null; answersFollowUp?: boolean }>; + runs: Array<{ runId: string | null; startedAt: number | null; acceptedDisposition: string | null; acceptedAt: number | null; answersFollowUp?: boolean; + doneRejected?: boolean }>; } export interface RetryBanner { firstShownAt: number; lastShownAt: number; shownMs: number; visibleSince: number | null; errorCode: string | null } @@ -398,9 +399,14 @@ export function hasDispositions(completion: CompletionEvidence | null | undefine return Boolean(completion?.runs.some(run => run.acceptedDisposition !== null)); } +function countsAsDone(run: CompletionEvidence["runs"][number] | undefined, comments: readonly CommentEvidence[], nonce: string): run is CompletionEvidence["runs"][number] { + return run?.acceptedDisposition === "done" + && (!run.doneRejected || comments.some(comment => comment.runId === run.runId && isDoneComment(comment.body, nonce))); +} + export function completedAtAnchor(completion: CompletionEvidence | null | undefined, comments: readonly CommentEvidence[], nonce: string): { at: number | null; source: "accepted_result" | "completed_at" | "done_comment" | null } { - const accepted = (completion?.runs ?? []).filter(run => run.acceptedDisposition === "done" && run.acceptedAt !== null).map(run => run.acceptedAt!); + const accepted = (completion?.runs ?? []).filter(run => countsAsDone(run, comments, nonce) && run.acceptedAt !== null).map(run => run.acceptedAt!); if (accepted.length) return { at: Math.min(...accepted), source: "accepted_result" }; if (completion?.completedAt != null) return { at: completion.completedAt, source: "completed_at" }; const comment = firstDoneCommentAt(comments, nonce); @@ -415,11 +421,11 @@ export function completionRunIds(comments: readonly CommentEvidence[], nonce: st const byId = new Map(completion.runs.flatMap(run => run.runId ? [[run.runId, run] as const] : [])); const reportedAt = (runId: string) => { const run = byId.get(runId); - if (run?.acceptedDisposition === "done" && run.acceptedAt !== null) return run.acceptedAt; + if (countsAsDone(run, comments, nonce) && run.acceptedAt !== null) return run.acceptedAt; const posted = comments.filter(comment => comment.runId === runId && typeof comment.createdAt === "number").map(comment => comment.createdAt!); return posted.length ? Math.min(...posted) : run?.startedAt ?? Number.POSITIVE_INFINITY; }; - const accepted = completion.runs.filter(run => run.acceptedDisposition === "done" && run.runId).map(run => run.runId!); + const accepted = completion.runs.filter(run => countsAsDone(run, comments, nonce) && run.runId).map(run => run.runId!); const ordered = [...new Set([...accepted, ...summaries])].sort((a, b) => reportedAt(a) - reportedAt(b)); return ordered.filter((runId, index) => index === 0 || !byId.get(runId)?.answersFollowUp); } diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts index 4cb126718a1..0d8793c36e6 100644 --- a/tests/restart-chaos/wake-evidence.ts +++ b/tests/restart-chaos/wake-evidence.ts @@ -4,7 +4,7 @@ import type { ServerProcessWindow } from "./multi-server.js"; import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js"; import { ISSUE_STATUSES } from "../../packages/shared/src/constants.js"; import type { CompletionEvidence } from "./staging-mix.js"; -import { laterAnswers, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js"; +import { laterAnswers, rejectedDoneRuns, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js"; import { AGENT_SLOTS_FILE, agentSlotsFromRows } from "./agent-slots.js"; import { agents, createDb, type Db } from "../../packages/db/src/index.js"; import { maxConcurrentRunsSql } from "../../server/src/services/heartbeat/timer-policy.js"; @@ -168,8 +168,10 @@ export interface EvidenceObserver { export function completionEvidence(evidence: { completedAt: number | null; wakes?: readonly DeliveryWake[]; runs: ReadonlyArray }): CompletionEvidence { const answers = laterAnswers(evidence); + const rejected = rejectedDoneRuns(evidence); return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, - acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt, answersFollowUp: Boolean(run.runId && answers.has(run.runId)) })) }; + acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt, answersFollowUp: Boolean(run.runId && answers.has(run.runId)), + doneRejected: Boolean(run.runId && rejected.has(run.runId)) })) }; } export function wakeEvidenceOnce(input: { databaseUrl: () => string; companyId: string; waveDir: string;