diff --git a/tests/restart-chaos/follow-up-delivery.ts b/tests/restart-chaos/follow-up-delivery.ts index cc4c40e6c30..0760f1471e3 100644 --- a/tests/restart-chaos/follow-up-delivery.ts +++ b/tests/restart-chaos/follow-up-delivery.ts @@ -6,6 +6,7 @@ export interface DeliveryRun { startedAt?: number | null; wakeupRequestId: string | null; contextSnapshot?: unknown; + retryOfRunId?: string | null; providerTurnStarted?: boolean | null; } @@ -71,6 +72,44 @@ export function undeliveredFollowUpRuns(evidence: DeliveryEvidence): Set return followUpDeliveries(evidence).undelivered; } +export function continuedRunIds(run: Pick, wake: DeliveryWake | undefined): string[] { + const context = object(run.contextSnapshot); + const payload = object(wake?.payload); + return [...new Set([run.retryOfRunId, context.retryOfRunId, context.previousRunId, object(context.executionContinuation).interruptedRunId, + object(payload._paperclipWakeContext).retryOfRunId, payload.retryOfRunId] + .filter((value): value is string => typeof value === "string" && value.length > 0))]; +} + +export function committedCompletion(run: { runId: string | null; acceptedDisposition?: string | null } | undefined, rejected: ReadonlySet): boolean { + return run?.acceptedDisposition === "done" && !(run.runId && rejected.has(run.runId)); +} + +export function followUpAnsweringRuns(evidence: { runs?: ReadonlyArray; + wakes?: readonly DeliveryWake[] } | null, rejected: ReadonlySet): Set { + const answering = new Set(laterAnswers(evidence).keys()); + if (!evidence) return answering; + const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); + const runs = (evidence.runs ?? []).filter((run): run is typeof run & { runId: string } => Boolean(run.runId)); + const byId = new Map(runs.map(run => [run.runId, run])); + const lineage = new Map(runs.map(run => [run.runId, continuedRunIds(run, run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined)])); + const committedAncestor = (runId: string): boolean => { + const visited = new Set([runId]); + const pending = [...(lineage.get(runId) ?? [])]; + while (pending.length) { + const id = pending.pop()!; + if (visited.has(id)) continue; + visited.add(id); + if (committedCompletion(byId.get(id), rejected)) return true; + pending.push(...(lineage.get(id) ?? [])); + } + return false; + }; + for (const run of [...runs].sort((a, b) => (a.createdAt ?? 0) - (b.createdAt ?? 0))) { + if ((lineage.get(run.runId) ?? []).some(id => answering.has(id)) && !committedAncestor(run.runId)) answering.add(run.runId); + } + return answering; +} + export const COMPLETION_INCOMPLETE_KEY = "native-completion-incomplete"; export function rejectedDoneRuns(evidence: { runs?: ReadonlyArray<{ runId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null }>; diff --git a/tests/restart-chaos/staging-mix-retry.test.ts b/tests/restart-chaos/staging-mix-retry.test.ts index 8d99ee8a8c4..903a213f094 100644 --- a/tests/restart-chaos/staging-mix-retry.test.ts +++ b/tests/restart-chaos/staging-mix-retry.test.ts @@ -10,6 +10,7 @@ import { } from "./staging-mix-retry.js"; import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample } from "./types.js"; import { completionEvidence } from "./wake-evidence.js"; +import { COMPLETION_INCOMPLETE_KEY } from "./follow-up-delivery.js"; const wave = (argv: string[]) => { const command = parseCommand(argv); @@ -554,3 +555,147 @@ describe("a follow-up whose run failed before any provider turn (w-20261114T2314 } }); }); + +describe("a recovery continuation of a follow-up run that never committed (w-20261117T170216-s7100 t128)", () => { + const nonce = "ed987e41"; + const comment = "956f5926-1621-4868-9d03-ca9a307b36e9"; + const first = "38a8772f-33f6-4291-bdd6-1f16494aa6ed", promoted = "5707e93a-05dc-4b7f-a7d3-ec5c25a17367", leaseRetry = "2c1d364e-6c46-423e-84f3-8a80e89a7cb5"; + const answering = "c5f9a5a4-bf76-42d3-a26d-771fe3a3d176", continuation = "833314ed-bd6c-4e41-9992-78629d065816"; + const t128 = (answeringDisposition: string | null = null): WakeEvidenceFile => ({ + completedAt: 1_794_946_774_445, + runs: [{ runId: first, createdAt: 1_794_941_952_144, startedAt: 1_794_942_615_951, invocationSource: "assignment", triggerDetail: "system", + wakeupRequestId: "55a6afa9", acceptedDisposition: "done", acceptedAt: 1_794_943_908_538, finishedAt: 1_794_943_909_767, status: "succeeded", providerTurnStarted: true, + contextSnapshot: { wakeReason: "issue_assigned" } }, + { runId: promoted, createdAt: 1_794_943_909_966, startedAt: 1_794_944_484_183, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "be67ffb7", acceptedDisposition: null, acceptedAt: null, finishedAt: 1_794_944_484_330, status: "failed", providerTurnStarted: false, + contextSnapshot: { wakeReason: "issue_commented", wakeCommentIds: [comment], wakeCommentId: comment } }, + { runId: leaseRetry, createdAt: 1_794_944_484_384, startedAt: 1_794_944_511_264, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "549cf01c", acceptedDisposition: null, acceptedAt: null, finishedAt: 1_794_944_511_437, status: "failed", providerTurnStarted: false, + contextSnapshot: { wakeReason: "transient_failure_retry", wakeCommentIds: [comment], wakeCommentId: comment, retryOfRunId: promoted } }, + { runId: answering, createdAt: 1_794_944_511_476, startedAt: 1_794_944_580_608, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "08e16978", acceptedDisposition: answeringDisposition, acceptedAt: answeringDisposition ? 1_794_945_390_000 : null, finishedAt: 1_794_945_394_924, + status: answeringDisposition ? "succeeded" : "failed", providerTurnStarted: true, + contextSnapshot: { wakeReason: "transient_failure_retry", wakeCommentIds: [comment], wakeCommentId: comment, retryOfRunId: leaseRetry } }, + { runId: continuation, createdAt: 1_794_945_559_524, startedAt: 1_794_946_114_761, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "88d1a3e0", acceptedDisposition: "done", acceptedAt: 1_794_946_772_351, finishedAt: 1_794_946_774_505, status: "succeeded", providerTurnStarted: true, + contextSnapshot: { wakeReason: "issue_recovery_action_restored", retryOfRunId: answering } }], + wakes: [{ id: "55a6afa9", reason: "issue_assigned", source: "assignment", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_941_952_144 }, + { id: "be67ffb7", reason: "issue_execution_promoted", source: "automation", triggerDetail: "system", requestedByActorType: "user", requestedByActorId: "local-board", + createdAt: 1_794_943_909_966, payload: { commentId: comment, _paperclipWakeContext: { wakeCommentIds: [comment] } } }, + { id: "549cf01c", reason: "transient_failure_retry", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: 1_794_944_484_384, payload: { retryReason: "transient_failure", retryOfRunId: promoted } }, + { id: "08e16978", reason: "transient_failure_retry", source: "automation", triggerDetail: "system", requestedByActorType: "system", requestedByActorId: null, + createdAt: 1_794_944_511_476, payload: { retryReason: "transient_failure", retryOfRunId: leaseRetry } }, + { id: "88d1a3e0", reason: "issue_recovery_action_restored", source: "automation", triggerDetail: "system", requestedByActorType: "system", + requestedByActorId: "execution-recovery", createdAt: 1_794_945_559_524, + payload: { recoveryActionId: "8d383ed2", _paperclipWakeContext: { wakeReason: "issue_recovery_action_restored", retryOfRunId: answering } } }], + }); + const posted = commentEvidence([ + { body: `The ledger document already contains all 26 rounds plus the \`FOLLOWUP-${nonce}\` line on its own line (revision 2). The \`${doneMarker(nonce)}\` comment was previously posted.`, + authorAgentId: "a", createdByRunId: continuation, createdAt: "2026-11-17T20:19:34.669Z" }, + { body: `Task complete. All 26 rounds ran and were recorded, the ledger document was saved, and the \`${doneMarker(nonce)}\` comment was posted.`, + authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-17T19:31:49.960Z" }, + { body: doneMarker(nonce), authorAgentId: "a", createdByRunId: first, createdAt: "2026-11-17T19:31:41.204Z" }]); + const completionOnce = (completion: CompletionEvidence) => gradeStagingMix(nonce, + { turnMinutes: 39, rounds: 4, subtasks: 0, followUpAfterMs: 1_102_604, arriveAtMs: 0, holdMs: 600_000 }, + { issueStatus: "done", documents: { ledger: `${ledger(nonce, 4)}\nFOLLOWUP-${nonce}` }, agentComments: [], files: {}, commands: {}, workspacePath: null, + stagingMix: { followUp: { at: 1_794_943_656_111, commentId: comment, duringRun: true, inOverlap: false, error: null }, followUpErrors: [], retries: [], comments: posted, + completion } }).checks.find(check => check.name === "completion_once")!; + const both = (evidence: WakeEvidenceFile) => [evidenceCompletion(evidence)!, completionEvidence({ completedAt: evidence.completedAt!, wakes: evidence.wakes, + runs: evidence.runs!.map(run => ({ ...run, acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null })) })]; + + it("passes when the continuation of an answering run that never committed completes the follow-up", () => { + const evidence = t128(); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(anchor).toEqual({ at: 1_794_943_908_538, source: "accepted_result" }); + expect(completionRunIds(posted, nonce, completion)).toEqual([first]); + const after = runsAfterDone("t128", evidence, anchor); + expect(after.started).toEqual([]); + expect(after.answeredFollowUp.map(turn => turn.runId)).toEqual([answering, continuation]); + expect(completionOnce(completion)).toMatchObject({ ok: true, detail: "runs=1" }); + } + }); + + const withLineage = (lineage: { context?: Record; retryOfRunId?: string }): WakeEvidenceFile => { + const evidence = t128(); + evidence.runs = evidence.runs!.map(run => run.runId === continuation + ? { ...run, retryOfRunId: lineage.retryOfRunId ?? null, contextSnapshot: { wakeReason: "issue_recovery_action_restored", ...lineage.context } } : run); + evidence.wakes = evidence.wakes!.map(wake => wake.id === "88d1a3e0" ? { ...wake, payload: { recoveryActionId: "8d383ed2" } } : wake); + return evidence; + }; + + it.each([ + { name: "only previousRunId", lineage: { context: { previousRunId: answering } } }, + { name: "only executionContinuation.interruptedRunId", lineage: { context: { executionContinuation: { interruptedRunId: answering } } } }, + { name: "only the retry_of_run_id column", lineage: { retryOfRunId: answering } }, + { name: "a retry_of_run_id column that names a sibling and a previousRunId that names the answering run", + lineage: { retryOfRunId: leaseRetry, context: { previousRunId: answering } } }, + ])("passes when the continuation carries $name", ({ lineage }) => { + const evidence = withLineage(lineage); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t128", evidence, anchor).started).toEqual([]); + expect(completionOnce(completion)).toMatchObject({ ok: true, detail: "runs=1" }); + } + }); + + it("still fails a retry of a committed continuation that keeps a stale previousRunId to the answering run", () => { + const retry = "9c9c9c9c-retry"; + const evidence = t128(); + evidence.runs!.push({ runId: retry, createdAt: 1_794_946_800_000, startedAt: 1_794_946_810_000, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: "w-retry", retryOfRunId: continuation, acceptedDisposition: "done", acceptedAt: 1_794_946_900_000, status: "succeeded", providerTurnStarted: true, + contextSnapshot: { wakeReason: "transient_failure_retry", previousRunId: answering, retryOfRunId: continuation } }); + evidence.wakes!.push({ id: "w-retry", reason: "transient_failure_retry", source: "automation", triggerDetail: "system", requestedByActorType: "system", + requestedByActorId: null, createdAt: 1_794_946_800_000, payload: { retryOfRunId: continuation } }); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t128", evidence, anchor).started.map(turn => turn.runId)).toEqual([retry]); + expect(completionOnce(completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 38a8772f, 9c9c9c9c each reported completion" }); + } + }); + + it("still fails a later continuation whose committed ancestor is reachable only through its lineage chain", () => { + const failedRetry = "7c7c7c7c-failed", finalRetry = "9d9d9d9d-final"; + const evidence = withLineage({ context: { previousRunId: answering } }); + evidence.runs!.push({ runId: failedRetry, createdAt: 1_794_946_800_000, startedAt: 1_794_946_810_000, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: null, retryOfRunId: continuation, acceptedDisposition: null, acceptedAt: null, status: "failed", providerTurnStarted: true, + contextSnapshot: { wakeReason: "transient_failure_retry", retryOfRunId: continuation } }, + { runId: finalRetry, createdAt: 1_794_946_900_000, startedAt: 1_794_946_910_000, invocationSource: "automation", triggerDetail: "system", + wakeupRequestId: null, retryOfRunId: failedRetry, acceptedDisposition: "done", acceptedAt: 1_794_947_000_000, status: "succeeded", providerTurnStarted: true, + contextSnapshot: { wakeReason: "transient_failure_retry", retryOfRunId: failedRetry, previousRunId: answering } }); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t128", evidence, anchor).started.map(turn => turn.runId)).toEqual([failedRetry, finalRetry]); + expect(completionOnce(completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 38a8772f, 9d9d9d9d each reported completion" }); + } + }); + + it("still fails a continuation with no lineage to the answering run", () => { + const evidence = withLineage({}); + for (const completion of both(evidence)) { + expect(completionOnce(completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 38a8772f, 833314ed each reported completion" }); + } + }); + + it("passes when the answering run's done result was rejected as incomplete", () => { + const evidence = t128("done"); + evidence.wakes!.push({ id: "incomplete-wake", reason: "issue_continuation_needed", source: "automation", triggerDetail: "system", requestedByActorType: "system", + requestedByActorId: null, createdAt: 1_794_945_400_000, payload: { continuationIdempotencyKey: COMPLETION_INCOMPLETE_KEY } }); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t128", evidence, anchor).started).toEqual([]); + expect(completionOnce(completion)).toMatchObject({ ok: true, detail: "runs=1" }); + } + }); + + it("still fails a continuation that completes again after the answering run committed its completion", () => { + const evidence = t128("done"); + for (const completion of both(evidence)) { + const anchor = completedAtAnchor(completion, posted, nonce); + expect(runsAfterDone("t128", evidence, anchor).started.map(turn => turn.runId)).toEqual([continuation]); + expect(completionOnce(completion)).toMatchObject({ ok: false, detail: "duplicate completion: runs 38a8772f, 833314ed each reported completion" }); + } + }); +}); diff --git a/tests/restart-chaos/staging-mix-retry.ts b/tests/restart-chaos/staging-mix-retry.ts index 89c4b329a6f..7f6fce2004b 100644 --- a/tests/restart-chaos/staging-mix-retry.ts +++ b/tests/restart-chaos/staging-mix-retry.ts @@ -6,7 +6,7 @@ import { bannerShownMs, completedAtAnchor, doneCommentsBeforeCompletion, DUPLICATE_CHECKS, firstDoneCommentAt, noteRetryBanner, type CompletionEvidence, } from "./staging-mix.js"; import type { InterruptionSample, OracleResult, TaskPlan, TaskRecord, TaskSample } from "./types.js"; -import { laterAnswers, laterAnswerWaits, rejectedDoneRuns, undeliveredFollowUpRuns, type LaterAnswer } from "./follow-up-delivery.js"; +import { followUpAnsweringRuns, laterAnswers, laterAnswerWaits, rejectedDoneRuns, undeliveredFollowUpRuns, type LaterAnswer } from "./follow-up-delivery.js"; type Json = Record; @@ -192,7 +192,7 @@ export interface ExtraTurn { export interface WakeEvidenceFile { completedAt?: number | null; runs?: Array<{ runId: string | null; createdAt: number | null; startedAt?: number | null; invocationSource: string | null; triggerDetail: string | null; - wakeupRequestId: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null; + wakeupRequestId: string | null; retryOfRunId?: string | null; acceptedDisposition?: string | null; acceptedAt?: number | null; finishedAt?: number | null; contextSnapshot?: unknown; status?: string | null; providerTurnStarted?: boolean | null }>; wakes?: Array<{ id: string | null; reason: string | null; source: string | null; triggerDetail: string | null; requestedByActorType: string | null; requestedByActorId: string | null; createdAt?: number | null; payload?: unknown }>; @@ -212,8 +212,8 @@ export function commentEvidence(rows: readonly CommentRow[] | null): Array<{ bod export function evidenceCompletion(evidence: WakeEvidenceFile | null, issueCompletedAt?: unknown): CompletionEvidence | null { const fromIssue = typeof issueCompletedAt === "string" && Number.isFinite(Date.parse(issueCompletedAt)) ? Date.parse(issueCompletedAt) : null; if (!evidence && fromIssue === null) return null; - const answers = laterAnswers(evidence); const rejected = rejectedDoneRuns(evidence); + const answers = followUpAnsweringRuns(evidence, rejected); return { completedAt: evidence?.completedAt ?? fromIssue, runs: (evidence?.runs ?? []).map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, acceptedDisposition: run.acceptedDisposition ?? null, acceptedAt: run.acceptedAt ?? null, answersFollowUp: Boolean(run.runId && answers.has(run.runId)), doneRejected: Boolean(run.runId && rejected.has(run.runId)) })) }; @@ -223,7 +223,7 @@ export function runsAfterDone(taskId: string, evidence: WakeEvidenceFile | null, { started: ExtraTurn[]; neverStarted: ExtraTurn[]; answeredFollowUp: ExtraTurn[] } { const doneAt = anchor.at; if (!evidence || doneAt === null) return { started: [], neverStarted: [], answeredFollowUp: [] }; - const answers = laterAnswers(evidence); + const answers = followUpAnsweringRuns(evidence, rejectedDoneRuns(evidence)); const wakes = new Map((evidence.wakes ?? []).map(wake => [wake.id, wake])); const turn = (run: NonNullable[number]): ExtraTurn => { const wake = run.wakeupRequestId ? wakes.get(run.wakeupRequestId) : undefined; diff --git a/tests/restart-chaos/wake-evidence.ts b/tests/restart-chaos/wake-evidence.ts index e1634941ec3..c5cf9d85155 100644 --- a/tests/restart-chaos/wake-evidence.ts +++ b/tests/restart-chaos/wake-evidence.ts @@ -4,7 +4,7 @@ import type { ServerProcessWindow } from "./multi-server.js"; import { unshiftJson, unshiftMs, type AgingShift } from "./time-faults.js"; import { ISSUE_STATUSES } from "../../packages/shared/src/constants.js"; import type { CompletionEvidence } from "./staging-mix.js"; -import { laterAnswers, rejectedDoneRuns, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js"; +import { followUpAnsweringRuns, rejectedDoneRuns, type DeliveryRun, type DeliveryWake } from "./follow-up-delivery.js"; import { AGENT_SLOTS_FILE, agentSlotsFromRows } from "./agent-slots.js"; import { agents, createDb, type Db } from "../../packages/db/src/index.js"; import { maxConcurrentRunsSql } from "../../server/src/services/heartbeat/timer-policy.js"; @@ -78,7 +78,7 @@ export function wakeEvidenceForIssue(issueId: string, input: WakeEvidenceInput) runs: runs.map(run => { const owner = controllerByBoot.get(String(run.controllerBootId ?? "")); return { runId: text(run.id), status: text(run.status), invocationSource: text(run.invocationSource), triggerDetail: text(run.triggerDetail), - wakeupRequestId: text(run.wakeupRequestId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), + wakeupRequestId: text(run.wakeupRequestId), retryOfRunId: text(run.retryOfRunId), createdAt: ms(run.createdAt), startedAt: ms(run.startedAt), finishedAt: ms(run.finishedAt), acceptedDisposition: text(run.acceptedDisposition), acceptedAt: ms(run.acceptedAt), contextSnapshot: run.contextSnapshot ?? null, providerTurnStarted: run.providerTurnStarted == null ? null : Boolean(run.providerTurnStarted), owner: owner ? { pid: owner.pid, bootId: owner.bootId, epoch: owner.epoch, processSpace: owner.processSpace } : null, servingAtCreate: servingAt(ms(run.createdAt), controllers, input.windows) }; @@ -97,7 +97,7 @@ async function readRows(db: Pick, companyId: string, s const client = db.$client; const runs = await client.unsafe( `select r.id::text as id, r.agent_id::text as "agentId", r.issue_id::text as "issueId", r.status, r.invocation_source as "invocationSource", r.trigger_detail as "triggerDetail", - r.wakeup_request_id::text as "wakeupRequestId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId", + r.wakeup_request_id::text as "wakeupRequestId", r.retry_of_run_id::text as "retryOfRunId", r.context_snapshot as "contextSnapshot", r.controller_boot_id::text as "controllerBootId", floor(extract(epoch from r.created_at) * 1000) as "createdAt", floor(extract(epoch from r.started_at) * 1000) as "startedAt", floor(extract(epoch from r.finished_at) * 1000) as "finishedAt", accepted.disposition as "acceptedDisposition", floor(extract(epoch from accepted.accepted_at) * 1000) as "acceptedAt", @@ -170,8 +170,8 @@ export interface EvidenceObserver { export function completionEvidence(evidence: { completedAt: number | null; wakes?: readonly DeliveryWake[]; runs: ReadonlyArray }): CompletionEvidence { - const answers = laterAnswers(evidence); const rejected = rejectedDoneRuns(evidence); + const answers = followUpAnsweringRuns(evidence, rejected); return { completedAt: evidence.completedAt, runs: evidence.runs.map(run => ({ runId: run.runId, startedAt: run.startedAt ?? null, acceptedDisposition: run.acceptedDisposition, acceptedAt: run.acceptedAt, answersFollowUp: Boolean(run.runId && answers.has(run.runId)), doneRejected: Boolean(run.runId && rejected.has(run.runId)) })) };