diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 1f0004da3a9..492b8f0cf76 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4621,6 +4621,10 @@ If shutdown cannot be proven, the run retains its checkpoint and the named `native_governed_wait_settlement_pending` condition. The existing bounded same-run recovery schedule owns further settlement; it grants no replacement or tool replay. The task explains that work is saved and shutdown is pending. +The active recovery action that schedules this same run's resume is the +authority for that resume, not a hold against it: the dispatch gate of the +resumed run (including a resolved-interaction continuation) skips only that +action and keeps every other reconciliation hold on the task. Exhausted or gated recovery holds continuation and names the shutdown problem. Actual provider failures and integrity failures retain their existing meaning. Accounting keeps its separate receipt and reconciliation requirements. diff --git a/server/src/modules/run-dispatch/adapters/postgres.test.ts b/server/src/modules/run-dispatch/adapters/postgres.test.ts index 6e99a1aeb98..1529eca1f79 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.test.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.test.ts @@ -979,6 +979,43 @@ describeEmbeddedPostgres("run-dispatch postgres adapter", () => { expect(dispatch).not.toHaveBeenCalled(); }); + // Staging run e892cf75: a governed wait was saved, shutdown confirmation of the + // retained execution was still pending, and the coordinator scheduled the + // same run to settle it. Its own recovery action then cancelled that resume. + it.each(["own-resume", "other-run", "no-resume-policy"] as const)( + "lets a saved governed wait settle on its own same-run resume (%s)", async mode => { + const f = await seedRecoveryAdmission(); + const context = { issueId: f.issueId, mutation: "interaction", wakeReason: "issue_commented", + interactionId: randomUUID(), interactionKind: "ask_user_questions", interactionStatus: "answered" }; + await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, f.runId)); + await db.update(nativeRunFinalizations).set({ phase: "retryable_failure", attempt: 1, + failureCode: "native_governed_wait_settlement_pending" }).where(eq(nativeRunFinalizations.runId, f.runId)); + const resumedRunId = mode === "other-run" ? randomUUID() : f.runId; + await db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, + kind: "active_run_watchdog", ownerType: "agent", ownerAgentId: f.agentId, returnOwnerAgentId: f.agentId, + cause: "native_governed_wait_settlement_pending", fingerprint: `governed-wait:${f.runId}`, + evidence: { runId: resumedRunId, sourceFailureCode: "native_governed_wait_settlement_pending", + recoveryMode: "exact_checkpoint_resume", coordinatorAttempt: 1, providerSessionEstablished: true }, + nextAction: "Resume the exact persisted native session on the same heartbeat run.", + wakePolicy: mode === "no-resume-policy" ? null + : { kind: "resume_native_run", runId: resumedRunId, notBefore: new Date().toISOString() }, + maxAttempts: 3, attemptCount: 1 }); + const dispatch = vi.fn(async () => "settled the saved governed wait"); + const gate = await createPostgresRunDispatchAdapter(db).dispatchResolvedInteractionIfCurrent({ companyId: f.companyId, + runId: f.runId, reservationId: f.reservation.id, recoveryLeaseOwner: f.owner, expectedStatus: "running", + enforceContinuationOwnership: true, now: new Date(), dispatch }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + if (mode === "own-resume") { + expect({ dispatched: gate.dispatched, status: run!.status, errorCode: run!.errorCode }) + .toEqual({ dispatched: true, status: "running", errorCode: null }); + if (gate.dispatched) expect(await gate.resultPromise).toBe("settled the saved governed wait"); + } else { + // Any other reconciliation hold on the task still stops this run. + expect(gate).toMatchObject({ dispatched: false, cancellation: { outcome: "cancelled", errorCode: "execution_reconciliation_required" } }); + expect(dispatch).not.toHaveBeenCalled(); + } + }); + it("rechecks recovery lease expiry after waiting for the issue lock", async () => { const f = await seedRecoveryAdmission(); const expiresAt = new Date(Date.now() + 1_000); diff --git a/server/src/modules/run-dispatch/adapters/postgres.ts b/server/src/modules/run-dispatch/adapters/postgres.ts index 5618daa66ef..13615065a5b 100644 --- a/server/src/modules/run-dispatch/adapters/postgres.ts +++ b/server/src/modules/run-dispatch/adapters/postgres.ts @@ -1277,7 +1277,12 @@ export function createPostgresRunDispatchAdapter( } }; } } - const recovery = await getExecutionBlocker(tx, run.companyId, issueId, { conversationResetCommentId: deriveCommentId(contextSnapshot) }); + // A native run's own scheduled resume is not a hold against that run: the + // same-run attempt is how its saved state (e.g. a governed wait) settles. + const recovery = await getExecutionBlocker(tx, run.companyId, issueId, { + conversationResetCommentId: deriveCommentId(contextSnapshot), + resumingNativeRunId: run.runtimeMode === "native" ? run.id : null, + }); if (recovery) return { issueId, facts: null, executionBlocker: recovery, decision: { stale: true as const, errorCode: "execution_reconciliation_required" as const, reason: recovery.nextAction, details: { issueId, recoveryActionId: recovery.recoveryActionId }, diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index 9509dfe9976..7bfaf9b054f 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -287,7 +287,26 @@ export function restoreHoldReleasedForCurrentOwnerPredicate() { )}), false)`; } -export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { conversationResetCommentId?: string | null }): Promise { +/** + * The coordinator's own scheduled resume of one native run: the active action + * that its failure wrote for that run and whose wake policy resumes that same + * run. It is the authority for that resume, not a hold against it. A saved + * governed wait whose shutdown confirmation is still pending records a + * reconciliation cause here, and its same-run resume is what settles it. + * Strictly true or false, because callers negate it. + */ +function ownNativeResumePredicate(runId: string) { + return sql`coalesce((${issueRecoveryActions.status} = 'active' + and ${issueRecoveryActions.wakePolicy}->>'kind' = 'resume_native_run' + and ${issueRecoveryActions.wakePolicy}->>'runId' = ${runId} + and ${issueRecoveryActions.evidence}->>'runId' = ${runId}), false)`; +} + +export async function getExecutionBlocker(db: Db, companyId: string, issueId: string, options?: { + conversationResetCommentId?: string | null; + /** The native run being dispatched; its own scheduled resume does not block it. */ + resumingNativeRunId?: string | null; +}): Promise { const [conversation] = await db.select({ agentId: issues.conversationAgentId, boundaryId: issues.conversationBoundaryCommentId, status: issues.status, assigneeAgentId: issues.assigneeAgentId, statusVersion: issues.statusVersion, @@ -330,6 +349,7 @@ export async function getExecutionBlocker(db: Db, companyId: string, issueId: st executionBlockerPredicate(), not(restoreHoldReleasedForCurrentOwnerPredicate()), boundary ? or(gt(issueRecoveryActions.createdAt, boundary.createdAt), nativeExecutionRecoveryActionPredicate()) : undefined, + options?.resumingNativeRunId ? not(ownNativeResumePredicate(options.resumingNativeRunId)) : undefined, )).orderBy(desc(issueRecoveryActions.updatedAt), desc(issueRecoveryActions.id)).limit(1); if (!action) return null; const parsedRunId = z.string().guid().safeParse(action.evidence.runId ?? action.evidence.sourceRunId); diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 4539c747378..a8e06c3219a 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -728,9 +728,9 @@ Codex app-server is late, missing or repeated. `--faults` takes them next to res `--provider-fault-message terminal` (default) picks the turn's end after an interruption, which a governed wait (a saved question or approval) or a board Stop sends. `interrupt_ack` picks the -acknowledgement of that interruption. Each rule applies once per task, to the first matching message. -Provider faults run only with `--env local --adapters codex,claude` (either or both) and only with -workloads that interrupt a turn (Q1, S1). G1 does not: its agent asks for approval through `call_api` +acknowledgement of that interruption. Each rule applies once per task, to the first matching message +(Q4: the second). Provider faults run only with `--env local --adapters codex,claude` (either or both) and only with +workloads that interrupt a turn (Q1, Q4, S1). G1 does not: its agent asks for approval through `call_api` and ends its own turn, so no rule ever applied there (waves w-20261110T050407-s206, w-20261110T063253-s208, w-20261110T085554-s211) and the board waited for a restart that never came. A PV1 task passes `provider_fault_applied` only when the held message was released; a provider that exited first turned the delay into a drop. @@ -832,6 +832,25 @@ question must settle on its own first. The oracle checks these points: Q1 is not a default workload. +**Q4** is Q1 twice in a row. The first answer wakes a resolved-interaction continuation run, and +that run saves the second question. A provider fault on a Q4 task takes the task's second matching +message (`occurrence` 2 in the rule): the interruption terminal of the second governed wait is late +(PV1) or lost (PV2). The wait must still settle without a failed or cancelled run. On a local Codex +runner the cancelled turn still settles inside the 30 s bound (waves w-20261111T151810-s111 and +w-20261111T152016-s111), so this cell does not by itself reach `native_governed_wait_settlement_pending`; +the run-dispatch adapter test covers that state's same-run resume. Staging run e892cf75 (2026-11-11) instead ended `cancelled` with +`execution_reconciliation_required`: the dispatch gate of the continuation treated the run's own +scheduled resume as a reconciliation hold, and the task stayed blocked with no Retry. + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex --workloads Q4 --faults PV2 --concurrency 1 --tasks 1 --seed 111 +``` + +The Q4 oracle checks that two questions were asked and both were answered, that both asking runs +ended `succeeded`, that no run of the task failed or was cancelled, and that the brief carries both +answers. Q4 is not a default workload. + ## Database contention (L1, `--db-pressure`) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index eae8f75317a..04856f7d269 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -184,8 +184,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const options: WaveOptions = { env, adapters, - // The governance scenarios (G1, H1), the journeys (Q1–Q3) and the contention run (L1) are opt-in: H1 needs a wave of its own. - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1" && id !== "L1" && id !== "F1" && id !== "T1" + // The governance scenarios (G1, H1), the journeys (Q1–Q4) and the contention run (L1) are opt-in: H1 needs a wave of its own. + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1" && id !== "L1" && id !== "F1" && id !== "T1" && id !== "Q4" && !takesJourneyTriggers(id) && (id !== "M1" || adapters.length >= 2))), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. faults: allFaults.filter((fault): fault is FaultKind => !isProviderFault(fault) && !isDiskFullFault(fault)), @@ -246,6 +246,11 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC // Each K1 kills the whole control plane: a second task in flight would lose its held message to the // first task's restart and miss its own, so a restart wave runs one task at a time. if (options.providerFaultRestart && options.concurrency !== 1) throw new Error("--provider-fault-restart restarts the whole control plane per task; run it with --concurrency 1"); + // The board holds its answers until the planned restart fires, but Q4's fault lands only after the + // first answer: the restart would wait for an answer that waits for the restart. + if ((options.providerFaultRestart || options.providerFaultRestartKinds?.length) && options.workloads.includes("Q4")) { + throw new Error("--provider-fault-restart does not cover Q4: its fault needs the first answer, which the board holds until the restart; run Q4 without it"); + } } else if (values["provider-fault-message"] !== undefined || values["provider-fault-delays"] !== undefined || values["provider-fault-restart"] !== undefined || values["provider-fault-restart-after-seconds"] !== undefined) { throw new Error("--provider-fault-message, --provider-fault-delays and --provider-fault-restart need a provider fault (PV1, PV2 or PV3) in --faults"); diff --git a/tests/restart-chaos/journeys.ts b/tests/restart-chaos/journeys.ts index 4feafb418f1..67a54a9fb72 100644 --- a/tests/restart-chaos/journeys.ts +++ b/tests/restart-chaos/journeys.ts @@ -35,7 +35,7 @@ export type JourneyTrigger = (typeof JOURNEY_TRIGGERS)[number]; /** Each workload's own fault boundaries, for the journey × fault matrix. */ export const WORKLOAD_TRIGGERS: Record = { A1: ["time"], B1: ["time"], B5: ["tool"], C1: ["comment", "result"], M1: ["time"], G1: ["approval"], - H1: ["paused"], R1: ["time"], S1: ["stop"], L1: ["time"], F1: ["time"], T1: ["time"], Q1: JOURNEY_TRIGGERS, Q2: JOURNEY_TRIGGERS, Q3: JOURNEY_TRIGGERS, + H1: ["paused"], R1: ["time"], S1: ["stop"], L1: ["time"], F1: ["time"], T1: ["time"], Q1: JOURNEY_TRIGGERS, Q2: JOURNEY_TRIGGERS, Q3: JOURNEY_TRIGGERS, Q4: ["time"], }; /** Q3 reviews must survive a delay longer than the implicit deadline #20340 removed. */ diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 2a74d8bcc4c..e39f9ac5900 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -818,7 +818,7 @@ export class TaskObserver { if (this.journey) return this.journey.awaitingBoard(); if (this.plan.workload === "G1") return Boolean(governance?.approvalSeenAt) && !governance?.decision; if (this.plan.workload === "H1") return Boolean(governance?.budgetPausedAt) && !governance?.resume && !governance?.pauseLostAt; - if (this.plan.workload === "Q1") return awaitingQuestionAnswer(this.record); + if (this.plan.workload === "Q1" || this.plan.workload === "Q4") return awaitingQuestionAnswer(this.record); return false; } @@ -853,10 +853,11 @@ export class TaskObserver { if (!sample.reachable || !this.record.issueId) return; if (this.plan.workload === "G1") await this.approvalStep(sample, now); if (this.plan.workload === "H1") await this.budgetStep(sample, now); - if (this.plan.workload === "Q1") { + if (this.plan.workload === "Q1" || this.plan.workload === "Q4") { await questionStep({ api: this.api, issueId: this.record.issueId, nonce: this.plan.nonce, record: this.record, sample, now, // An `answered` fault waits for the answer, so the board may answer before it fires. - boardMayAct: this.boardMayAct(sample, "answered"), note: (kind, data) => this.timeline.add(kind, { taskId: this.plan.taskId, ...data }) }); + boardMayAct: this.boardMayAct(sample, "answered"), sequence: this.plan.workload === "Q4", + note: (kind, data) => this.timeline.add(kind, { taskId: this.plan.taskId, ...data }) }); } } @@ -1035,7 +1036,7 @@ export class TaskObserver { ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), ...(this.record.governance ? { governance: this.record.governance } : {}), - ...(this.plan.workload === "Q1" ? { question: await collectQuestionArtifacts(this.api, issueId, this.record) } : {}), + ...(this.plan.workload === "Q1" || this.plan.workload === "Q4" ? { question: await collectQuestionArtifacts(this.api, issueId, this.record) } : {}), ...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}), }; } diff --git a/tests/restart-chaos/provider-faults.test.ts b/tests/restart-chaos/provider-faults.test.ts index c0b04c469f5..3c31a00f3f9 100644 --- a/tests/restart-chaos/provider-faults.test.ts +++ b/tests/restart-chaos/provider-faults.test.ts @@ -128,6 +128,17 @@ describe("provider message fault shim", () => { expect((await turnThroughShim(root, "nonce cccc2222")).terminals).toHaveLength(1); }); + it("occurrence 2 passes the task's first interrupted terminal and drops its second (Q4)", async () => { + const { seam: installed, root } = await seam(); + await installed.arm([task("ffff5555", { kind: "PV2", message: "terminal", occurrence: 2 })]); + expect((await turnThroughShim(root, "nonce ffff5555")).terminals).toHaveLength(1); + expect((await turnThroughShim(root, "nonce ffff5555")).terminals).toHaveLength(0); + expect((await turnThroughShim(root, "nonce ffff5555")).terminals).toHaveLength(1); + const events = await installed.eventsFor("ffff5555"); + expect(events.filter(event => event.event === "passed")).toMatchObject([{ kind: "PV2", occurrence: 2, seen: 1 }]); + expect(events.filter(event => event.event === "applied")).toMatchObject([{ kind: "PV2", message: "terminal", action: "drop" }]); + }); + it("targets the interruption acknowledgement", async () => { const { seam: installed, root } = await seam(); await installed.arm([task("dddd3333", { kind: "PV2", message: "interrupt_ack" })]); diff --git a/tests/restart-chaos/provider-faults.ts b/tests/restart-chaos/provider-faults.ts index 3c444dd3535..59808caf906 100644 --- a/tests/restart-chaos/provider-faults.ts +++ b/tests/restart-chaos/provider-faults.ts @@ -45,6 +45,11 @@ export interface ProviderFaultRule { message: ProviderMessage; /** PV1 only. */ delayMs?: number; + /** + * Take the task's Nth matching message instead of its first (Q4: the second saved question, + * asked by the run that continues from the first answer). Earlier ones pass unchanged. + */ + occurrence?: number; /** * `--provider-fault-restart K1|K2|K4` (PV1, PV2): restart the control plane this long after the fault * applied, so the held or dropped message straddles a restart. `holdMs` is the K4 outage. @@ -59,7 +64,7 @@ export const DEFAULT_PROVIDER_FAULT_RESTART_AFTER_MS = 3_000; export interface ProviderFaultEvent { at: number; pid: number; - event: "spawned" | "bound" | "interrupt_requested" | "applied" | "released" | "held_message_discarded" | "exited" | "spawn_failed"; + event: "spawned" | "bound" | "interrupt_requested" | "passed" | "applied" | "released" | "held_message_discarded" | "exited" | "spawn_failed"; nonce?: string | null; kind?: ProviderFaultKind; message?: ProviderMessage; @@ -70,12 +75,15 @@ export interface ProviderFaultEvent { } /** - * Workloads whose run sends turn/interrupt: a saved question (Q1) and a board Stop (S1). Not G1: its agent + * Workloads whose run sends turn/interrupt: a saved question (Q1, and Q4 a second one from the answer's + * continuation run) and a board Stop (S1). Not G1: its agent * asks for approval through `call_api` and ends its own turn, so nothing interrupts the provider and no * rule ever applies (waves w-20261110T050407-s206, w-20261110T063253-s208 and w-20261110T085554-s211: * applied=0 for Codex and Claude, and the board then waits for a restart that never comes). */ -export const PROVIDER_FAULT_WORKLOADS: readonly WorkloadId[] = ["Q1", "S1"]; +export const PROVIDER_FAULT_WORKLOADS: readonly WorkloadId[] = ["Q1", "Q4", "S1"]; +/** Q4's fault targets its second saved question, the one asked inside the answer's continuation run. */ +export const Q4_PROVIDER_FAULT_OCCURRENCE = 2; export function isProviderFault(value: string): value is ProviderFaultKind { return (PROVIDER_FAULT_KINDS as readonly string[]).includes(value); @@ -139,6 +147,7 @@ export function withProviderFaultRules(tasks: TaskPlan[], options: Pick 1 ? ` #${rule.occurrence}` : ""}` + + `${rule.delayMs !== undefined ? ` ${rule.delayMs}ms` : ""}` + (rule.restart ? ` +${rule.restart.kind}@${rule.restart.afterAppliedMs}ms${rule.restart.holdMs ? `/hold${rule.restart.holdMs}ms` : ""}` : ""); } diff --git a/tests/restart-chaos/provider-shim/acpx-fault-tap.cjs b/tests/restart-chaos/provider-shim/acpx-fault-tap.cjs index 508c3522dad..9e61766d5da 100644 --- a/tests/restart-chaos/provider-shim/acpx-fault-tap.cjs +++ b/tests/restart-chaos/provider-shim/acpx-fault-tap.cjs @@ -41,8 +41,19 @@ function createFaultRuntime(config, now = Date.now) { }, /** Every rule applies at most once per task, across sidecar and provider restarts. */ claim(rule) { - const marker = path.join(config.stateDirectory, `${rule.nonce}.${rule.kind}.${rule.message}.applied`); - try { writeFileSync(marker, String(process.pid), { flag: "wx" }); return true; } catch { return false; } + const base = path.join(config.stateDirectory, `${rule.nonce}.${rule.kind}.${rule.message}`); + // `occurrence` N (default 1): pass the first N-1 matching messages of the task, take the Nth. + const occurrence = Number.isSafeInteger(rule.occurrence) && rule.occurrence > 1 ? rule.occurrence : 1; + if (occurrence > 1) { + let seen = 0; + try { seen = readFileSync(`${base}.seen`, "utf8").length; } catch { /* none yet */ } + if (seen < occurrence - 1) { + try { appendFileSync(`${base}.seen`, "x"); } catch { /* best effort */ } + this.log({ event: "passed", nonce: rule.nonce, kind: rule.kind, message: rule.message, occurrence, seen: seen + 1 }); + return false; + } + } + try { writeFileSync(`${base}.applied`, String(process.pid), { flag: "wx" }); return true; } catch { return false; } }, }; } diff --git a/tests/restart-chaos/provider-shim/codex-shim.mjs b/tests/restart-chaos/provider-shim/codex-shim.mjs index dad34e4a612..ef7f374afba 100644 --- a/tests/restart-chaos/provider-shim/codex-shim.mjs +++ b/tests/restart-chaos/provider-shim/codex-shim.mjs @@ -11,7 +11,7 @@ // Rules are armed per task in the control file, by the task's nonce. A process is // bound to a nonce once the nonce appears in its traffic (the task's prompt or a // tool result). Every rule applies at most once per task (an exclusive marker file -// survives provider restarts). Every decision is appended to the log file. +// survives provider restarts), to its `occurrence`-th matching message (default the first). Every decision is appended to the log file. // Plain Node built-ins only: this runs in the provider process, not the harness. import { spawn } from "node:child_process"; import { appendFileSync, readFileSync, statSync, writeFileSync } from "node:fs"; @@ -66,8 +66,20 @@ function parse(line) { } function claim(rule) { - const marker = path.join(config.stateDirectory, `${rule.nonce}.${rule.kind}.${rule.message}.applied`); - try { writeFileSync(marker, String(process.pid), { flag: "wx" }); return true; } catch { return false; } + const base = path.join(config.stateDirectory, `${rule.nonce}.${rule.kind}.${rule.message}`); + // `occurrence` N (default 1): pass the first N-1 matching messages of the task, take the Nth. + // The count is a file, so it survives provider restarts like the marker does. + const occurrence = Number.isSafeInteger(rule.occurrence) && rule.occurrence > 1 ? rule.occurrence : 1; + if (occurrence > 1) { + let seen = 0; + try { seen = readFileSync(`${base}.seen`, "utf8").length; } catch { /* none yet */ } + if (seen < occurrence - 1) { + try { appendFileSync(`${base}.seen`, "x"); } catch { /* best effort */ } + log({ event: "passed", nonce: rule.nonce, kind: rule.kind, message: rule.message, occurrence, seen: seen + 1 }); + return false; + } + } + try { writeFileSync(`${base}.applied`, String(process.pid), { flag: "wx" }); return true; } catch { return false; } } const child = spawn(config.realCommand, process.argv.slice(2), { stdio: ["pipe", "pipe", "inherit"], env: process.env }); diff --git a/tests/restart-chaos/question-handoff.test.ts b/tests/restart-chaos/question-handoff.test.ts index ade2c8df09a..7b5ae3a013c 100644 --- a/tests/restart-chaos/question-handoff.test.ts +++ b/tests/restart-chaos/question-handoff.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it } from "vitest"; import { parseCommand } from "./config.js"; -import { awaitingQuestionAnswer, gradeQuestionHandoff, q1AnswerMarker, q1Code, questionStep, type QuestionArtifacts } from "./question-handoff.js"; +import { withProviderFaultRules } from "./provider-faults.js"; +import { + awaitingQuestionAnswer, gradeQuestionHandoff, gradeQuestionSequence, q1AnswerMarker, q1Code, q4SecondQuestion, q4Tone, q4ToneMarker, + questionStep, type QuestionArtifacts, +} from "./question-handoff.js"; import type { OracleArtifacts, TaskRecord, TaskSample } from "./types.js"; import { workloadDefinitions } from "./workloads.js"; @@ -20,6 +24,72 @@ function fakeApi(interactions: Array>) { const pendingQuestion = { id: "q1", kind: "ask_user_questions", status: "pending", sourceRunId: "run-1", payload: { questions: [{ id: "codename", options: [{ id: "opt-a", label: "ALPHA" }, { id: "opt-b", label: "BRAVO" }] }] } }; +describe("Q4 second saved question from a continuation", () => { + const second = { id: "q2", kind: "ask_user_questions", status: "pending", sourceRunId: "run-1", + payload: { questions: [{ id: "tone", prompt: q4SecondQuestion(NONCE), options: [{ id: "opt-c", label: "CALM" }, { id: "opt-d", label: "BRISK" }] }] } }; + + it("is an opt-in provider-fault workload whose fault takes the second interrupt", () => { + expect(workloadDefinitions.Q4.description(NONCE)).toContain(q4SecondQuestion(NONCE)); + const command = parseCommand(["--adapters", "codex", "--workloads", "Q4", "--faults", "PV2"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.workloads).toEqual(["Q4"]); + const plain = parseCommand(["--adapters", "codex,claude"]); + if (plain.mode !== "wave") throw new Error("expected a wave"); + expect(plain.options.workloads).not.toContain("Q4"); + const [planned] = withProviderFaultRules([{ taskId: "t01", index: 0, workload: "Q4", adapter: "codex", fault: "PV2", + fireAfterMs: 0, holdMs: 0, followUpAfterMs: null, nonce: NONCE }], {}); + expect(planned!.providerFault).toMatchObject({ kind: "PV2", message: "terminal", occurrence: 2 }); + }); + + it("answers each question once, after its asking run settled, with the nonce's options", async () => { + const task = record(); + const interactions: Array> = [pendingQuestion]; + const { api, posts } = fakeApi(interactions); + const step = (status: string, now: number) => questionStep({ api, issueId: "i1", nonce: NONCE, record: task, sample: sample(status), now, + boardMayAct: true, sequence: true, note: () => undefined }); + await step("running", 1_000); + expect(awaitingQuestionAnswer(task)).toBe(true); + await step("succeeded", 2_000); + interactions[0] = { ...pendingQuestion, status: "answered" }; + await step("succeeded", 2_500); + expect(awaitingQuestionAnswer(task)).toBe(false); + interactions.push(second); + await step("running", 3_000); + expect(posts).toHaveLength(1); + await step("succeeded", 4_000); + await step("succeeded", 5_000); + expect(posts.map(post => post.route)).toEqual(["/api/issues/i1/interactions/q1/respond", "/api/issues/i1/interactions/q2/respond"]); + expect(posts[1]!.body).toEqual({ answers: [{ questionId: "tone", optionIds: [q4Tone(NONCE) === "CALM" ? "opt-c" : "opt-d"] }] }); + expect(task.question?.answers?.map(answer => answer.code)).toEqual([q1Code(NONCE), q4Tone(NONCE)]); + }); + + const sequenceArtifacts = (status: string, errorCode: string | null, secondAsker = "run-2"): OracleArtifacts => ({ issueStatus: "done", + agentComments: [], files: {}, commands: {}, workspacePath: null, + documents: { brief: `${q1AnswerMarker(NONCE, q1Code(NONCE))}\n${q4ToneMarker(NONCE, q4Tone(NONCE))}\n- a\n- b\n- c\n` }, + question: { interactionIds: ["q1", "q2"], seenAt: 1, askedByRunId: "run-1", pendingNow: false, statuses: { q1: "answered", q2: "answered" }, + answers: [{ at: 2_000, interactionId: "q1", code: q1Code(NONCE), httpStatus: 200, error: null, askedByRunId: "run-1" }, + { at: 4_000, interactionId: "q2", code: q4Tone(NONCE), httpStatus: 200, error: null, askedByRunId: secondAsker }], + runs: [{ id: "run-1", status: "succeeded", errorCode: null, createdAt: new Date(500).toISOString() }, + { id: "run-2", status, errorCode, createdAt: new Date(2_500).toISOString() }, + { id: "run-3", status: "succeeded", errorCode: null, createdAt: new Date(4_500).toISOString() }] } }); + + it("fails a cancelled run, as staging run e892cf75 ended", () => { + expect(gradeQuestionSequence(NONCE, sequenceArtifacts("succeeded", null)).verdict).toBe("pass"); + const cancelled = gradeQuestionSequence(NONCE, sequenceArtifacts("cancelled", "execution_reconciliation_required")); + expect(cancelled.checks.filter(check => !check.ok).map(check => check.name)).toEqual(["asking_runs_settled", "no_failed_or_cancelled_run"]); + }); + + it("fails when one run asked both questions, so no continuation saved the second wait", () => { + const sameRun = gradeQuestionSequence(NONCE, sequenceArtifacts("succeeded", null, "run-1")); + expect(sameRun.checks.filter(check => !check.ok).map(check => check.name)).toEqual(["second_asked_by_continuation"]); + }); + + it("rejects --provider-fault-restart, whose restart would wait for an answer that waits for it", () => { + expect(() => parseCommand(["--adapters", "codex", "--workloads", "Q4", "--faults", "PV2", "--provider-fault-restart", "K1", + "--concurrency", "1"])).toThrow(/does not cover Q4/); + }); +}); + describe("Q1 saved question", () => { it("is registered as an opt-in workload", () => { expect(workloadDefinitions.Q1.id).toBe("Q1"); diff --git a/tests/restart-chaos/question-handoff.ts b/tests/restart-chaos/question-handoff.ts index fbb93a9ac1d..a548a8a7ac6 100644 --- a/tests/restart-chaos/question-handoff.ts +++ b/tests/restart-chaos/question-handoff.ts @@ -27,7 +27,15 @@ export interface QuestionRecord { askedByRunId: string | null; /** The question is pending at the latest poll. */ pendingNow: boolean; - answer?: { at: number; interactionId: string; code: string; httpStatus: number | null; error: string | null }; + answer?: QuestionAnswer; + /** Q4: every answer the board gave, in order (`answer` is the first). */ + answers?: QuestionAnswer[]; +} + +export interface QuestionAnswer { + at: number; interactionId: string; code: string; httpStatus: number | null; error: string | null; + /** Q4: the run that asked the answered question. */ + askedByRunId?: string | null; } export interface QuestionArtifacts extends QuestionRecord { @@ -57,13 +65,49 @@ const description = (nonce: string) => [ "4. When the document is saved, mark this task done.", ].join("\n"); +/** + * Q4: two saved questions in a row. The second is asked by the run that the first answer woke (a + * resolved-interaction continuation), so its governed wait is saved inside a continuation run. With + * a provider fault on that second wait (provider-faults.ts `occurrence` 2), shutdown confirmation of + * the retained execution is late or lost, and the run's same-run resume must still settle the wait + * by itself. Staging run e892cf75 was instead cancelled by its own recovery action. + */ +export const Q4_TONES = ["CALM", "BRISK"] as const; +export function q4Tone(nonce: string): (typeof Q4_TONES)[number] { + return Number.parseInt(nonce.slice(-2, -1), 16) % 2 === 0 ? "CALM" : "BRISK"; +} +export const q4SecondQuestion = (nonce: string) => `Which tone should the Q4-${nonce} brief use?`; +export const q4ToneMarker = (nonce: string, tone: string) => `TONE-${nonce}-${tone}`; + +const q4Description = (nonce: string) => [ + "This task needs two answers from the board, one after the other, before you write anything.", + `1. Ask exactly one question with your \`request_human_input\` tool: a questions card (continuationPolicy \`wake_assignee\`,`, + ` idempotencyKey \`q4a-${nonce}\`) with one required single-select question "${q1Question(nonce)}"`, + ` and exactly two options labelled \`${Q1_CODES[0]}\` and \`${Q1_CODES[1]}\`. Then end your turn and wait for the answer.`, + "2. When you are woken with that answer, ask the second question the same way, with your `request_human_input` tool:", + ` a questions card (continuationPolicy \`wake_assignee\`, idempotencyKey \`q4b-${nonce}\`) with one required single-select`, + ` question "${q4SecondQuestion(nonce)}" and exactly two options labelled \`${Q4_TONES[0]}\` and \`${Q4_TONES[1]}\`.`, + " Then end your turn and wait for the answer.", + "3. Never guess an answer, never ask either question twice, and create no documents, files or tasks before both answers.", + "4. When you are woken with the second answer, save this issue's document with key `brief` (title \"Brief\"), in markdown.", + ` It must contain the line \`${q1AnswerMarker(nonce, "")}\` and the line \`${q4ToneMarker(nonce, "")}\`, each exactly`, + " and on its own line, with and replaced by the chosen options, and at least three bullet points", + " starting with \"- \" on why a saved question must survive a provider that stops answering.", + "5. When the document is saved, mark this task done.", +].join("\n"); + function questionInteractions(rows: Json[] | null | undefined): Json[] { return (rows ?? []).filter(row => row.kind === "ask_user_questions"); } -/** Q1 board step, called on every poll. */ -export async function questionStep(input: { api: Pick; issueId: string; nonce: string; record: TaskRecord; - sample: TaskSample; now: number; boardMayAct: boolean; note: (kind: string, data: Record) => void }): Promise { +type QuestionStepInput = { api: Pick; issueId: string; nonce: string; record: TaskRecord; + sample: TaskSample; now: number; boardMayAct: boolean; note: (kind: string, data: Record) => void; + /** Q4: answer each saved question once, in order. */ + sequence?: boolean }; + +/** Q1 and Q4 board step, called on every poll. */ +export async function questionStep(input: QuestionStepInput): Promise { + if (input.sequence) return questionSequenceStep(input); const question = input.record.question ??= { interactionIds: [], seenAt: null, askedByRunId: null, pendingNow: false }; const rows = await input.api.get(`/api/issues/${input.issueId}/interactions`, 5_000).catch(() => null); if (rows === null) return; @@ -102,8 +146,52 @@ export async function questionStep(input: { api: Pick; input.note("question_answered", { ...answer }); } -/** The task waits on the board while its question is pending and unanswered. */ +async function questionSequenceStep(input: QuestionStepInput): Promise { + const question = input.record.question ??= { interactionIds: [], seenAt: null, askedByRunId: null, pendingNow: false }; + const answers = question.answers ??= []; + const rows = await input.api.get(`/api/issues/${input.issueId}/interactions`, 5_000).catch(() => null); + if (rows === null) return; + const interactions = questionInteractions(rows); + for (const interaction of interactions) if (!question.interactionIds.includes(String(interaction.id))) question.interactionIds.push(String(interaction.id)); + const pending = interactions.find(interaction => interaction.status === "pending" + && !answers.some(answer => answer.interactionId === String(interaction.id))); + question.pendingNow = Boolean(pending); + if (!pending) return; + const askedByRunId = typeof pending.sourceRunId === "string" ? pending.sourceRunId + : (input.sample.runs ?? []).find(run => run.status === "running")?.id ?? null; + if (question.seenAt === null) { + question.seenAt = input.now; + const created = Date.parse(String(pending.createdAt ?? "")); + question.savedAt = Number.isFinite(created) ? Math.min(created, input.now) : input.now; + question.askedByRunId = askedByRunId; + input.note("question_pending", { interactionId: pending.id, askedByRunId, ordinal: answers.length + 1 }); + } + // Each saved question must settle on its own first: answer only once nothing of the task is running. + if (!input.boardMayAct || (input.sample.runs ?? []).some(run => ACTIVE.has(run.status))) return; + const target = ((pending.payload?.questions ?? []) as Json[])[0]; + const prompt = String(target?.prompt ?? ""); + const code = prompt.includes(q4SecondQuestion(input.nonce)) ? q4Tone(input.nonce) : q1Code(input.nonce); + const option = ((target?.options ?? []) as Json[]).find(candidate => String(candidate.label ?? "").toUpperCase().includes(code)); + const answer: QuestionAnswer = { at: input.now, interactionId: String(pending.id), code, httpStatus: null, error: null, askedByRunId }; + try { + if (!target) throw new Error("the pending question card has no questions"); + await input.api.post(`/api/issues/${input.issueId}/interactions/${pending.id}/respond`, { + answers: [{ questionId: String(target.id), optionIds: option ? [String(option.id)] : [], ...(option ? {} : { otherText: code }) }], + }); + answer.httpStatus = 200; + } catch (error) { + answer.httpStatus = typeof (error as { status?: unknown }).status === "number" ? (error as { status: number }).status : null; + answer.error = error instanceof Error ? error.message : String(error); + } + answers.push(answer); + question.answer ??= answer; + question.pendingNow = false; + input.note("question_answered", { ...answer, ordinal: answers.length }); +} + +/** The task waits on the board while a question is pending and unanswered. */ export function awaitingQuestionAnswer(record: TaskRecord): boolean { + if (record.question?.answers) return Boolean(record.question.pendingNow); return Boolean(record.question?.pendingNow) && !record.question?.answer; } @@ -149,6 +237,55 @@ export function gradeQuestionHandoff(nonce: string, artifacts: OracleArtifacts): return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; } +/** + * Q4 oracle. Beyond Q1: two questions, each answered once; no run of the task failed or was + * cancelled (staging run e892cf75 ended `cancelled` with execution_reconciliation_required); + * both asking runs settled; the brief carries both answers. + */ +export function gradeQuestionSequence(nonce: string, artifacts: OracleArtifacts): OracleResult { + const question = artifacts.question; + const brief = artifacts.documents.brief ?? ""; + const bullets = brief.split(/\r?\n/).filter(line => /^\s*[-*] \S/.test(line)).length; + const runs = question?.runs ?? []; + const answers = question?.answers ?? []; + const stopped = runs.filter(run => ["failed", "timed_out", "cancelled"].includes(run.status)); + const askers = answers.map(answer => answer.askedByRunId ? runs.find(run => run.id === answer.askedByRunId) : undefined); + const statuses = Object.values(question?.statuses ?? {}); + const code = q1Code(nonce), tone = q4Tone(nonce); + const checks = [ + { name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` }, + { name: "two_questions_asked", ok: question?.interactionIds.length === 2, detail: `questions=${question?.interactionIds.length ?? 0}` }, + { name: "both_answered_by_board", ok: answers.length === 2 && answers.every(answer => answer.httpStatus === 200), + detail: answers.map(answer => `${answer.code}:${answer.httpStatus}${answer.error ? ` ${answer.error}` : ""}`).join(",") || "none" }, + { name: "questions_settled", ok: statuses.length === 2 && statuses.every(status => status === "answered"), detail: statuses.join(",") || "none" }, + // The second question must come from the run the first answer woke, the case under test. + { name: "second_asked_by_continuation", ok: askers.length === 2 && Boolean(askers[0] && askers[1]) && askers[0]!.id !== askers[1]!.id + && Boolean(askers[1]!.createdAt) && Date.parse(askers[1]!.createdAt!) > answers[0]!.at, + detail: askers.map(run => run ? `${run.id}@${run.createdAt ?? "?"}` : "unknown").join(",") || "none" }, + { name: "asking_runs_settled", ok: askers.length === 2 && askers.every(run => run?.status === "succeeded"), + detail: askers.map((run, index) => run ? `${run.id}:${run.status}${run.errorCode ? `:${run.errorCode}` : ""}` + : `run=${answers[index]?.askedByRunId ?? "unknown"}`).join(",") || "none" }, + { name: "no_failed_or_cancelled_run", ok: stopped.length === 0, + detail: stopped.map(run => `${run.id}:${run.status}:${run.errorCode ?? ""}`).join(",") || "none" }, + { name: "brief_has_answers", ok: brief.includes(q1AnswerMarker(nonce, code)) && brief.includes(q4ToneMarker(nonce, tone)), + detail: `${q1AnswerMarker(nonce, code)} ${q4ToneMarker(nonce, tone)}` }, + { name: "brief_has_three_bullets", ok: bullets >= 3, detail: `bullets=${bullets}` }, + ]; + return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; +} + +export const questionSequenceWorkload: WorkloadDefinition = { + id: "Q4", + label: "Second saved question, asked by the first answer's continuation, settles and is answered", + usesRepository: false, + fireWindowMs: [20_000, 60_000], + budgetMs: 20 * 60_000, + title: nonce => `Chaos Q4 question sequence ${nonce}`, + description: q4Description, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade: gradeQuestionSequence, +}; + export const questionHandoffWorkload: WorkloadDefinition = { id: "Q1", label: "Saved question settles and is answered", diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 9e3cf8a48be..79e95820a3f 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -32,7 +32,7 @@ export type TaskFaultKind = FaultKind | "K13" | "K0" | ProviderFaultKind | DiskF * F1: an absolute symlink pointing out of the repository workspace, kept at rest (staging save-back class F). * T1: a 100-second shell tool in the agent's workspace, so a fault lands while the remote Runner runs a tool (staging class R). */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1", "Q1", "L1", "Q2", "Q3", "F1", "T1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1", "Q1", "L1", "Q2", "Q3", "Q4", "F1", "T1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; @@ -325,7 +325,7 @@ export interface OracleArtifacts { approvals?: Array<{ id: string; status: string; title: string | null }>; /** G1 and H1: the board's actions and what the observer saw around them. */ governance?: GovernanceRecord; - /** Q1: the saved question and the task's runs. */ + /** Q1, Q4: the saved questions and the task's runs. */ question?: QuestionArtifacts; /** PV1–PV3: the shim's log lines for this task. */ providerFaultEvents?: ProviderFaultEvent[]; @@ -417,7 +417,7 @@ export interface TaskRecord { governance?: GovernanceRecord; /** K14 waves: the build serving when the task launched. */ serverBuildAtLaunch?: "upgrade_from" | "current"; - /** Q1: the saved question and the board's answer. */ + /** Q1, Q4: the saved questions and the board's answers. */ question?: QuestionRecord; /** Q2, Q3: the board's journey actions (journey-driver.ts). */ journey?: import("./journeys.js").JourneyRecord; diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index 6e0156121f8..73cf39825fe 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -1,4 +1,4 @@ -import { questionHandoffWorkload } from "./question-handoff.js"; +import { questionHandoffWorkload, questionSequenceWorkload } from "./question-handoff.js"; import { l1Workload } from "./contention.js"; import { journeyWorkloadDefinitions } from "./journeys.js"; import type { OracleArtifacts, OracleResult, WorkloadId } from "./types.js"; @@ -215,6 +215,7 @@ function budgetChecks(artifacts: OracleArtifacts) { export const workloadDefinitions: Record = { Q1: questionHandoffWorkload, + Q4: questionSequenceWorkload, L1: l1Workload, ...journeyWorkloadDefinitions, G1: {