diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts index 8d6f60e8ad8..6e9694699cb 100644 --- a/packages/shared/src/crash-point-registry.ts +++ b/packages/shared/src/crash-point-registry.ts @@ -121,6 +121,13 @@ export const CRASH_POINTS = { invariants: ["I2"], prerequisite: "A reuse_by_environment lease that expires or is replaced (destroyReusableSandboxLease). The chaos Daytona environment's leases are ephemeral by default.", }, + "agent_directory.operation_lock.before_acquire": { + description: "A remote agent-directory operation (collect, checkpoint, release) is about to take its per-company-and-agent Postgres lock; no agent file has been read or written. In fail mode the call site throws the lock's own conflict (\"Agent directory operation is already active\"), staging class A's terminal step, so a recovery that collects a stopped run's agent files meets a held lock.", + environments: ["daytona"], + kinds: ["copyback_owner_hold"], + invariants: ["I2", "I7"], + prerequisite: "A remote (Daytona) agent directory copy: a task whose run materialized the agent's files in a sandbox.", + }, "environment.stop_retain.after_claim_before_stop": { description: "A stop-and-retain intent and its claim are committed; the provider stop has not been retried.", environments: ["daytona"], diff --git a/packages/shared/src/crash-points.test.ts b/packages/shared/src/crash-points.test.ts index 804a3acfd7a..3564a1f62d2 100644 --- a/packages/shared/src/crash-points.test.ts +++ b/packages/shared/src/crash-points.test.ts @@ -53,7 +53,13 @@ describe("parseCrashPointSpec", () => { describe("parseCrashPointControl", () => { it("reads a token and a spec, and treats anything else as disarmed", () => { - expect(parseCrashPointControl(`t1 ${first}:3\n`)).toEqual({ token: "t1", spec: { name: first, nth: 3 } }); + expect(parseCrashPointControl(`t1 ${first}:3\n`)).toEqual({ token: "t1", spec: { name: first, nth: 3 }, mode: "kill" }); + expect(parseCrashPointControl(`t1 ${first}:2 fail\n`)).toEqual({ token: "t1", spec: { name: first, nth: 2 }, mode: "fail" }); + expect(parseCrashPointControl(`t1 ${first}:2 fail pid=42`)).toEqual({ token: "t1", spec: { name: first, nth: 2 }, mode: "fail", pid: 42 }); + expect(parseCrashPointControl(`t1 ${first} pid=42`)).toEqual({ token: "t1", spec: { name: first, nth: 1 }, mode: "kill", pid: 42 }); + expect(parseCrashPointControl(`t1 ${first} fail fail`)).toBeNull(); + expect(parseCrashPointControl(`t1 ${first}:2 explode`)).toBeNull(); + expect(parseCrashPointControl(`t1 ${first}:2 fail extra`)).toBeNull(); expect(parseCrashPointControl("")).toBeNull(); expect(parseCrashPointControl(first)).toBeNull(); expect(parseCrashPointControl("t1 sandbox.create.after_typo")).toBeNull(); @@ -185,6 +191,35 @@ describe("crashPoint armed by the environment", { timeout: 60_000 }, () => { } }); + it("fails the nth hit once in fail mode, with the call site's own error, and never kills", () => { + const dir = mkdtempSync(path.join(os.tmpdir(), "crash-point-fail-")); + try { + const control = path.join(dir, "control"); + const receipt = path.join(dir, "receipt.jsonl"); + const hit = (label: string) => `try { crashPoint(${JSON.stringify(first)}, { fail: () => new Error("lock held") }); process.stdout.write("${label}-ok\\n"); }` + + ` catch (error) { process.stdout.write("${label}-" + error.message + "\\n"); }`; + const source = [ + `const { crashPoint } = await import(${JSON.stringify(path.join(import.meta.dirname, "crash-points.ts"))});`, + `const { writeFileSync } = await import("node:fs");`, + `writeFileSync(${JSON.stringify(control)}, "t1 ${first}:2 fail\\n");`, + hit("one"), hit("two"), hit("three"), + `try { crashPoint(${JSON.stringify(second)}); crashPoint(${JSON.stringify(first)}); process.stdout.write("plain-ok\\n"); } catch (error) { process.stdout.write("plain-" + error.name + "\\n"); }`, + `writeFileSync(${JSON.stringify(control)}, "t3 ${first} fail pid=1\\n");`, + `try { crashPoint(${JSON.stringify(first)}); process.stdout.write("other-pid-ok\\n"); } catch (error) { process.stdout.write("other-pid-" + error.name + "\\n"); }`, + `writeFileSync(${JSON.stringify(control)}, "t2 ${first} fail pid=" + process.pid + "\\n");`, + `try { crashPoint(${JSON.stringify(first)}); } catch (error) { process.stdout.write("default-" + error.name + "\\n"); }`, + ].join("\n"); + const result = runChild({ PAPERCLIP_CRASH_POINT_CONTROL: control, PAPERCLIP_CRASH_POINT_RECEIPT: receipt }, source); + expect(result.signal).toBeNull(); + expect(result.stdout).toBe("one-ok\ntwo-lock held\nthree-ok\nplain-ok\nother-pid-ok\ndefault-CrashPointInjectedFailure\n"); + const receipts = readFileSync(receipt, "utf8").trim().split("\n").map(line => JSON.parse(line)); + expect(receipts).toEqual([expect.objectContaining({ point: first, hit: 2, token: "t1", mode: "fail" }), + expect.objectContaining({ point: first, hit: 1, token: "t2", mode: "fail" })]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("stays disarmed in production", () => { const result = runChild({ PAPERCLIP_CRASH_POINT: first, NODE_ENV: "production" }); expect(result.signal).toBeNull(); diff --git a/packages/shared/src/crash-points.ts b/packages/shared/src/crash-points.ts index d9f31d95c78..c15259f3961 100644 --- a/packages/shared/src/crash-points.ts +++ b/packages/shared/src/crash-points.ts @@ -16,10 +16,17 @@ * * Runtime arming (one booted server, many points), instead of PAPERCLIP_CRASH_POINT: * - * PAPERCLIP_CRASH_POINT_CONTROL= each hit reads " [:]" from the file; + * PAPERCLIP_CRASH_POINT_CONTROL= each hit reads " [:] [fail]" from the file; * no file or an empty one means disarmed. Hits count per * token, so writing a new token re-arms with a fresh count. * + * Fail mode (control file only): a trailing `fail` makes the nth hit throw instead of + * killing the process, once. An optional `pid=` binds a line to one process, so a + * server started earlier (still draining) never counts its hits. The call site may name the error its real step throws + * (`crashPoint(name, { fail: () => conflict("...") })`); otherwise it throws + * `CrashPointInjectedFailure`. The restart-chaos harness uses it to fail the Nth + * attempt of a recovery step (`--fail-recovery-step`), so recovery chains are exercised. + * * The variable is read once, when this module is first imported. It is ignored * when `NODE_ENV=production`, and an unknown name throws at import so a typo * cannot silently arm nothing. Disarmed, `crashPoint` is a single comparison @@ -47,6 +54,10 @@ export const CRASH_POINT_CONTROL_ENV = "PAPERCLIP_CRASH_POINT_CONTROL"; export const CRASH_POINT_LOG_PREFIX = "[crash-point]"; export type CrashPointSpec = { name: CrashPointName; nth: number }; +/** `kill` (default) SIGKILLs the process at the hit; `fail` throws once (control file only). */ +export type CrashPointMode = "kill" | "fail"; +/** What the call site's real step throws when it fails; used in fail mode. */ +export type CrashPointOptions = { fail?: () => Error }; export type CrashPointReceipt = { point: CrashPointName; @@ -55,8 +66,18 @@ export type CrashPointReceipt = { at: string; /** Runtime arming only: the control file token the hit counted under. */ token?: string; + /** Present for fail mode: the hit threw instead of killing the process. */ + mode?: "fail"; }; +/** Thrown at a point armed in fail mode whose call site names no error of its own. */ +export class CrashPointInjectedFailure extends Error { + constructor(readonly point: CrashPointName, readonly hit: number) { + super(`injected failure at ${point} (hit ${hit})`); + this.name = "CrashPointInjectedFailure"; + } +} + /** Thrown by a point armed with `armCrashPointForTest`. */ export class CrashPointReached extends Error { constructor(readonly point: CrashPointName, readonly hit: number) { @@ -84,7 +105,7 @@ export function parseCrashPointSpec(raw: string | undefined): CrashPointSpec | n type Armed = { spec: CrashPointSpec; hits: number; - mode: "kill" | "throw"; + mode: "kill" | "throw" | "fail"; receiptPath: string | null; /** Hits count only once this file exists; null when ungated. */ gatePath: string | null; @@ -93,13 +114,21 @@ type Armed = { token?: string | null; }; -/** Parses a control file line, " [:]"; null when empty or malformed. */ -export function parseCrashPointControl(text: string): { token: string; spec: CrashPointSpec } | null { +/** Parses a control file line, " [:] [fail] [pid=]"; null when empty or malformed. */ +export function parseCrashPointControl(text: string): { token: string; spec: CrashPointSpec; mode: CrashPointMode; pid?: number } | null { const [token, raw, ...rest] = text.trim().split(/\s+/); - if (!token || !raw || rest.length > 0) return null; + if (!token || !raw || rest.length > 2) return null; + let mode: CrashPointMode = "kill"; + let pid: number | undefined; + for (const option of rest) { + const match = /^pid=([1-9][0-9]{0,9})$/.exec(option); + if (option === "fail" && mode === "kill") mode = "fail"; + else if (match && pid === undefined) pid = Number(match[1]); + else return null; + } try { const spec = parseCrashPointSpec(raw); - return spec ? { token, spec } : null; + return spec ? { token, spec, mode, ...(pid !== undefined ? { pid } : {}) } : null; } catch { return null; } @@ -132,13 +161,13 @@ let armed: Armed | null = armFromEnvironment(); * immediately. Armed for this name, the nth hit kills the process with SIGKILL * (or throws `CrashPointReached` under `armCrashPointForTest`). */ -export function crashPoint(name: CrashPointName): void { +export function crashPoint(name: CrashPointName, options?: CrashPointOptions): void { if (armed === null) return; if (armed.controlPath) { if (!readControl(armed)) return; } if (armed.spec.name !== name) return; - reach(armed); + reach(armed, options); } /** Runtime arming: loads the control file's spec; a new token resets the count. */ @@ -147,15 +176,18 @@ function readControl(current: Armed): boolean { try { text = readFileSync(current.controlPath!, "utf8"); } catch { return false; } const control = parseCrashPointControl(text); if (!control) return false; + // A line bound to another process is disarmed here. + if (control.pid !== undefined && control.pid !== process.pid) return false; if (control.token !== current.token) { current.token = control.token; current.spec = control.spec; + current.mode = control.mode; current.hits = 0; } return true; } -function reach(current: Armed): void { +function reach(current: Armed, options?: CrashPointOptions): void { if (current.gatePath !== null) { if (!existsSync(current.gatePath)) return; current.gatePath = null; @@ -166,10 +198,15 @@ function reach(current: Armed): void { const receipt: CrashPointReceipt = { point: current.spec.name, hit: current.hits, pid: process.pid, at: new Date().toISOString(), ...(current.token ? { token: current.token } : {}), + ...(current.mode === "fail" ? { mode: "fail" as const } : {}), }; if (current.receiptPath) { try { appendFileSync(current.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line below still records it */ } } + if (current.mode === "fail") { + try { process.stderr.write(`${CRASH_POINT_LOG_PREFIX} FAIL ${JSON.stringify(receipt)}\n`); } catch { /* ignore */ } + throw options?.fail?.() ?? new CrashPointInjectedFailure(current.spec.name, current.hits); + } try { process.stderr.write(`${CRASH_POINT_LOG_PREFIX} SIGKILL ${JSON.stringify(receipt)}\n`); } catch { /* ignore */ } process.kill(process.pid, "SIGKILL"); } diff --git a/server/src/__tests__/crash-point-call-sites.test.ts b/server/src/__tests__/crash-point-call-sites.test.ts index ea26342286b..60d619f4d75 100644 --- a/server/src/__tests__/crash-point-call-sites.test.ts +++ b/server/src/__tests__/crash-point-call-sites.test.ts @@ -6,7 +6,8 @@ import { OWNABLE_KINDS } from "../services/recovery/ownership-registry.js"; const repositoryRoot = path.resolve(import.meta.dirname, "../../.."); const SOURCE_ROOTS = ["server/src", "packages/paperclip-runner/src"]; -const CALL = /\bcrashPoint\(\s*"([^"]+)"\s*\)/g; +// A call may pass fail-mode options after the name: crashPoint("", { fail: ... }). +const CALL = /\bcrashPoint\(\s*"([^"]+)"\s*[,)]/g; function sourceFiles(directory: string): string[] { return readdirSync(directory, { withFileTypes: true }).flatMap(entry => { diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index b1c8de72e5d..3e277345440 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -20,6 +20,7 @@ import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "./re import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js"; import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; import { logger } from "../middleware/logger.js"; +import { crashPoint } from "@paperclipai/shared/crash-points"; type Copy = typeof copies.$inferSelect; function adoptionAuthority(row: Copy) { @@ -654,6 +655,8 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string if (!disposableRemoteCopy(row)) return run(); // Different controllers have different cache roots. Serialize their active // collection/cleanup operations in Postgres, without holding row locks over I/O. + // Test instrumentation only (disarmed in production): restart chaos fails the Nth attempt with the lock's own conflict. + crashPoint("agent_directory.operation_lock.before_acquire", { fail: () => conflict("Agent directory operation is already active") }); return withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => { const [lock] = await tx.execute(sql`select pg_try_advisory_xact_lock(hashtextextended(${`agent-directory:${row.companyId}:${row.agentId}`}, 0)) as acquired, (select setting::bigint from pg_settings where name = 'idle_in_transaction_session_timeout') as idle_timeout_ms`); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 8252eebfb1f..8c84a559b04 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -3853,10 +3853,11 @@ export function environmentRuntimeService( )`, )).returning(), { companyId: input.leaseRow.companyId, runId: input.leaseRow.heartbeatRunId ?? undefined }); if (!claimed) return null; - crashPoint("environment.teardown.after_claim_before_destroy"); const lease = toEnvironmentLeaseSnapshot(claimed); const renewal = renewCleanupClaim(lease.id, attemptId, "scoped cleanup ownership renewal failed"); try { + // Kill mode leaves the claim in flight; fail mode (restart chaos only) is a provider destroy that did not finish. + crashPoint("environment.teardown.after_claim_before_destroy", { fail: () => new Error("sandbox teardown incomplete: provider destroy did not finish") }); const driver = getDriver(getLeaseDriverKey(lease, input.environment)); if (!driver?.destroyRunLease) return lease; const released = await driver.destroyRunLease({ diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c46e7976c96..8cf6139f013 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11913,7 +11913,9 @@ async function executePaperclipNativeSessionAttempt( // the work-based budget (provider-attempt-outcome.ts). payload: { hostname: hostname(), providerAttempt: attempt, requestedAt: new Date().toISOString() }, }); - crashPoint("native.process_start.after_requested_before_spawn"); + // Fail mode (restart chaos only) is a runnerd start that failed before the process existed. + crashPoint("native.process_start.after_requested_before_spawn", { fail: () => Object.assign( + new Error("spawn paperclip-runnerd EAGAIN"), { code: "EAGAIN", syscall: "spawn paperclip-runnerd" }) }); }, input: runnerExecution, startup: executionContext && (!usesPluginTask || executionContext.pluginSupported) ? executionContext : undefined, diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index e96feb427fd..ee87afb64aa 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -22,9 +22,11 @@ node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck ``` Options: `--triggers time,comment,retry,queued,tool,result` (fault timing, below), -`--workloads A1,B1,B5,C1` (opt-in: G1, H1, L1, R1, S1), `--db-pressure ` (below), `--budget-cents` (company hard cap, +`--workloads A1,B1,B5,C1` (opt-in: G1, H1, L1, R1, S1, W2, W2Q, W2C), `--db-pressure ` (below), `--budget-cents` (company hard cap, default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance), `--k4-hold-max-seconds` (raise the seeded K4 hold upper end from 300 s, up to the 600 s outage cap). +`--faults K0` plans cells with no fault, for workloads whose shape is the fault (W2, below). `--fail-recovery-step`, +`--long-turn-minutes`, `--k15-overlap-seconds` and `--k15-fresh-home` are described in their own sections. `--fault-after-seconds ` fires every planned fault `n` seconds after its task launches (5–3600; with `--triggers time`), keeping the rest of the seeded plan, for example to restart a Daytona R1 task after its workspace preparation. `--matrix` crosses every selected workload with every fault kind (below), and `--server-from ` @@ -389,6 +391,35 @@ minutes. In staging class R, recovery then found the remote Runner gone `deadlineExceeded`, and the restart continues. A probe that answers after the deadline writes nothing, because its evidence is from after the outage window. +### Redeploy without a volume, with overlap (`--k15-fresh-home`, `--k15-overlap-seconds`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 --storage s3 \ + --adapters codex,opencode --workloads A1,C1 --faults K15 --k15-fresh-home --k15-overlap-seconds 30-150 --concurrency 3 --tasks 6 --seed 9320 +``` + +Staging's deploys after 07:25Z on 2026-11-11 reported `volumeId: null`, and the old container kept running +for a while after the new one was up. Plain K15 restarts on the same instance home and has no overlap. + +- **`--k15-fresh-home`** (Daytona only): every deploy generation gets a new, empty instance home + (`home-g` in the wave's temporary root) and no `RAILWAY_VOLUME_*` variables. Only what a hosted + deploy gets from outside its container is carried over: the configuration (database, storage) and the + secrets master key. Controller-local files such as `runtime-context-assets` bundles and runtime caches + are gone, as in staging class A-deploy (`ENOENT … runtime-context-assets/bundles/…`). Use + `--storage s3`, as staging does. A local run keeps its workspace and runner under the instance home, so + the option refuses `--env local`. +- **`--k15-overlap-seconds ` or `-`** (5–600 s; staging's shape is 30–150): the wave's port + becomes an edge proxy (`edge-proxy.ts`), and each server listens on its own port behind it. A K15 boots + the next generation beside the old server, sends new connections to it once its `/api/ready` reports + `ready`, keeps the old server running for the seeded overlap, then stops the old one with SIGTERM. There + is no outage, so the seeded K15 hold is the overlap; it does not combine with `--k15-hold-seconds`. + The timeline records `k15_fresh_home`, `k15_traffic_switched` and `fault_server_down` (the old server's + stop). + +Pass is the usual grade: no Retry (R3), no stuck record (R6), and the oracle. Staging's terminal errors here +were `ENOENT` on a runtime asset, `Agent directory operation is already active` after a 9.5-minute +`controller_lease_held` wait. + ## Absolute symlink in the workspace (F1) ```sh @@ -957,6 +988,102 @@ plus `predecessor_unretirable_fell_back` and `follow_up_fresh_sandbox` (the succ on a different sandbox than the asking run). SR1U needs `--env daytona --daytona-warm`, and it arms its fault for the whole server, so it does not share a wave with SR1. +## Re-wake after success (W2, W2Q, W2C) + +```sh +# Ten tasks on one Codex agent with warm sandboxes; each task is woken again 0–60 s after its run succeeded. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 3 \ + --adapters codex --workloads W2,W2Q,W2C --faults K0 --concurrency 10 --tasks 10 --seed 5201 +# The same locally (warm local Runners). +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --lifecycle warm \ + --adapters codex --workloads W2,W2Q,W2C --faults K0 --concurrency 10 --tasks 10 --seed 5101 +``` + +In one 8-hour staging window (2026-11-11), 20 of 25 failures happened with no restart at all. One agent had +10–13 runs at once on reused sandboxes, and 30–60 s after a 20–43 minute run of a task succeeded, an +ordinary wake (`issue_commented`, `issue_monitor_due`) started the next run of that task. That run had to +retire the previous run's warm owner first, and failed before any provider work: `Controller journal drain +timed out`, `runner did not durably suspend before checkpoint` (class C+), or +`journal_tool_input_unanswered` after a predecessor that ended on an unanswered question (class C3). + +The workload is the fault, so these cells usually run with `--faults K0` (no fault). Every task of an +adapter runs on that adapter's one agent: `--adapters codex --tasks 10 --concurrency 10` is ten tasks on one +agent. Phase 1 ends with the task left `in_progress` (`in_review` would gate completion on a review). Then the harness, as the board, wakes the same task the +seeded 0–60 s after that run settled (`rewake.ts`): + +- **W2**: a board comment, or, by the task's nonce, a board-scheduled issue monitor due now, so the + scheduler's own `issue_monitor_due` wake starts the run. +- **W2Q**: phase 1 asks one `request_human_input` question and parks. A board comment (a note to include, + not the answer) wakes the task while the question is pending. The board answers after that run settled. +- **W2C**: as W2, and first the board changes the agent's `adapterConfig.idleTimeoutMs`, so the next run + cannot resume the saved session. SR1 (`session-faults.ts`, #20844) is the same change while a question + is parked; W2C is the change after a run succeeded. + +Until the wake is served (and, for W2Q, answered), the task waits on the board: not a rest, no Retry. The +oracle checks that phase 1 succeeded, the wake was sent and a run created after it succeeded, +`no_failed_attempt` (no run of the task failed or timed out), the task is `done`, and the brief's markers. +A task left blocked gets the usual Retry (R3), which fails the grade. A restart fault (for example K2) +may also join; it lands around the first run or the wake. + +Add `--long-turn-minutes 20` for staging's long predecessor turns. + +## Fail the Nth recovery step (`--fail-recovery-step`) + +```sh +# Local: K1, then the replacement's first runnerd start fails once (spawn EAGAIN). +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local --adapters codex --workloads A1 \ + --faults K1 --fail-recovery-step runnerd-start:1 --fault-after-seconds 10 --concurrency 1 --tasks 1 --seed 5303 +# Daytona: K1, then the replacement's first sandbox teardown does not finish. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-cpu 3 \ + --adapters codex --workloads A1 --faults K1 --fail-recovery-step teardown:1 --fault-after-seconds 30 --concurrency 1 --tasks 1 --seed 5312 +``` + +Every other fault hits the control plane once and lets recovery run. On staging the first fault was often +harmless and a later step failed: a bootstrap retry met a missing runtime asset, a reattach met +`Agent directory operation is already active`, a bounded retry met an unfinished teardown. + +`--fail-recovery-step [:]` lets the wave's first restart fault (K1, K2, K4, K6, K14, K15 or +RR1–RR3) fire normally. When that fault's replacement server starts, the harness arms one crash point in +fail mode: the nth hit of the point in the replacement throws, once, the error its real step throws. The +control line is bound to the replacement's pid (`pid=`), so an old server still draining during a K15 +overlap never counts hits, and the harness disarms it once the receipt lands, so a later restart (RR, K6, +another task's fault) does not fail the step again. `nth` +1 fails recovery's first attempt at the step, 2 its second, 3 its third. Steps: + +| Step | Crash point | Error thrown | Environments | +| --- | --- | --- | --- | +| `runnerd-start` | `native.process_start.after_requested_before_spawn` | `spawn paperclip-runnerd EAGAIN` | local, Daytona | +| `teardown` | `environment.teardown.after_claim_before_destroy` | `sandbox teardown incomplete: provider destroy did not finish` | Daytona | +| `directory-lock` | `agent_directory.operation_lock.before_acquire` | `Agent directory operation is already active` (409) | Daytona | + +Any other registered crash point name also works; without its own error it throws +`CrashPointInjectedFailure`. Hits count across the server, so with several tasks in flight the nth hit can +belong to any of them: run one task per wave to aim at one task's nth attempt. A step that recovery calls +only once is never reached with `nth` 2 or more, and the task fails `recovery_step_failed_once`. On +2026-11-11 an A1 task with K2 on Daytona never reached `directory-lock`: pick a workload whose recovery +collects remote agent files. + +Every task the armed fault crossed gets two more checks: `recovery_step_failed_once` (the receipt in +`recovery-step-receipts.jsonl`; a step never reached tested nothing) and `no_failed_attempt`. The fault's +receipt in `faults.json` names the armed step (`recoveryStep`). + +## Long turn (`--long-turn-minutes`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --daytona-cpu 3 \ + --adapters codex --workloads W2 --faults K0 --long-turn-minutes 20 --concurrency 4 --tasks 4 --seed 5401 +``` + +Staging's failures followed turns of 14–43 minutes; every other workload finishes in a few. With +`--long-turn-minutes ` (15–60, opt-in), every task's first turn first runs n minutes of real tool work: +rounds of one foreground shell command each (about 90 s of hashing and sleeping, short of provider +shell-tool timeouts) that append `ROUND ` to a log in the working directory. The agent +saves the log as the issue document `long-turn` and then does its workload in the same turn +(`long-turn.ts`). Any workload and fault can take it; the task's time budget grows by n minutes. Two checks +join the oracle: `long_turn_rounds` (every round reported, spanning n minutes) and `long_turn_run_observed` +(the harness saw one run of the task last n minutes). Use `--fault-after-seconds` to land a fault late in +the turn. Waves get long and expensive: keep them small. + ## Database contention (L1, `--db-pressure`) ```sh @@ -1474,6 +1601,10 @@ and grades only (no server logs). 4. Run the point: `node scripts/crash-points.mjs --points `. A `not_reached` result means the workload never crosses it. +**Fail mode.** A control-file line may end in `fail` (` : fail`): the nth hit then +throws once instead of killing the server. An optional `pid=` binds the line to one server process. A call site can name the error its real step throws +(`crashPoint("", { fail: () => conflict("...") })`). `--fail-recovery-step` uses it. + Disarmed, a point costs one comparison (about 4 ns), and the arming variables (`PAPERCLIP_CRASH_POINT`, `PAPERCLIP_CRASH_POINT_CONTROL`) are ignored under `NODE_ENV=production`, so points may sit on the path before `turn.start`. diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 1e532c83885..0dba1ec0ed8 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -5,7 +5,10 @@ import { applyJourneyOptions } from "./journey-matrix.js"; import { takesJourneyTriggers } from "./journeys.js"; import { STORAGE_FAULT_KINDS, type StorageFaultKind } from "./storage-chaos.js"; import { isMultiServerFault, validateMultiServerOptions } from "./multi-server.js"; -import { parseK15HoldSeconds } from "./redeploy.js"; +import { parseK15DeployShape, parseK15HoldSeconds } from "./redeploy.js"; +import { validateRecoveryStep } from "./recovery-step-faults.js"; +import { parseLongTurnMinutes } from "./long-turn.js"; +import { isRewakeWorkload } from "./rewake.js"; import { parseTimeFaultOptions } from "./time-faults-config.js"; import { parseStorageTrigger } from "./storage-trigger.js"; import { parseNativeTrigger, parseNativeTriggerOutage } from "./native-trigger.js"; @@ -137,14 +140,18 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "rr-gap-seconds": { type: "string" }, "rr-stop-on": { type: "string" }, "k15-hold-seconds": { type: "string" }, + "k15-overlap-seconds": { type: "string" }, + "k15-fresh-home": { type: "boolean" }, "tm1-hold-seconds": { type: "string" }, + "fail-recovery-step": { type: "string" }, + "long-turn-minutes": { type: "string" }, }, }); const keep = values.keep === true; const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-pack-mismatch", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "tm1-hold-seconds"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-pack-mismatch", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "k15-overlap-seconds", "k15-fresh-home", "tm1-hold-seconds", "fail-recovery-step", "long-turn-minutes"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -186,23 +193,25 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (lifecycle !== undefined && lifecycle !== "per_turn" && lifecycle !== "warm" && lifecycle !== "default") throw new Error("--lifecycle must be per_turn, warm or default"); const adapters = list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]); // `--faults` takes restart faults (K*), provider message faults (PV*, provider-faults.ts), the - // sandbox disk-full fault (DF1, disk-full-faults.ts) and session faults (SR*, session-faults.ts). - const allFaults = list(values.faults, [...FAULT_KINDS, ...PROVIDER_FAULT_KINDS, ...DISK_FULL_FAULT_KINDS, ...SESSION_FAULT_KINDS], "faults", + // sandbox disk-full fault (DF1, disk-full-faults.ts), session faults (SR*, session-faults.ts) and K0, cells with no fault. + const allFaults = list(values.faults, [...FAULT_KINDS, ...PROVIDER_FAULT_KINDS, ...DISK_FULL_FAULT_KINDS, ...SESSION_FAULT_KINDS, "K0"], "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]); const providerFaults = allFaults.filter(isProviderFault); const diskFullFaults = allFaults.filter(isDiskFullFault); const sessionFaults = allFaults.filter(isSessionFault); + const baseline = allFaults.includes("K0"); const options: WaveOptions = { env, adapters, // The governance scenarios (G1, H1), the journeys (Q1–Q4) and the contention run (L1) are opt-in: H1 needs a wave of its own. workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1" && id !== "L1" && id !== "F1" && id !== "T1" && id !== "Q4" - && !takesJourneyTriggers(id) && (id !== "M1" || adapters.length >= 2))), + && !takesJourneyTriggers(id) && !isRewakeWorkload(id) && (id !== "M1" || adapters.length >= 2))), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. - faults: allFaults.filter((fault): fault is FaultKind => !isProviderFault(fault) && !isDiskFullFault(fault) && !isSessionFault(fault)), + faults: allFaults.filter((fault): fault is FaultKind => !isProviderFault(fault) && !isDiskFullFault(fault) && !isSessionFault(fault) && fault !== "K0"), ...(providerFaults.length ? { providerFaults } : {}), ...(diskFullFaults.length ? { diskFullFaults } : {}), ...(sessionFaults.length ? { sessionFaults } : {}), + ...(baseline ? { baseline: true } : {}), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), tasks: integer(values.tasks, "tasks", stormMinutes ? 200 : 8, 1, 200), @@ -239,6 +248,11 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC // The matrix planner builds its cells from restart and provider faults only; DF1 would silently drop out. if (values.matrix) throw new Error("The disk-full fault (DF1) is not a --matrix cell; run it as a planned wave"); } + if (baseline && (stormMinutes || crashPoint)) throw new Error("--faults K0 plans cells with no fault; a storm or crash-point wave has its own faults"); + Object.assign(options, validateRecoveryStep(options, values["fail-recovery-step"])); + Object.assign(options, parseLongTurnMinutes(values["long-turn-minutes"])); + // The matrix planner builds its own task plans; a long turn would silently drop out of them. + if (options.longTurnMinutes && values.matrix) throw new Error("--long-turn-minutes is not a --matrix option; run it as a planned wave"); if (providerFaults.length) { if (stormMinutes) throw new Error("Provider message faults (PV*) are planned per task; do not combine them with --storm-minutes"); if (env !== "local") throw new Error("Provider message faults (PV*) shim the local provider; run them with --env local"); @@ -294,6 +308,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC Object.assign(options, parseTimeFaultOptions(options, values["time-age-hours"], values["tm1-hold-seconds"])); Object.assign(options, parseRepeatedRestartOptions(options, values["rr-gap-seconds"], values["rr-stop-on"])); Object.assign(options, parseK15HoldSeconds(options, values["k15-hold-seconds"])); + Object.assign(options, parseK15DeployShape(options, values["k15-overlap-seconds"], values["k15-fresh-home"] === true)); const upgradeFrom = values["upgrade-from"]; if (options.faults.includes("K14") !== (upgradeFrom !== undefined)) { throw new Error("K14 (upgrade mid-run) and --upgrade-from go together: pass both or neither"); diff --git a/tests/restart-chaos/edge-proxy.ts b/tests/restart-chaos/edge-proxy.ts new file mode 100644 index 00000000000..b9f2700f102 --- /dev/null +++ b/tests/restart-chaos/edge-proxy.ts @@ -0,0 +1,38 @@ +/** + * `--k15-overlap-seconds`: the wave's stable port in front of the serving deploy generation, as the hosting + * platform's edge is in front of its containers. Each server of such a wave listens on its own loopback port; + * the proxy forwards new connections to the current one. A K15 switches the target once the next generation is + * ready, while the old server keeps its open connections until it is stopped, as a rolling deploy does. + */ +import net from "node:net"; + +export interface EdgeProxy { + readonly port: number; + /** The backend port new connections go to. */ + target: number; + close(): Promise; +} + +export async function startEdgeProxy(port: number, target: number): Promise { + const sockets = new Set(); + const proxy: EdgeProxy = { port, target, close: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise(resolve => server.close(() => resolve())); + } }; + const server = net.createServer(client => { + const upstream = net.connect(proxy.target, "127.0.0.1"); + for (const socket of [client, upstream]) { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + // A backend that is gone or restarting refuses the connection: the client sees a reset, as at the edge. + socket.on("error", () => { client.destroy(); upstream.destroy(); }); + } + client.pipe(upstream); + upstream.pipe(client); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(port, "127.0.0.1", () => { server.off("error", reject); resolve(); }); + }); + return proxy; +} diff --git a/tests/restart-chaos/journeys.ts b/tests/restart-chaos/journeys.ts index 67a54a9fb72..7d49bc0494e 100644 --- a/tests/restart-chaos/journeys.ts +++ b/tests/restart-chaos/journeys.ts @@ -36,6 +36,7 @@ export type JourneyTrigger = (typeof JOURNEY_TRIGGERS)[number]; export const WORKLOAD_TRIGGERS: Record = { A1: ["time"], B1: ["time"], B5: ["tool"], C1: ["comment", "result"], M1: ["time"], G1: ["approval"], H1: ["paused"], R1: ["time"], S1: ["stop"], L1: ["time"], F1: ["time"], T1: ["time"], Q1: JOURNEY_TRIGGERS, Q2: JOURNEY_TRIGGERS, Q3: JOURNEY_TRIGGERS, Q4: ["time"], + W2: ["time"], W2Q: ["time"], W2C: ["time"], }; /** Q3 reviews must survive a delay longer than the implicit deadline #20340 removed. */ diff --git a/tests/restart-chaos/long-turn.ts b/tests/restart-chaos/long-turn.ts new file mode 100644 index 00000000000..8760db54d64 --- /dev/null +++ b/tests/restart-chaos/long-turn.ts @@ -0,0 +1,100 @@ +/** + * `--long-turn-minutes ` (15–60): every task's first turn does n minutes of real tool work before its + * workload, for any cell (README "Long turn"). + * + * The staging failures of 2026-11-11 followed turns of 14–43 minutes, while every chaos workload finishes in + * a few minutes. A long turn grows what the run holds when its fault or next wake arrives: the provider + * session's history, the controller journal, tool receipts and the sandbox's working files. + * + * The agent runs one shell command per round, in the foreground, each about 90 s (well short of provider + * shell-tool timeouts). Each round appends `ROUND ` to a log in its working directory and + * hashes data as it goes. Then it saves the log as the issue document `long-turn` and carries on with the + * workload. The oracle checks the rounds the agent reported and, from the harness's own observations, that + * one run of the task ran at least n minutes. + */ +import type { OracleArtifacts, OracleResult } from "./types.js"; +import type { WorkloadDefinition } from "./workloads.js"; + +export const LONG_TURN_MINUTES_RANGE: readonly [number, number] = [15, 60]; +/** Seconds of one round (45 ticks of 2 s). */ +export const LONG_TURN_ROUND_SECONDS = 90; +export const LONG_TURN_DOCUMENT = "long-turn"; +/** Sandbox and host clocks differ, and a round may start a little late: slack on the reported span. */ +const SPAN_SLACK_SECONDS = 60; + +export function parseLongTurnMinutes(raw: string | undefined): { longTurnMinutes?: number } { + if (raw === undefined) return {}; + if (!/^[0-9]+$/.test(raw)) throw new Error("--long-turn-minutes takes whole minutes"); + const minutes = Number(raw); + const [min, max] = LONG_TURN_MINUTES_RANGE; + if (minutes < min || minutes > max) throw new Error(`--long-turn-minutes must be between ${min} and ${max}`); + return { longTurnMinutes: minutes }; +} + +export function longTurnRounds(minutes: number): number { + return Math.ceil((minutes * 60) / LONG_TURN_ROUND_SECONDS); +} + +export const longTurnLog = (nonce: string) => `long-turn-${nonce}.log`; + +/** One round: ticks of real work (hashing) with a sleep, then one log line with the round's own clock window. */ +export function longTurnCommand(nonce: string, round: number): string { + const ticks = LONG_TURN_ROUND_SECONDS / 2; + return `s=$(date +%s); i=0; h=${nonce}; while [ $i -lt ${ticks} ]; do i=$((i+1)); h=$(printf '%s %s' "$h" "$i" | cksum | cut -d' ' -f1); sleep 2; done;` + + ` echo "ROUND ${round} $s $(date +%s) $h" >> ${longTurnLog(nonce)}; tail -n 1 ${longTurnLog(nonce)}`; +} + +export function longTurnPreamble(nonce: string, minutes: number): string { + const rounds = longTurnRounds(minutes); + return [ + `Before anything else in this task, do about ${minutes} minutes of preparation in your working directory, in this same turn.`, + `Run this shell command ${rounds} times, once per round, as separate tool calls in the foreground, replacing with the round number 1 to ${rounds}`, + `(each takes about ${LONG_TURN_ROUND_SECONDS} seconds; do not run rounds in the background or in parallel, and do not skip or shorten any):`, + `\`${longTurnCommand(nonce, 0).replace("ROUND 0 ", "ROUND ")}\``, + `After the last round, save the full contents of \`${longTurnLog(nonce)}\` as this issue's document with key \`${LONG_TURN_DOCUMENT}\``, + "(title \"Long turn\"), exactly as the file reads. Then, still in this turn, do the task below.", + "", + ].join("\n"); +} + +/** The rounds the agent reported, in order. */ +export function parseLongTurnLog(text: string | null | undefined): Array<{ round: number; start: number; end: number }> { + return (text ?? "").split(/\r?\n/).flatMap(line => { + const match = /^\s*ROUND ([0-9]+) ([0-9]{9,11}) ([0-9]{9,11})(?: \S+)?\s*$/.exec(line); + return match ? [{ round: Number(match[1]), start: Number(match[2]), end: Number(match[3]) }] : []; + }); +} + +export function longTurnChecks(minutes: number, artifacts: Pick): OracleResult["checks"] { + const rounds = parseLongTurnLog(artifacts.documents[LONG_TURN_DOCUMENT]); + const wanted = longTurnRounds(minutes); + const complete = rounds.length >= wanted && rounds.slice(0, wanted).every((entry, index) => entry.round === index + 1 + && entry.end - entry.start >= LONG_TURN_ROUND_SECONDS - 10); + const span = rounds.length ? rounds[rounds.length - 1]!.end - rounds[0]!.start : 0; + // The harness's own evidence: a run of the task that ran for the whole turn. + const longest = Math.max(0, ...(artifacts.runs ?? []).map(run => { + const started = Date.parse(run.createdAt ?? ""), finished = Date.parse(run.finishedAt ?? ""); + return Number.isFinite(started) && Number.isFinite(finished) ? (finished - started) / 1000 : 0; + })); + return [ + { name: "long_turn_rounds", ok: complete && span >= minutes * 60 - SPAN_SLACK_SECONDS, + detail: `rounds=${rounds.length}/${wanted} span=${span}s` }, + { name: "long_turn_run_observed", ok: longest >= minutes * 60 - SPAN_SLACK_SECONDS, detail: `longest run ${Math.round(longest)}s` }, + ]; +} + +/** The workload with the long turn in front: description, budget, collected documents and oracle. */ +export function withLongTurn(definition: WorkloadDefinition, minutes: number | undefined): WorkloadDefinition { + if (!minutes) return definition; + return { + ...definition, + budgetMs: definition.budgetMs + minutes * 60_000, + description: (nonce, agents) => `${longTurnPreamble(nonce, minutes)}${definition.description(nonce, agents)}`, + collect: { ...definition.collect, documents: [...new Set([...definition.collect.documents, LONG_TURN_DOCUMENT])] }, + grade(nonce, artifacts, agents) { + const base = definition.grade(nonce, artifacts, agents); + const checks = [...base.checks, ...longTurnChecks(minutes, artifacts)]; + return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; + }, + }; +} diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 8312373d9f6..b34a3a4b0b9 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -41,6 +41,7 @@ import { closeJourneyApps } from "./journey-driver.js"; import { matrixMayLaunch, planMatrixWave } from "./journey-matrix.js"; import { redeployWave } from "./redeploy.js"; import { withT1CrossingCheck } from "./workloads.js"; +import { parseRecoveryStepReceipts, withRecoveryStepCheck } from "./recovery-step-faults.js"; import { captureWaveRunnerLogs } from "./daytona-runner-logs.js"; import { StormRun, TASK_FAULT_LOG, observedFaultSchedule, planWave, replayOnCurrentBuild, replayRecordFor, replayWave, type TaskFaultLog } from "./scheduler.js"; import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js"; @@ -244,6 +245,14 @@ async function main(): Promise { timeline.add("db_pressure_stopped", { ...stats }); }; const violationKey = (violation: { rule: string; kind: string; recordId: string }) => `${violation.rule}:${violation.kind}:${violation.recordId}`; + // `--k15-overlap-seconds`, `--k15-fresh-home`: the staging deploy shape (redeploy.ts). + supervisor.deployShape = { overlap: plan.options.k15Overlap === true, freshHome: plan.options.k15FreshHome === true }; + // `--fail-recovery-step`: fail the nth hit of one recovery step once, after the wave's first restart fault. + const recoveryStepReceipts = path.join(waveDir, "recovery-step-receipts.jsonl"); + if (plan.options.failRecoveryStep) { + supervisor.armRecoveryStepOnFirstFault(plan.options.failRecoveryStep, { + controlPath: path.join(supervisor.paths.temporaryRoot, "crash-point.control"), receiptPath: recoveryStepReceipts }); + } if (crashPoint) { supervisor.crashPointHoldMs = plan.options.crashPointHoldMs ?? 0; supervisor.useCrashPointControl({ controlPath: path.join(supervisor.paths.temporaryRoot, "crash-point.control"), @@ -704,8 +713,12 @@ async function main(): Promise { } // T1: a mid-tool fault only counts when one of the task's faults started while its loop ran (workloads.ts). // DF1 restarts nothing, so T1's mid-tool restart check does not apply; the disk-full check replaces it. - const toolOracle = isDiskFullFault(observer.plan.fault) ? withDiskFullCheck(oracle, observer.plan, observer.record) + const t1Oracle = isDiskFullFault(observer.plan.fault) ? withDiskFullCheck(oracle, observer.plan, observer.record) : withT1CrossingCheck(oracle, observer.plan, artifacts?.documents.brief, faults); + // `--fail-recovery-step`: the step failed in the replacement, and no attempt of the task failed (recovery-step-faults.ts). + const armedStep = faults.find(receipt => receipt.recoveryStep); + const toolOracle = withRecoveryStepCheck(t1Oracle, armedStep?.recoveryStep ? { faultId: armedStep.faultId, ...armedStep.recoveryStep } : null, + observer.record.faultIds, parseRecoveryStepReceipts(await readFile(recoveryStepReceipts, "utf8").catch(() => "")), artifacts?.runs ?? []); const grade = gradeTask(observer.record, faults, observer.definition.budgetMs, toolOracle, universalByIssue ? { runMessages: allRunMessages, recoveryActions: universalByIssue.get(observer.record.issueId ?? "")?.recoveryActions ?? [] } : undefined); let evidence: string | null = null; diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 06fba901307..c86614d119b 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -17,6 +17,8 @@ import type { UiRetryClick } from "./board-ui.js"; import { runOracleCommand } from "./oracle-sandbox.js"; import { awaitingQuestionAnswer, collectQuestionArtifacts, questionStep } from "./question-handoff.js"; import { isSessionFault, rotateAgentSession } from "./session-faults.js"; +import { awaitingRewake, collectRewakeArtifacts, isRewakeWorkload, rewakeStep } from "./rewake.js"; +import { withLongTurn } from "./long-turn.js"; import { captureRemoteWorkspaceFiles, readRemoteWorkspaceFile } from "./remote-workspace-files.js"; import { taskFaultSchedule } from "./scheduler.js"; import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; @@ -233,7 +235,8 @@ export class TaskObserver { /** Q1–Q3: the board's side of the journey (journey-driver.ts). */ readonly journey: JourneyDriver | null; - get definition() { return workloadDefinitions[this.plan.workload]; } + /** The workload, with `--long-turn-minutes` tool work in front of its first turn (long-turn.ts). */ + get definition() { return withLongTurn(workloadDefinitions[this.plan.workload], this.plan.longTurnMinutes); } /** * Crash-point suite: create the issue without an assignee, so no wake is @@ -820,6 +823,7 @@ export class TaskObserver { if (this.plan.workload === "G1") return Boolean(governance?.approvalSeenAt) && !governance?.decision; if (this.plan.workload === "H1") return Boolean(governance?.budgetPausedAt) && !governance?.resume && !governance?.pauseLostAt; if (this.plan.workload === "Q1" || this.plan.workload === "Q4") return awaitingQuestionAnswer(this.record); + if (isRewakeWorkload(this.plan.workload)) return awaitingRewake(this.record); return false; } @@ -862,6 +866,10 @@ export class TaskObserver { note: (kind, data) => this.timeline.add(kind, { taskId: this.plan.taskId, ...data }), ...(sessionFault ? { beforeAnswer: () => rotateAgentSession(this.api, this.lab.agents[this.plan.adapter].id, this.plan.index, now, sessionFault) } : {}) }); } + if (isRewakeWorkload(this.plan.workload)) { + await rewakeStep({ api: this.api, record: this.record, plan: this.plan, issueId: this.record.issueId, agentId: this.lab.agents[this.plan.adapter].id, + sample, now, boardMayAct: this.boardMayAct(sample), note: (kind, data) => this.timeline.add(kind, { taskId: this.plan.taskId, ...data }) }); + } } private async approvalStep(sample: TaskSample, now: number): Promise { @@ -1039,7 +1047,11 @@ export class TaskObserver { ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), ...(this.record.governance ? { governance: this.record.governance } : {}), - ...(this.plan.workload === "Q1" || this.plan.workload === "Q4" ? { question: await collectQuestionArtifacts(this.api, issueId, this.record) } : {}), + ...(this.plan.workload === "Q1" || this.plan.workload === "Q4" || this.plan.workload === "W2Q" ? { question: await collectQuestionArtifacts(this.api, issueId, this.record) } : {}), + ...(isRewakeWorkload(this.plan.workload) ? { rewake: await collectRewakeArtifacts(this.api, issueId, this.record) } : {}), + // The task's runs, for the shape checks (`no_failed_attempt`, the long turn's own evidence). + runs: await this.api.get(`/api/issues/${issueId}/runs`, 10_000).then(rows => (rows ?? []).map(run => ({ id: String(run.runId ?? run.id), + status: String(run.status), errorCode: run.errorCode ?? null, createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null }))).catch(() => []), ...(this.journey ? { journey: await this.journey.collect(issueId).finally(() => this.journey!.close()) } : {}), }; } diff --git a/tests/restart-chaos/recovery-step-faults.ts b/tests/restart-chaos/recovery-step-faults.ts new file mode 100644 index 00000000000..d2d37d0d185 --- /dev/null +++ b/tests/restart-chaos/recovery-step-faults.ts @@ -0,0 +1,90 @@ +/** + * `--fail-recovery-step [:]`: fail the Nth attempt of one recovery step once (README "Fail the Nth + * recovery step"). + * + * Every other fault hits the control plane once and then lets recovery run. On staging the first fault was + * often harmless and a later recovery step failed: the retry of a bootstrap met a missing runtime asset, a + * reattach met `Agent directory operation is already active`, a bounded retry met an unfinished teardown. + * This option lets the wave's first restart fault fire normally. When that fault's replacement server + * starts, the harness arms one crash point in fail mode (`packages/shared/src/crash-points.ts`): the nth hit + * of the point in the replacement throws the error its real step throws, once. nth = 1 fails recovery's + * first attempt at the step, 2 its second, and so on, so recovery chains are exercised. + * + * Hits count across the whole server: with several tasks in flight, the nth hit may belong to any of them. + * Run one task per wave (`--tasks 1 --concurrency 1`) to aim at one task's nth attempt. + */ +import { CRASH_POINTS, isCrashPointName, type CrashPointName } from "../../packages/shared/src/crash-point-registry.js"; +import { noFailedAttemptCheck } from "./rewake.js"; +import type { EnvironmentKind, FaultKind, OracleResult } from "./types.js"; + +/** Short names for the steps staging failed at; any registered crash point name is also accepted. */ +export const RECOVERY_STEPS: Record = { + // A runnerd start that fails before the process exists (spawn EAGAIN), local and Daytona. + "runnerd-start": "native.process_start.after_requested_before_spawn", + // A sandbox teardown whose provider destroy did not finish (Daytona). + teardown: "environment.teardown.after_claim_before_destroy", + // The agent-directory lock still held by another operation (Daytona): staging class A's terminal error. + "directory-lock": "agent_directory.operation_lock.before_acquire", +}; + +/** Restart faults whose replacement start arms the step. */ +export const RECOVERY_STEP_FAULTS: readonly FaultKind[] = ["K1", "K2", "K4", "K6", "K14", "K15", "RR1", "RR2", "RR3"]; + +export interface RecoveryStepSpec { point: CrashPointName; nth: number } + +export function parseRecoveryStep(raw: string): RecoveryStepSpec { + const match = /^([a-z0-9_.-]+)(?::([1-9][0-9]?))?$/.exec(raw.trim()); + if (!match) throw new Error("--fail-recovery-step takes [:], for example directory-lock:2"); + const point = RECOVERY_STEPS[match[1]!] ?? match[1]!; + if (!isCrashPointName(point)) { + throw new Error(`--fail-recovery-step: unknown step "${match[1]}"; use ${Object.keys(RECOVERY_STEPS).join(", ")} or a crash point name`); + } + return { point, nth: match[2] ? Number(match[2]) : 1 }; +} + +/** The control-file spec the supervisor writes: ": fail". */ +export function recoveryStepControlSpec(spec: RecoveryStepSpec): string { + return `${spec.point}:${spec.nth} fail`; +} + +export function validateRecoveryStep(options: { env: EnvironmentKind; faults: readonly string[]; crashPoint?: string; stormMinutes?: number }, + raw: string | undefined): { failRecoveryStep?: string } { + if (raw === undefined) return {}; + const spec = parseRecoveryStep(raw); + if (options.crashPoint) throw new Error("--fail-recovery-step arms its own point; do not combine it with --crash-point"); + if (!options.faults.some(fault => RECOVERY_STEP_FAULTS.includes(fault as FaultKind))) { + throw new Error(`--fail-recovery-step fails a step of the recovery after a restart; add one of ${RECOVERY_STEP_FAULTS.join(", ")} to --faults`); + } + const environments = CRASH_POINTS[spec.point].environments as readonly string[]; + if (!environments.includes(options.env)) { + throw new Error(`--fail-recovery-step ${spec.point} is reached only with --env ${environments.join(" or ")}`); + } + return { failRecoveryStep: `${spec.point}:${spec.nth}` }; +} + +export interface RecoveryStepHit { point: string; hit: number; pid: number; token?: string; mode?: string; at?: string } + +/** Every fail-mode receipt the server appended. */ +export function parseRecoveryStepReceipts(text: string): RecoveryStepHit[] { + return text.split("\n").flatMap(line => { + try { + const parsed = JSON.parse(line) as RecoveryStepHit; + return parsed.mode === "fail" && typeof parsed.point === "string" ? [parsed] : []; + } catch { return []; } + }); +} + +/** + * The task's checks, added to every task the armed fault crossed: the armed step failed in the replacement (else + * the cell tested nothing), and recovery absorbed it: no attempt of the task failed. + */ +export function withRecoveryStepCheck(oracle: OracleResult, armed: { faultId: string; spec: string; token: string } | null, + taskFaultIds: readonly string[], hits: readonly RecoveryStepHit[], + runs: ReadonlyArray<{ id: string; status: string; errorCode: string | null }>): OracleResult { + if (!armed || !taskFaultIds.includes(armed.faultId)) return oracle; + const hit = hits.find(entry => entry.token === armed.token); + const checks = [...oracle.checks, { name: "recovery_step_failed_once", ok: Boolean(hit), + detail: hit ? `${hit.point} hit ${hit.hit} failed at ${hit.at ?? "?"} after ${armed.faultId}` : `${armed.spec} never reached after ${armed.faultId}` }, + ...(oracle.checks.some(check => check.name === "no_failed_attempt") ? [] : [noFailedAttemptCheck(runs)])]; + return { ...oracle, checks, verdict: checks.every(check => check.ok) ? "pass" : "fail" }; +} diff --git a/tests/restart-chaos/redeploy.ts b/tests/restart-chaos/redeploy.ts index 2af7d2ef4f5..8deb3cad1cf 100644 --- a/tests/restart-chaos/redeploy.ts +++ b/tests/restart-chaos/redeploy.ts @@ -46,6 +46,44 @@ export function parseK15HoldSeconds(options: { faults: readonly string[] }, raw: return { k15HoldMs: [min * 1000, max * 1000] }; } +/** `--k15-overlap-seconds` bounds: staging's old server kept running 30–150 s after the new one was up. */ +export const K15_OVERLAP_SECONDS_RANGE: readonly [number, number] = [5, 600]; + +/** + * The staging deploy shape (README "Redeploy without a volume, with overlap"): + * + * - `--k15-overlap-seconds ` or `-`: a K15 boots the next deploy generation beside the old server, + * switches traffic (the wave's stable port) to it once it is ready, keeps the old server running for the seeded + * overlap, then stops the old one with SIGTERM. The seeded K15 hold becomes the overlap; there is no outage. + * - `--k15-fresh-home`: every deploy generation gets a new, empty instance home and no Railway volume, as staging's + * deploys after 07:25Z reported `volumeId: null`. The database, secrets key and storage stay (they live outside + * the instance home on staging); controller-local files such as `runtime-context-assets` do not. Daytona only: + * a local run's workspace and runner live under the instance home. + */ +export function parseK15DeployShape(options: { env: string; faults: readonly string[]; k15HoldMs?: [number, number] }, + overlapRaw: string | undefined, freshHome: boolean): { k15HoldMs?: [number, number]; k15Overlap?: true; k15FreshHome?: true } { + const out: { k15HoldMs?: [number, number]; k15Overlap?: true; k15FreshHome?: true } = {}; + if ((overlapRaw !== undefined || freshHome) && !options.faults.includes("K15")) { + throw new Error("--k15-overlap-seconds and --k15-fresh-home shape the K15 redeploy; add K15 to --faults"); + } + if (overlapRaw !== undefined) { + if (options.k15HoldMs) throw new Error("--k15-overlap-seconds replaces the K15 outage; do not combine it with --k15-hold-seconds"); + const match = /^([0-9]+)(?:-([0-9]+))?$/.exec(overlapRaw.trim()); + if (!match) throw new Error("--k15-overlap-seconds takes or - seconds"); + const min = Number(match[1]); + const max = match[2] === undefined ? min : Number(match[2]); + const [low, high] = K15_OVERLAP_SECONDS_RANGE; + if (min < low || max > high || max < min) throw new Error(`--k15-overlap-seconds must be between ${low} and ${high} seconds, min not above max`); + out.k15HoldMs = [min * 1000, max * 1000]; + out.k15Overlap = true; + } + if (freshHome) { + if (options.env !== "daytona") throw new Error("--k15-fresh-home runs only with --env daytona: a local run's workspace and runner live in the instance home"); + out.k15FreshHome = true; + } + return out; +} + /** The seeded hold of one K15. Draws only when a range is set, so existing seeds keep their plans. */ export function k15HoldMs(range: readonly [number, number] | undefined, int: (min: number, max: number) => number): number { return range ? int(range[0], range[1]) : 0; @@ -98,18 +136,23 @@ export function nextRedeployIdentity(waveId: string, current: RedeployIdentity): * Server environment for one generation. The volume mount is the instance home: on staging the * instance directory lives on the exclusive volume that survives each deploy. */ -export function redeployEnvironment(identity: RedeployIdentity, volumeMountPath: string): Record { +export function redeployEnvironment(identity: RedeployIdentity, volumeMountPath: string | null): Record { return { [REDEPLOY_HOSTNAME_ENV]: identity.hostname, RAILWAY_PROJECT_ID: identity.projectId, RAILWAY_ENVIRONMENT_ID: identity.environmentId, RAILWAY_SERVICE_ID: identity.serviceId, RAILWAY_DEPLOYMENT_ID: identity.deploymentId, - RAILWAY_VOLUME_ID: identity.volumeId, - RAILWAY_VOLUME_MOUNT_PATH: volumeMountPath, + // `--k15-fresh-home`: no volume, as staging's later deploys reported `volumeId: null`. + ...(volumeMountPath === null ? {} : { RAILWAY_VOLUME_ID: identity.volumeId, RAILWAY_VOLUME_MOUNT_PATH: volumeMountPath }), }; } +/** `--k15-fresh-home`: generation N's own instance home (generation 0 keeps the wave's first home). */ +export function generationHome(temporaryRoot: string, firstHome: string, generation: number): string { + return generation === 0 ? firstHome : `${temporaryRoot}/home-g${generation}`; +} + /** Node arguments that load the preload ahead of the server entry's own `--import` loader. */ export function redeployNodeArgs(args: readonly string[]): string[] { return ["--import", REDEPLOY_PRELOAD_PATH, ...args]; diff --git a/tests/restart-chaos/rewake.ts b/tests/restart-chaos/rewake.ts new file mode 100644 index 00000000000..54d87c5a49c --- /dev/null +++ b/tests/restart-chaos/rewake.ts @@ -0,0 +1,293 @@ +/** + * W2, W2Q, W2C: the same task woken again right after a run of it finished (README "Re-wake after success"). + * + * On staging, 20 of 25 failures in one 8-hour window happened with no restart at all. Ten or more runs of + * one agent ran at once on reused (warm) sandboxes, and 30–60 s after a long run of a task succeeded, an + * ordinary wake (a comment, an issue monitor) started the next run of that same task. That run first had to + * retire the previous run's warm owner, and it failed before any provider work ("Controller journal drain + * timed out", "runner did not durably suspend before checkpoint", `journal_tool_input_unanswered`). + * + * The workload is the fault. Every task runs on its adapter's one agent, so `--tasks 10 --concurrency 10` + * puts ten tasks on one agent. Phase 1 ends with the task left `in_progress` (`in_review` would gate completion on a + * review). Then the harness, acting as the + * board, wakes the same task 0–60 s after that run settled (the seeded `followUpAfterMs`): + * + * W2 a board comment, or (by the task's nonce) a board-scheduled issue monitor (`issue_monitor_due`). + * W2Q phase 1 parks on an unanswered `request_human_input` question; a comment wakes the task before the + * board answers (staging class C3). The board answers after that run settled. + * W2C as W2, and the board changes the agent's `adapterConfig.idleTimeoutMs` before the wake, so the next + * run cannot resume the saved session (SR1, session-faults.ts, is the same change while parked). + * + * Pass: no attempt failed, the wake was served by a run that succeeded, and the task ends `done` with its + * markers. A task left blocked gets the usual Retry (R3), which itself fails the grade. + */ +import type { ChaosApi } from "./api.js"; +import { q1AnswerMarker, q1Code, q1Question, Q1_CODES, questionStep } from "./question-handoff.js"; +import type { OracleArtifacts, OracleResult, TaskPlan, TaskRecord, TaskSample, WorkloadId } from "./types.js"; +import type { WorkloadDefinition } from "./workloads.js"; + +type Json = Record; + +export const REWAKE_WORKLOADS = ["W2", "W2Q", "W2C"] as const satisfies readonly WorkloadId[]; +export type RewakeWorkload = (typeof REWAKE_WORKLOADS)[number]; +export function isRewakeWorkload(id: string): id is RewakeWorkload { + return (REWAKE_WORKLOADS as readonly string[]).includes(id); +} + +/** Seeded delay between the first run settling and the re-wake (staging: 30–60 s). */ +export const REWAKE_DELAY_RANGE_MS: readonly [number, number] = [0, 60_000]; +/** W2C: the configuration change commits at least this long before the wake. */ +export const CONFIG_SETTLE_MS = 5_000; +/** A wake that started no run within this long after it was sent is recorded as not served. */ +export const REWAKE_SERVE_TIMEOUT_MS = 180_000; +const ACTIVE = new Set(["queued", "running", "scheduled_retry"]); +const FAILED = new Set(["failed", "timed_out"]); + +export type RewakeKind = "comment" | "monitor"; + +/** W2Q is woken by a comment (C3's shape); W2 and W2C alternate by nonce between a comment and a monitor. */ +export function rewakeKind(plan: Pick): RewakeKind { + if (plan.workload === "W2Q") return "comment"; + return Number.parseInt(plan.nonce.slice(-2, -1), 16) % 2 === 0 ? "comment" : "monitor"; +} + +const briefMarker = (nonce: string) => `CHAOS-${nonce}`; +export const rewakeMarker = (nonce: string) => `REWAKE-${nonce}`; +export const noteMarker = (nonce: string) => `NOTE-${nonce}`; + +export interface RewakeRecord { + kind: RewakeKind; + /** The run that finished phase 1 (for W2Q, the run that asked). */ + firstRunId: string | null; + /** First poll that saw phase 1 finished and nothing of the task active. */ + firstSettledAt: number | null; + /** W2C: the board's agent configuration change, made before the wake. */ + configChange?: { at: number; idleTimeoutMs: number; httpStatus: number | null; error: string | null }; + sent?: { at: number; kind: RewakeKind; statusAtWake: string | null; httpStatus: number | null; error: string | null; + commentId?: string | null; nextCheckAt?: string }; + /** First run created after the wake was sent. */ + servedByRunId?: string | null; + servedSeenAt?: number | null; +} + +export interface RewakeArtifacts extends RewakeRecord { + runs: Array<{ id: string; status: string; errorCode: string | null; createdAt: string | null; finishedAt: string | null }>; +} + +const phaseTwo = (nonce: string) => [ + "Phase 2 happens only when the board wakes you again with a comment or a scheduled monitor check on this task.", + `Then add the line \`${rewakeMarker(nonce)}\` on its own line to the \`brief\` document (keep everything else), and mark this task done.`, + "If you are woken for any other reason before that, change nothing and end your turn.", +].join("\n"); + +const w2Description = (nonce: string) => [ + "This task has two phases.", + "Phase 1 (now): write a short brief on why idempotent retries matter for background job queues.", + "Save it as this issue's document with key `brief` (title \"Brief\"), in markdown.", + `It must contain the line \`${briefMarker(nonce)}\` exactly, on its own line, and at least three bullet points that start with "- ".`, + "Then leave this task `in_progress` and end your turn. Do not mark it done or move it to review in phase 1. Do not create files or other tasks.", + phaseTwo(nonce), +].join("\n"); + +const w2qDescription = (nonce: string) => [ + "This task needs one answer from the board before you write anything.", + `1. Ask exactly one question with your \`request_human_input\` tool: a questions card (continuationPolicy \`wake_assignee\`,`, + ` idempotencyKey \`w2q-${nonce}\`) with one required single-select question "${q1Question(nonce)}"`, + ` and exactly two options labelled \`${Q1_CODES[0]}\` and \`${Q1_CODES[1]}\`.`, + "2. Then end your turn and wait for the answer. Do not guess, do not ask again, and do not create documents, files or tasks yet.", + " If the board comments before it answers, follow the comment's note and end your turn again; do not ask again.", + "3. When you are woken with the answer, save this issue's document with key `brief` (title \"Brief\"), in markdown.", + ` It must contain the line \`${q1AnswerMarker(nonce, "")}\` exactly, on its own line, with replaced by the chosen option,`, + " every line the board's comments asked you to include, and at least three bullet points starting with \"- \".", + "4. When the document is saved, mark this task done.", +].join("\n"); + +/** The board's re-wake comment. W2Q's asks only for a note, so its run must end while the question stays pending. */ +export function rewakeComment(plan: Pick): string { + if (plan.workload === "W2Q") { + return `Note before I answer your question: when you write the brief, also include the line \`${noteMarker(plan.nonce)}\` on its own line. ` + + "I will answer shortly; do not ask again and do not write the brief yet."; + } + return `Phase 2 now: add the line \`${rewakeMarker(plan.nonce)}\` on its own line to the \`brief\` document, then mark this task done.`; +} + +/** W2C: one adapter setting in the session's configuration fingerprint, harmless to the run (as SR1 changes). */ +export function rewakeIdleTimeoutMs(taskIndex: number): number { + return 300_000 + 1_000 + taskIndex; +} + +/** The task waits on the board between its first run settling and the wake being served (and, for W2Q, the answer). */ +export function awaitingRewake(record: TaskRecord, now = Date.now()): boolean { + const rewake = record.rewake; + if (!rewake?.firstSettledAt) return false; + if (!rewake.sent) return true; + if (rewake.sent.error) return false; + if (!rewake.servedByRunId && now - rewake.sent.at < REWAKE_SERVE_TIMEOUT_MS) return true; + return record.plan.workload === "W2Q" && Boolean(record.question?.pendingNow) && !record.question?.answer; +} + +/** The first run of the task that succeeded, when nothing of the task is active. */ +function settledFirstRun(sample: TaskSample): string | null { + const runs = sample.runs ?? []; + if (runs.some(run => ACTIVE.has(run.status))) return null; + const succeeded = runs.filter(run => run.status === "succeeded") + .sort((a, b) => Date.parse(a.createdAt ?? "") - Date.parse(b.createdAt ?? "")); + return succeeded[0]?.id ?? null; +} + +export interface RewakeStepInput { + api: Pick; + record: TaskRecord; + plan: TaskPlan; + issueId: string; + agentId: string; + sample: TaskSample; + now: number; + /** The server is ready, no fault is in progress and the task's planned faults fired. */ + boardMayAct: boolean; + note: (kind: string, data: Record) => void; +} + +/** Board step of W2, W2Q and W2C, called on every poll. */ +export async function rewakeStep(input: RewakeStepInput): Promise { + const { plan, record, sample, now } = input; + const rewake = record.rewake ??= { kind: rewakeKind(plan), firstRunId: null, firstSettledAt: null }; + if (plan.workload === "W2Q") { + // The question is answered only after the comment's run settled; questionStep keeps Q1's record. + await questionStep({ api: input.api, issueId: input.issueId, nonce: plan.nonce, record, sample, now, note: input.note, + boardMayAct: input.boardMayAct && Boolean(rewake.servedByRunId) && !(sample.runs ?? []).some(run => ACTIVE.has(run.status)) }); + } + if (!sample.reachable) return; + if (rewake.sent) { + if (!rewake.servedByRunId) { + const served = (sample.runs ?? []).filter(run => run.createdAt && Date.parse(run.createdAt) >= rewake.sent!.at - 1_000) + .sort((a, b) => Date.parse(a.createdAt!) - Date.parse(b.createdAt!))[0]; + if (served) { + rewake.servedByRunId = served.id; + rewake.servedSeenAt = now; + input.note("rewake_served", { runId: served.id }); + } + } + return; + } + if (rewake.firstSettledAt === null) { + // W2Q: phase 1 is the asking run, settled while the question is pending. + if (plan.workload === "W2Q" && !record.question?.pendingNow) return; + if (plan.workload !== "W2Q" && sample.issueStatus === "done") return; + const first = settledFirstRun(sample); + if (!first) return; + rewake.firstRunId = plan.workload === "W2Q" ? record.question?.askedByRunId ?? first : first; + rewake.firstSettledAt = now; + input.note("rewake_first_run_settled", { runId: rewake.firstRunId, status: sample.issueStatus ?? null }); + } + if (!input.boardMayAct || (sample.runs ?? []).some(run => ACTIVE.has(run.status))) return; + if (plan.workload === "W2C" && !rewake.configChange) { + const idleTimeoutMs = rewakeIdleTimeoutMs(plan.index); + const change: NonNullable = { at: now, idleTimeoutMs, httpStatus: null, error: null }; + try { + await input.api.patch(`/api/agents/${input.agentId}`, { adapterConfig: { idleTimeoutMs } }); + change.httpStatus = 200; + } catch (error) { + change.httpStatus = statusOf(error); + change.error = error instanceof Error ? error.message : String(error); + } + rewake.configChange = change; + input.note("rewake_config_changed", { ...change }); + return; + } + if (now < rewake.firstSettledAt + (plan.followUpAfterMs ?? 0)) return; + // W2C: the change commits at least CONFIG_SETTLE_MS before the wake, however late the board made it. + if (rewake.configChange && now < rewake.configChange.at + CONFIG_SETTLE_MS) return; + const sent: NonNullable = { at: now, kind: rewake.kind, statusAtWake: sample.issueStatus ?? null, httpStatus: null, error: null }; + try { + if (rewake.kind === "comment") { + const comment = await input.api.post(`/api/issues/${input.issueId}/comments`, { body: rewakeComment(plan) }); + sent.commentId = typeof comment?.id === "string" ? comment.id : null; + } else { + // A board-scheduled monitor due now: the scheduler's own `issue_monitor_due` wake, as on staging. + sent.nextCheckAt = new Date(now + 2_000).toISOString(); + await input.api.patch(`/api/issues/${input.issueId}`, { executionPolicy: { monitor: { nextCheckAt: sent.nextCheckAt, + notes: `Phase 2 check for ${plan.nonce}`, scheduledBy: "board" } } }); + } + sent.httpStatus = 200; + } catch (error) { + sent.httpStatus = statusOf(error); + sent.error = error instanceof Error ? error.message : String(error); + } + rewake.sent = sent; + input.note("rewake_sent", { ...sent, delayMs: now - rewake.firstSettledAt }); +} + +function statusOf(error: unknown): number | null { + return typeof (error as { status?: unknown }).status === "number" ? (error as { status: number }).status : null; +} + +export async function collectRewakeArtifacts(api: Pick, issueId: string, record: TaskRecord): Promise { + const runs = await api.get(`/api/issues/${issueId}/runs`, 10_000).catch(() => []); + return { + ...(record.rewake ?? { kind: rewakeKind(record.plan), firstRunId: null, firstSettledAt: null }), + runs: (runs ?? []).map(run => ({ id: String(run.runId ?? run.id), status: String(run.status), errorCode: run.errorCode ?? null, + createdAt: run.createdAt ?? null, finishedAt: run.finishedAt ?? null })), + }; +} + +/** No run of the task failed or timed out; the shared pass condition of the shape cells. */ +export function noFailedAttemptCheck(runs: ReadonlyArray<{ id: string; status: string; errorCode: string | null }>) { + const failed = runs.filter(run => FAILED.has(run.status)); + return { name: "no_failed_attempt", ok: failed.length === 0, + detail: failed.map(run => `${run.id}:${run.status}:${run.errorCode ?? ""}`).join(",") || "none" }; +} + +export function gradeRewake(plan: Pick, artifacts: OracleArtifacts): OracleResult { + const rewake = artifacts.rewake; + const runs = rewake?.runs ?? []; + const brief = artifacts.documents.brief ?? ""; + const bullets = brief.split(/\r?\n/).filter(line => /^\s*[-*] \S/.test(line)).length; + const first = runs.find(run => run.id === rewake?.firstRunId); + const served = runs.find(run => run.id === rewake?.servedByRunId); + const sent = rewake?.sent; + const markers = plan.workload === "W2Q" + ? [q1AnswerMarker(plan.nonce, q1Code(plan.nonce)), noteMarker(plan.nonce)] : [briefMarker(plan.nonce), rewakeMarker(plan.nonce)]; + const checks: OracleResult["checks"] = [ + { name: "issue_done", ok: artifacts.issueStatus === "done", detail: `status=${artifacts.issueStatus ?? "unknown"}` }, + { name: "first_run_succeeded", ok: first?.status === "succeeded", + detail: first ? `${first.id} ${first.status}${first.errorCode ? ` ${first.errorCode}` : ""}` : "phase 1 never settled" }, + { name: "rewake_sent", ok: sent?.httpStatus === 200, + detail: sent ? `${sent.kind} status=${sent.statusAtWake ?? "?"} http=${sent.httpStatus}${sent.error ? ` ${sent.error}` : ""}` + + ` after ${rewake?.firstSettledAt ? Math.round((sent.at - rewake.firstSettledAt) / 1000) : "?"}s` : "never sent" }, + { name: "rewake_served", ok: served?.status === "succeeded", + detail: served ? `${served.id} ${served.status}${served.errorCode ? ` ${served.errorCode}` : ""}` : "no run after the wake" }, + noFailedAttemptCheck(runs), + ...(plan.workload === "W2C" ? [{ name: "config_changed_before_wake", ok: rewake?.configChange?.httpStatus === 200 + && (sent === undefined || rewake.configChange.at < sent.at), + detail: rewake?.configChange ? `idleTimeoutMs=${rewake.configChange.idleTimeoutMs} http=${rewake.configChange.httpStatus}` : "not changed" }] : []), + ...(plan.workload === "W2Q" ? [{ name: "question_answered_after_wake", ok: artifacts.question?.answer?.httpStatus === 200 + && sent !== undefined && artifacts.question.answer.at > sent.at, detail: artifacts.question?.answer + ? `${artifacts.question.answer.code} http=${artifacts.question.answer.httpStatus}` : "never answered" }, + { name: "one_question_asked", ok: artifacts.question?.interactionIds.length === 1, detail: `questions=${artifacts.question?.interactionIds.length ?? 0}` }] : []), + ...markers.map(marker => ({ name: `brief_contains_${marker.split("-")[0]!.toLowerCase()}_marker`, ok: brief.includes(marker), detail: marker })), + { name: "brief_has_three_bullets", ok: bullets >= 3, detail: `bullets=${bullets}` }, + ]; + return { verdict: checks.every(check => check.ok) ? "pass" : "fail", checks }; +} + +const definition = (id: RewakeWorkload, label: string, description: (nonce: string) => string): WorkloadDefinition => ({ + id, + label, + usesRepository: false, + // A restart fault, if any, lands around the first run or the wake. + fireWindowMs: [20_000, 120_000], + // The seeded re-wake delay after the first run settled. + followUpWindowMs: REWAKE_DELAY_RANGE_MS, + budgetMs: 20 * 60_000, + title: nonce => `Chaos ${id} re-wake ${nonce}`, + description, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade: (nonce, artifacts) => gradeRewake({ workload: id, nonce }, artifacts), +}); + +export const rewakeWorkloadDefinitions: Record = { + W2: definition("W2", "Same task woken again after its run succeeded", w2Description), + W2Q: definition("W2Q", "Parked question, then woken by a comment before the answer", w2qDescription), + W2C: definition("W2C", "Agent configuration changed, then the same task woken again", w2Description), +}; diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index ef829e802b4..f151252844b 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -81,8 +81,8 @@ const STORM_RESULT_FAULTS: readonly FaultKind[] = ["K1", "K2"]; * A `result` task in a storm also fires its own K1 or K2 just after its run * accepts a result, the window of RC-30, RC-33 and RC-41. */ -function stormCellFaults(options: Pick, trigger: TriggerId): readonly TaskFaultKind[] { - if (!options.stormMinutes) return [...options.faults, ...(options.providerFaults ?? []), ...(options.diskFullFaults ?? []), ...(options.sessionFaults ?? [])]; +function stormCellFaults(options: Pick, trigger: TriggerId): readonly TaskFaultKind[] { + if (!options.stormMinutes) return [...options.faults, ...(options.providerFaults ?? []), ...(options.diskFullFaults ?? []), ...(options.sessionFaults ?? []), ...(options.baseline ? ["K0" as const] : [])]; if (trigger === "time") return ["K13"]; if (trigger !== "result") throw new Error(`A K13 storm fires on its own clock; --triggers ${trigger} does not combine with it (use time or result)`); const faults = options.faults.filter(fault => STORM_RESULT_FAULTS.includes(fault)); @@ -98,7 +98,7 @@ function pinnedFireAfterMs(faultAfterMs: number | undefined, drawn: number): num return faultAfterMs ?? drawn; } -export function planTasks(options: Pick): TaskPlan[] { +export function planTasks(options: Pick): TaskPlan[] { const rng = createRng(options.seed); const triggers = options.triggers?.length ? options.triggers : ["time" as const]; for (const trigger of triggers) { @@ -128,7 +128,7 @@ export function planTasks(options: Pick options.longTurnMinutes ? { ...task, longTurnMinutes: options.longTurnMinutes } : task), ...(options.stormMinutes ? { storm: planStorm(options.seed, options.stormMinutes, options.faults, options.k4HoldMaxMs, options.k15HoldMs) } : {}) }; } diff --git a/tests/restart-chaos/shape-cells.test.ts b/tests/restart-chaos/shape-cells.test.ts new file mode 100644 index 00000000000..6170f0e51af --- /dev/null +++ b/tests/restart-chaos/shape-cells.test.ts @@ -0,0 +1,252 @@ +import net from "node:net"; +import { describe, expect, it } from "vitest"; +import { parseCommand } from "./config.js"; +import { startEdgeProxy } from "./edge-proxy.js"; +import { longTurnChecks, longTurnCommand, longTurnRounds, parseLongTurnLog, withLongTurn } from "./long-turn.js"; +import { q1AnswerMarker, q1Code } from "./question-handoff.js"; +import { parseRecoveryStep, parseRecoveryStepReceipts, recoveryStepControlSpec, withRecoveryStepCheck } from "./recovery-step-faults.js"; +import { generationHome, parseK15DeployShape, redeployEnvironment, redeployIdentity } from "./redeploy.js"; +import { awaitingRewake, gradeRewake, noteMarker, rewakeKind, rewakeMarker, rewakeStep, type RewakeArtifacts } from "./rewake.js"; +import { planTasks, planWave, taskFaultSchedule } from "./scheduler.js"; +import type { OracleArtifacts, RunSample, TaskPlan, TaskRecord, TaskSample } from "./types.js"; +import { workloadDefinitions } from "./workloads.js"; + +const wave = (argv: string[]) => { + const command = parseCommand(argv); + if (command.mode !== "wave") throw new Error("expected a wave"); + return command.options; +}; + +const run = (id: string, status: string, createdAt: number): RunSample => ({ id, status, createdAt: new Date(createdAt).toISOString(), startedAt: null, + finishedAt: null, errorCode: null, retryOfRunId: null, wakeCommentIds: [], livenessState: null }); +const sample = (issueStatus: string, runs: RunSample[]): TaskSample => ({ at: 0, reachable: true, serverReady: true, issueStatus, runs }); + +describe("W2, W2Q, W2C: re-wake after success", () => { + it("are opt-in workloads that run as K0 cells on one agent", () => { + expect(wave(["--adapters", "codex"]).workloads).not.toContain("W2"); + const options = wave(["--adapters", "codex", "--workloads", "W2,W2Q,W2C", "--faults", "K0", "--tasks", "10", "--concurrency", "10", "--lifecycle", "warm"]); + expect(options).toMatchObject({ faults: [], baseline: true, workloads: ["W2", "W2Q", "W2C"] }); + const tasks = planTasks(options); + expect(tasks).toHaveLength(10); + expect(new Set(tasks.map(task => task.adapter))).toEqual(new Set(["codex"])); + expect(tasks.every(task => task.fault === "K0" && taskFaultSchedule(task).length === 0)).toBe(true); + // The seeded re-wake delay is 0–60 s after the first run settled. + expect(tasks.every(task => task.followUpAfterMs !== null && task.followUpAfterMs >= 0 && task.followUpAfterMs <= 60_000)).toBe(true); + expect(() => parseCommand(["--workloads", "W2", "--faults", "K0", "--storm-minutes", "5"])).toThrow("K0"); + // A restart may also land around the first run or the wake. + expect(planTasks(wave(["--workloads", "W2", "--faults", "K2", "--adapters", "codex", "--tasks", "2"])).map(task => task.fault)).toEqual(["K2", "K2"]); + }); + + it("asks phase 1 to leave the task open and phase 2 to wait for the board", () => { + const text = workloadDefinitions.W2.description("0bcd1234"); + expect(text).toContain("leave this task `in_progress`"); + expect(text).toContain(rewakeMarker("0bcd1234")); + expect(workloadDefinitions.W2Q.description("0bcd1234")).toContain("request_human_input"); + expect(rewakeKind({ workload: "W2Q", nonce: "0bcd1234" })).toBe("comment"); + expect(new Set(["00", "10", "20", "30"].map(tail => rewakeKind({ workload: "W2", nonce: `0bcd12${tail}` })))).toEqual(new Set(["comment", "monitor"])); + }); + + function harness(plan: Partial) { + const calls: Array<{ method: string; route: string; body: unknown }> = []; + const record = { plan: { taskId: "t01", index: 3, nonce: "0bcd1204", followUpAfterMs: 10_000, workload: "W2", ...plan } } as unknown as TaskRecord; + const api = { + async get(route: string) { return (route.endsWith("/interactions") ? interactions : []) as T; }, + async post(route: string, body?: unknown) { calls.push({ method: "post", route, body }); return { id: "c1" } as T; }, + async patch(route: string, body?: unknown) { calls.push({ method: "patch", route, body }); return {} as T; }, + }; + let interactions: Array> = []; + const notes: string[] = []; + const step = (s: TaskSample, now: number, boardMayAct = true) => rewakeStep({ api, record, plan: record.plan, issueId: "i1", agentId: "a1", + sample: s, now, boardMayAct, note: kind => notes.push(kind) }); + return { calls, record, notes, step, setInteractions: (rows: Array>) => { interactions = rows; } }; + } + + it("W2: wakes the task with a comment after the seeded delay, and waits until a run serves it", async () => { + const { calls, record, notes, step } = harness({}); + await step(sample("in_progress", [run("r1", "running", 1_000)]), 2_000); + expect(awaitingRewake(record, 2_000)).toBe(false); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 5_000); + expect(record.rewake).toMatchObject({ firstRunId: "r1", firstSettledAt: 5_000 }); + expect(awaitingRewake(record, 5_000)).toBe(true); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 14_000); + expect(calls).toHaveLength(0); + // Faults still pending: the board waits. + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 16_000, false); + expect(calls).toHaveLength(0); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 16_000); + expect(calls).toEqual([{ method: "post", route: "/api/issues/i1/comments", body: { body: expect.stringContaining(rewakeMarker("0bcd1204")) } }]); + expect(awaitingRewake(record, 17_000)).toBe(true); + await step(sample("in_progress", [run("r1", "succeeded", 1_000), run("r2", "running", 16_500)]), 20_000); + expect(record.rewake).toMatchObject({ servedByRunId: "r2" }); + expect(awaitingRewake(record, 20_000)).toBe(false); + expect(notes).toEqual(["rewake_first_run_settled", "rewake_sent", "rewake_served"]); + }); + + it("W2 monitor: schedules a board monitor due now", async () => { + const { calls, step } = harness({ nonce: "0bcd1214" }); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 5_000); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 15_000); + expect(calls).toEqual([{ method: "patch", route: "/api/issues/i1", body: { executionPolicy: { monitor: { + nextCheckAt: new Date(17_000).toISOString(), notes: "Phase 2 check for 0bcd1214", scheduledBy: "board" } } } }]); + }); + + it("W2C: changes the agent configuration first, then wakes at least 5 s later", async () => { + const { calls, record, step } = harness({ workload: "W2C", followUpAfterMs: 0 }); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 5_000); + expect(calls).toEqual([{ method: "patch", route: "/api/agents/a1", body: { adapterConfig: { idleTimeoutMs: 301_003 } } }]); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 7_000); + expect(calls).toHaveLength(1); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 10_000); + expect(calls).toHaveLength(2); + }); + + it("W2C: waits CONFIG_SETTLE_MS after a late configuration change", async () => { + const { calls, record, step } = harness({ workload: "W2C", followUpAfterMs: 0 }); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 5_000, false); + // The board was held for 30 s; the seeded delay is long past when it may act. + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 35_000); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 38_000); + expect(calls).toHaveLength(1); + await step(sample("in_review", [run("r1", "succeeded", 1_000)]), 40_000); + expect(calls).toHaveLength(2); + expect(record.rewake?.configChange).toMatchObject({ httpStatus: 200, idleTimeoutMs: 301_003 }); + }); + + it("W2Q: comments while the question is pending, and answers only after that run settled", async () => { + const { calls, record, step, setInteractions } = harness({ workload: "W2Q", followUpAfterMs: 0 }); + setInteractions([{ id: "q1", kind: "ask_user_questions", status: "pending", sourceRunId: "r1", + payload: { questions: [{ id: "codename", options: [{ id: "opt-a", label: "ALPHA" }, { id: "opt-b", label: "BRAVO" }] }] } }]); + await step(sample("in_progress", [run("r1", "running", 1_000)]), 2_000); + expect(record.rewake?.firstSettledAt).toBeNull(); + await step(sample("in_progress", [run("r1", "succeeded", 1_000)]), 4_000); + expect(calls.map(call => call.route)).toEqual(["/api/issues/i1/comments"]); + expect(calls[0]!.body).toEqual({ body: expect.stringContaining(noteMarker("0bcd1204")) }); + await step(sample("in_progress", [run("r1", "succeeded", 1_000), run("r2", "running", 4_500)]), 6_000); + expect(calls).toHaveLength(1); + expect(awaitingRewake(record, 6_000)).toBe(true); + await step(sample("in_progress", [run("r1", "succeeded", 1_000), run("r2", "succeeded", 4_500)]), 9_000); + expect(calls.map(call => call.route)).toEqual(["/api/issues/i1/comments", "/api/issues/i1/interactions/q1/respond"]); + expect(awaitingRewake(record, 9_000)).toBe(false); + }); + + const rewakeArtifacts = (overrides: Partial = {}): RewakeArtifacts => ({ kind: "comment", firstRunId: "r1", firstSettledAt: 5_000, + sent: { at: 16_000, kind: "comment", statusAtWake: "in_review", httpStatus: 200, error: null }, servedByRunId: "r2", + runs: [{ id: "r1", status: "succeeded", errorCode: null, createdAt: null, finishedAt: null }, + { id: "r2", status: "succeeded", errorCode: null, createdAt: null, finishedAt: null }], ...overrides }); + const artifacts = (brief: string, rewake: RewakeArtifacts, extra: Partial = {}): OracleArtifacts => ({ issueStatus: "done", + documents: { brief }, agentComments: [], files: {}, commands: {}, workspacePath: null, rewake, ...extra }); + + it("grades a served wake and fails a failed attempt before it (staging class C+)", () => { + const brief = `CHAOS-0bcd1204\n- a\n- b\n- c\n${rewakeMarker("0bcd1204")}\n`; + expect(gradeRewake({ workload: "W2", nonce: "0bcd1204" }, artifacts(brief, rewakeArtifacts())).verdict).toBe("pass"); + const failed = gradeRewake({ workload: "W2", nonce: "0bcd1204" }, artifacts(brief, rewakeArtifacts({ servedByRunId: "r2", runs: [ + { id: "r1", status: "succeeded", errorCode: null, createdAt: null, finishedAt: null }, + { id: "r2", status: "failed", errorCode: "provider_transport_failed", createdAt: null, finishedAt: null }, + { id: "r3", status: "succeeded", errorCode: null, createdAt: null, finishedAt: null }] }))); + expect(failed.checks.filter(check => !check.ok).map(check => check.name)).toEqual(["rewake_served", "no_failed_attempt"]); + const w2q = gradeRewake({ workload: "W2Q", nonce: "0bcd1204" }, artifacts(`${q1AnswerMarker("0bcd1204", q1Code("0bcd1204"))}\n${noteMarker("0bcd1204")}\n- a\n- b\n- c`, + rewakeArtifacts(), { question: { interactionIds: ["q1"], seenAt: 1, askedByRunId: "r1", pendingNow: false, statuses: {}, runs: [], + answer: { at: 20_000, interactionId: "q1", code: q1Code("0bcd1204"), httpStatus: 200, error: null } } })); + expect(w2q.verdict).toBe("pass"); + }); +}); + +describe("--fail-recovery-step", () => { + it("names a step by its short name or crash point, after a restart fault, in an environment that reaches it", () => { + expect(parseRecoveryStep("directory-lock:2")).toEqual({ point: "agent_directory.operation_lock.before_acquire", nth: 2 }); + expect(parseRecoveryStep("native.process_start.after_requested_before_spawn")).toEqual({ point: "native.process_start.after_requested_before_spawn", nth: 1 }); + expect(recoveryStepControlSpec({ point: "environment.teardown.after_claim_before_destroy", nth: 3 })).toBe("environment.teardown.after_claim_before_destroy:3 fail"); + expect(() => parseRecoveryStep("nope")).toThrow("unknown step"); + expect(wave(["--faults", "K1", "--fail-recovery-step", "runnerd-start:2"]).failRecoveryStep).toBe("native.process_start.after_requested_before_spawn:2"); + expect(() => parseCommand(["--faults", "K0", "--workloads", "A1", "--fail-recovery-step", "runnerd-start"])).toThrow("add one of"); + expect(() => parseCommand(["--faults", "K1", "--fail-recovery-step", "directory-lock"])).toThrow("--env daytona"); + }); + + it("checks that the armed step failed and that no attempt of a crossed task failed", () => { + const hits = parseRecoveryStepReceipts([JSON.stringify({ point: "x", hit: 2, pid: 1, token: "rs-f01", mode: "fail" }), + JSON.stringify({ point: "y", hit: 1, pid: 1 }), "partial"].join("\n")); + expect(hits).toHaveLength(1); + const base = { verdict: "pass" as const, checks: [{ name: "issue_done", ok: true }] }; + const armed = { faultId: "f01", spec: "x:2", token: "rs-f01" }; + expect(withRecoveryStepCheck(base, armed, ["f02"], hits, [])).toBe(base); + expect(withRecoveryStepCheck(base, armed, ["f01"], hits, [{ id: "r1", status: "succeeded", errorCode: null }]).verdict).toBe("pass"); + const failed = withRecoveryStepCheck(base, armed, ["f01"], hits, [{ id: "r1", status: "failed", errorCode: "runner_remote_process_identity_unavailable" }]); + expect(failed.checks.find(check => check.name === "no_failed_attempt")).toMatchObject({ ok: false }); + expect(withRecoveryStepCheck(base, armed, ["f01"], [], []).checks.find(check => check.name === "recovery_step_failed_once")).toMatchObject({ ok: false }); + }); +}); + +describe("--long-turn-minutes", () => { + it("puts rounds of real tool work in front of any workload, with a longer budget and its own checks", () => { + const options = wave(["--workloads", "A1", "--faults", "K1", "--long-turn-minutes", "15", "--adapters", "codex", "--tasks", "1"]); + expect(options.longTurnMinutes).toBe(15); + expect(planWave("w-20261111T000000-s1", options).tasks[0]).toMatchObject({ longTurnMinutes: 15 }); + expect(() => parseCommand(["--long-turn-minutes", "5"])).toThrow("between 15 and 60"); + expect(() => parseCommand(["--workloads", "Q1", "--faults", "K1", "--matrix", "--long-turn-minutes", "15"])).toThrow("not a --matrix option"); + const definition = withLongTurn(workloadDefinitions.A1, 15); + expect(definition.budgetMs).toBe(workloadDefinitions.A1.budgetMs + 15 * 60_000); + expect(definition.collect.documents).toEqual(["brief", "long-turn"]); + expect(definition.description("0bcd1234")).toContain(`Run this shell command ${longTurnRounds(15)} times`); + expect(longTurnCommand("0bcd1234", 3)).toContain("ROUND 3 $s"); + expect(withLongTurn(workloadDefinitions.A1, undefined)).toBe(workloadDefinitions.A1); + }); + + it("checks the reported rounds and a run the harness saw last the whole turn", () => { + const start = 1_800_000_000; + const log = Array.from({ length: 10 }, (_, index) => `ROUND ${index + 1} ${start + index * 92} ${start + index * 92 + 91} 1234`).join("\n"); + expect(parseLongTurnLog(log)).toHaveLength(10); + const runs = [{ id: "r1", status: "succeeded", errorCode: null, createdAt: new Date(0).toISOString(), finishedAt: new Date(16 * 60_000).toISOString() }]; + expect(longTurnChecks(15, { documents: { "long-turn": log }, runs }).every(check => check.ok)).toBe(true); + const short = longTurnChecks(15, { documents: { "long-turn": log.split("\n").slice(0, 4).join("\n") }, runs: [{ ...runs[0]!, finishedAt: new Date(5 * 60_000).toISOString() }] }); + expect(short.map(check => check.ok)).toEqual([false, false]); + }); +}); + +describe("K15 staging deploy shape", () => { + it("parses the overlap and the fresh home, and refuses what does not fit", () => { + const options = wave(["--env", "daytona", "--daytona-image", `ghcr.io/x/y@sha256:${"a".repeat(64)}`, "--faults", "K15", "--k15-overlap-seconds", "30-150", + "--k15-fresh-home", "--concurrency", "2"]); + expect(options).toMatchObject({ k15HoldMs: [30_000, 150_000], k15Overlap: true, k15FreshHome: true }); + expect(planTasks(options).every(task => task.fault !== "K15" || (task.holdMs >= 30_000 && task.holdMs <= 150_000))).toBe(true); + expect(() => parseK15DeployShape({ env: "local", faults: ["K15"] }, undefined, true)).toThrow("--env daytona"); + expect(() => parseK15DeployShape({ env: "daytona", faults: ["K1"] }, "30", false)).toThrow("add K15"); + expect(() => parseK15DeployShape({ env: "daytona", faults: ["K15"], k15HoldMs: [1, 2] }, "30", false)).toThrow("--k15-hold-seconds"); + expect(() => parseK15DeployShape({ env: "daytona", faults: ["K15"] }, "2-3", false)).toThrow("between 5 and 600"); + }); + + it("drops the volume variables for a fresh home, and gives each generation its own home", () => { + const identity = redeployIdentity("w-20261111T000000-s1", 1); + expect(redeployEnvironment(identity, "/tmp/home")).toMatchObject({ RAILWAY_VOLUME_MOUNT_PATH: "/tmp/home" }); + expect(redeployEnvironment(identity, null)).not.toHaveProperty("RAILWAY_VOLUME_ID"); + expect(generationHome("/tmp/root", "/tmp/root/home", 0)).toBe("/tmp/root/home"); + expect(generationHome("/tmp/root", "/tmp/root/home", 2)).toBe("/tmp/root/home-g2"); + }); + + it("switches new connections at the edge while old connections stay on the old backend", async () => { + const listen = (reply: string) => new Promise(resolve => { + const server = net.createServer(socket => socket.end(reply)); + server.listen(0, "127.0.0.1", () => resolve(server)); + }); + const port = (server: net.Server) => (server.address() as net.AddressInfo).port; + const read = (edgePort: number) => new Promise((resolve, reject) => { + const socket = net.connect(edgePort, "127.0.0.1"); + let text = ""; + socket.on("data", chunk => { text += chunk.toString(); }); + socket.on("end", () => resolve(text)); + socket.on("error", reject); + }); + const [a, b] = await Promise.all([listen("A"), listen("B")]); + const free = await listen(""); + const edgePort = port(free); + await new Promise(resolve => free.close(() => resolve())); + const edge = await startEdgeProxy(edgePort, port(a)); + try { + expect(await read(edgePort)).toBe("A"); + edge.target = port(b); + expect(await read(edgePort)).toBe("B"); + } finally { + await edge.close(); + a.close(); b.close(); + } + }); +}); diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index ddb982d2be8..671e5b1b7f7 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -16,7 +16,7 @@ import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import { randomBytes } from "node:crypto"; import { createWriteStream, readFileSync, statSync, writeFileSync, type WriteStream } from "node:fs"; -import { mkdir, readFile, symlink, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; import { createRequire } from "node:module"; import os from "node:os"; import path from "node:path"; @@ -39,7 +39,8 @@ import { isRepeatedRestartFault, newRepeatedRestartRecord, recordFirstRestart, r type RepeatedRestartRecord, type RestartStep } from "./repeated-restarts.js"; import type { FaultKind, FaultReceipt, TriggerId } from "./types.js"; import { isMultiServerFault, runMultiServerFault, type MultiServerKind, type MultiServerRecord, type ServerProcessWindow } from "./multi-server.js"; -import { nextRedeployIdentity, redeployEnvironment, redeployIdentity, redeployNodeArgs, redeployReceipt, type RedeployIdentity } from "./redeploy.js"; +import { generationHome, nextRedeployIdentity, redeployEnvironment, redeployIdentity, redeployNodeArgs, redeployReceipt, type RedeployIdentity } from "./redeploy.js"; +import { startEdgeProxy, type EdgeProxy } from "./edge-proxy.js"; /** First boot applies every migration; a loaded host can need longer (RESTART_CHAOS_HEALTH_TIMEOUT_MS). */ const HEALTH_TIMEOUT_MS = (() => { @@ -93,7 +94,9 @@ export function readCrashPointReceipt(receiptPath: string, pid: number | undefin try { text = readFileSync(receiptPath, "utf8"); } catch { return null; } for (const line of text.split("\n").reverse()) { try { - const parsed = JSON.parse(line) as { point?: unknown; hit?: unknown; pid?: unknown }; + const parsed = JSON.parse(line) as { point?: unknown; hit?: unknown; pid?: unknown; mode?: unknown }; + // A fail-mode hit (`--fail-recovery-step`) threw in the server; it did not kill it. + if (parsed.mode === "fail") continue; if (pid !== undefined && parsed.pid === pid && typeof parsed.point === "string" && typeof parsed.hit === "number") { const token = (parsed as { token?: unknown }).token; return { name: parsed.point, hit: parsed.hit, pid, ...(typeof token === "string" ? { token } : {}) }; @@ -115,6 +118,8 @@ async function prepareProviderPath(repositoryRoot: string, temporaryRoot: string const manifest = JSON.parse(await readFile(codexPackage, "utf8")) as { bin?: string | Record }; const codexBin = typeof manifest.bin === "string" ? manifest.bin : manifest.bin?.codex; if (!codexBin) throw new Error("Pinned Codex executable not found"); + // A K15 with a fresh home or an overlap prepares the environment again: replace the link. + await rm(path.join(toolBin, "codex"), { force: true }); await symlink(path.resolve(path.dirname(codexPackage), codexBin), path.join(toolBin, "codex")); return [toolBin, path.join(repositoryRoot, "packages/paperclip-runner/node_modules/.bin"), inherited].filter(Boolean).join(path.delimiter); } @@ -162,6 +167,16 @@ export class ChaosSupervisor { * before the next deploy starts, while the server is down. It must not throw. */ redeployEvidence: ((faultId: string, phase: "stopped" | "before_start") => Promise) | null = null; + /** K15 deploy shape (redeploy.ts `parseK15DeployShape`); main.ts sets it before `start`. */ + deployShape: { overlap: boolean; freshHome: boolean } = { overlap: false, freshHome: false }; + /** `--k15-fresh-home`: the instance home and config of the serving generation (generation 0 uses `paths`). */ + private serverHome: { home: string; configPath: string }; + /** `--k15-overlap-seconds`: the edge on the wave's port, and the serving generation's own port behind it. */ + private edge: EdgeProxy | null = null; + private backendPort: number; + /** `--fail-recovery-step`: ":", armed in fail mode when the first restart fault starts its replacement. */ + private pendingRecoveryStep: string | null = null; + private recoveryStepWatch: NodeJS.Timeout | null = null; /** Wave secrets, kept across a K14 switch as an in-place upgrade keeps its configuration. */ private readonly secrets = { PAPERCLIP_AGENT_JWT_SECRET: secret(), PAPERCLIP_DECISION_SIGNING_SECRET: secret(), PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: secret(), BETTER_AUTH_SECRET: secret() }; @@ -169,6 +184,8 @@ export class ChaosSupervisor { private constructor(readonly repositoryRoot: string, readonly port: number, readonly paths: SupervisorPaths, private readonly timeline: Timeline, upgradeFrom: string | null, redeployWaveId: string | null) { this.serverRoot = upgradeFrom ?? repositoryRoot; + this.serverHome = { home: paths.home, configPath: paths.configPath }; + this.backendPort = port; if (redeployWaveId) this.redeploy = { waveId: redeployWaveId, identity: redeployIdentity(redeployWaveId, 0) }; this.baseUrl = `http://127.0.0.1:${port}`; this.log = createWriteStream(paths.serverLogPath, { flags: "a", mode: 0o600 }); @@ -211,6 +228,12 @@ export class ChaosSupervisor { this.database = await prepareControllerDatabase(configPath, temporaryRoot); this.timeline.add("database_started", {}); this.extraEnvironment = extraEnvironment; + if (this.deployShape.overlap) { + // The wave's port becomes the edge; each generation listens behind it on its own port. + this.backendPort = await reserveRunnerE2EServerPort(); + this.edge = await startEdgeProxy(this.port, this.backendPort); + this.timeline.add("edge_proxy_started", { port: this.port, target: this.backendPort }); + } await this.prepareServerEnvironment(this.serverRoot); const child = await this.spawnServer(); await this.waitForHealth(child); @@ -224,7 +247,8 @@ export class ChaosSupervisor { * switch, as an in-place upgrade keeps them. */ private async prepareServerEnvironment(root: string): Promise { - const { temporaryRoot, home, configPath } = this.paths; + const { temporaryRoot } = this.paths; + const { home, configPath } = this.serverHome; const extraEnvironment = this.extraEnvironment; const runnerBinary = resolvePaperclipRunnerBinaryForHarness( [{ profile: { generation: "native" } } as unknown as MatrixExecution], root); @@ -239,7 +263,7 @@ export class ChaosSupervisor { PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"), }; const overrides = { - NODE_ENV: "test", PORT: String(this.port), XDG_CACHE_HOME: path.join(temporaryRoot, "xdg-cache"), + NODE_ENV: "test", PORT: String(this.backendPort), XDG_CACHE_HOME: path.join(temporaryRoot, "xdg-cache"), PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "default", ...this.secrets, PAPERCLIP_BIND: "loopback", PAPERCLIP_BIND_HOST: "127.0.0.1", PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", @@ -254,7 +278,7 @@ export class ChaosSupervisor { ...waveServerUrlEnvironment(this.port), ...extraEnvironment, // K15: the generation's emulated host name and Railway deployment identity; the volume is the instance home. - ...(this.redeploy ? redeployEnvironment(this.redeploy.identity, home) : {}), + ...(this.redeploy ? redeployEnvironment(this.redeploy.identity, this.deployShape.freshHome ? null : home) : {}), }; const environment = buildPaperclipServerEnvironment(source, overrides); const workspaceStorage = extraEnvironment.PAPERCLIP_STORAGE_PROVIDER === "s3" @@ -474,6 +498,8 @@ export class ChaosSupervisor { let outage: ControllerOutageReceipt | undefined; if (kind === "TM1" || kind === "TM2") await this.freeze(kind, previous, holdMs, receipt); else if (isMultiServerFault(kind)) await this.multiServer(kind, previous, holdMs, receipt); + // `--k15-overlap-seconds`: the next deploy boots beside the old server; its hold is the overlap. + else if (kind === "K15" && this.deployShape.overlap) await this.rollingRedeploy(previous, holdMs, receipt); else outage = await replaceController({ requestId: `chaos-${receipt.faultId}-${requestedBy}`, previousPid: previous.pid, @@ -506,10 +532,12 @@ export class ChaosSupervisor { if (kind === "K15") { // After the hold, before the next deploy starts: what its recovery will find in the sandboxes. await this.redeployEvidence?.(receipt.faultId, "before_start"); - receipt.redeploy = this.advanceDeploy(receipt.faultId); + receipt.redeploy = await this.advanceDeploy(receipt.faultId); } this.timeline.add("fault_restarting", { faultId: receipt.faultId }); - return this.spawnServer(); + const replacement = await this.spawnServer(); + this.armPendingRecoveryStep(receipt, replacement.pid); + return replacement; }, waitForReady: async replacement => { // RR with `--rr-stop-on`: watch the first replacement from spawn; waiting for health first would let a @@ -693,18 +721,101 @@ export class ChaosSupervisor { } /** K15: move the next server to the next deploy generation (redeploy.ts). */ - private advanceDeploy(faultId: string): NonNullable { + private async advanceDeploy(faultId: string): Promise> { if (!this.redeploy || !this.environment) throw new Error("K15 needs a redeploy wave (ChaosSupervisor.create redeployWaveId)"); const from = this.redeploy.identity; const to = nextRedeployIdentity(this.redeploy.waveId, from); this.redeploy.identity = to; - Object.assign(this.environment, redeployEnvironment(to, this.paths.home)); + if (this.deployShape.freshHome) this.serverHome = await this.freshGenerationHome(faultId, to.generation); + if (this.deployShape.overlap) this.backendPort = await reserveRunnerE2EServerPort(); + if (this.deployShape.freshHome || this.deployShape.overlap) await this.prepareServerEnvironment(this.serverRoot); + else Object.assign(this.environment, redeployEnvironment(to, this.paths.home)); const receipt = redeployReceipt(from, to); this.timeline.add("redeploy_switched", { faultId, ...receipt }); this.log.write(`\n[restart-chaos] ${faultId} K15 redeploy ${from.hostname}/${from.deploymentId} -> ${to.hostname}/${to.deploymentId}\n`); return receipt; } + /** + * `--k15-fresh-home`: a new, empty instance home for the next generation. Only what a hosted deploy gets from + * outside its container is carried over: the configuration (database, storage) and the secrets master key. + */ + private async freshGenerationHome(faultId: string, generation: number): Promise<{ home: string; configPath: string }> { + const home = generationHome(this.paths.temporaryRoot, this.paths.home, generation); + const instanceRoot = path.join(home, "instances", "default"); + await mkdir(path.join(instanceRoot, "secrets"), { recursive: true, mode: 0o700 }); + const config = JSON.parse(await readFile(this.paths.configPath, "utf8")) as Record; + const keyFile = path.join(instanceRoot, "secrets", "master.key"); + await copyFile(String(config.secrets?.localEncrypted?.keyFilePath ?? path.join(path.dirname(this.paths.configPath), "secrets", "master.key")), keyFile); + config.secrets = { ...config.secrets, localEncrypted: { ...config.secrets?.localEncrypted, keyFilePath: keyFile } }; + config.logging = { ...config.logging, logDir: path.join(instanceRoot, "logs") }; + const configPath = path.join(instanceRoot, "config.json"); + await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, { mode: 0o600 }); + this.timeline.add("k15_fresh_home", { faultId, generation, home }); + return { home, configPath }; + } + + /** + * `--k15-overlap-seconds`: the staging deploy shape. The next generation boots beside the old server on its own + * port; once it is ready, the edge sends new connections to it; the old server keeps running for the overlap and + * then gets SIGTERM. There is no outage. + */ + private async rollingRedeploy(previous: ChildProcess, overlapMs: number, receipt: FaultReceipt): Promise { + if (!this.edge) throw new Error("--k15-overlap-seconds needs the edge proxy started with the wave"); + const fromPort = this.backendPort; + receipt.redeploy = await this.advanceDeploy(receipt.faultId); + receipt.restartedAt = Date.now(); + this.timeline.add("fault_restarting", { faultId: receipt.faultId, overlapMs }); + const next = await this.spawnServer(); + // Bound to the next generation: the old server keeps serving during the overlap and must not count hits. + this.armPendingRecoveryStep(receipt, next.pid); + const backendUrl = `http://127.0.0.1:${this.backendPort}`; + receipt.healthyAt = await this.waitForHealth(next, receipt.phases, backendUrl); + const deadline = Date.now() + READY_TIMEOUT_MS; + while (await this.readPhase(backendUrl) !== "ready") { + if (this.shuttingDown) throw new Error("Harness shutdown interrupted the rolling redeploy"); + if (this.exited(next)) throw new Error(`Next deploy exited during startup (code=${next.exitCode} signal=${next.signalCode})`); + if (Date.now() > deadline) throw new Error(`Next deploy did not report ready within ${READY_TIMEOUT_MS}ms`); + await delay(250); + } + this.edge.target = this.backendPort; + this.timeline.add("k15_traffic_switched", { faultId: receipt.faultId, fromPort, toPort: this.backendPort, overlapMs }); + for (const until = Date.now() + overlapMs; Date.now() < until && !this.shuttingDown;) await delay(Math.min(1_000, until - Date.now())); + await this.stopper(previous, "SIGTERM"); + receipt.stoppedAt = Date.now(); + this.timeline.add("fault_server_down", { faultId: receipt.faultId, previousPid: previous.pid, overlapMs }); + await this.redeployEvidence?.(receipt.faultId, "stopped"); + } + + /** `--fail-recovery-step`: arm the step in fail mode for the wave's first restart fault only. */ + armRecoveryStepOnFirstFault(spec: string, control: CrashPointControl): void { + this.useCrashPointControl(control); + this.pendingRecoveryStep = spec; + } + + /** + * Arms the step for the replacement process only (`pid=`), and disarms the control file once its receipt + * landed, so a later server (RR, K6, another task's restart) never fails the step again. + */ + private armPendingRecoveryStep(receipt: FaultReceipt, pid: number | undefined): void { + if (!this.pendingRecoveryStep || !pid || !this.crashPointControl) return; + const token = `rs-${receipt.faultId}`; + this.armCrashPoint(token, `${this.pendingRecoveryStep} fail pid=${pid}`); + receipt.recoveryStep = { spec: this.pendingRecoveryStep, token }; + this.pendingRecoveryStep = null; + const receiptPath = this.crashPointControl.receiptPath; + const watch = setInterval(() => { + let text = ""; + try { text = readFileSync(receiptPath, "utf8"); } catch { return; } + if (!text.includes(`"token":"${token}"`)) return; + clearInterval(watch); + this.disarmCrashPoint(); + this.timeline.add("recovery_step_disarmed", { faultId: receipt.faultId, token }); + }, 500); + watch.unref(); + this.recoveryStepWatch = watch; + } + /** * K14: move the server from the older build to this checkout. Only the first K14 switches; a later * one restarts the current build (recorded with `switched: false`). @@ -834,8 +945,10 @@ export class ChaosSupervisor { async stop(): Promise { this.shuttingDown = true; + if (this.recoveryStepWatch) clearInterval(this.recoveryStepWatch); const errors: string[] = []; try { await this.stopper.stopAll(); } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); } + await this.edge?.close().catch(error => { errors.push(error instanceof Error ? error.message : String(error)); }); try { await this.database?.cleanup(); } catch (error) { errors.push(error instanceof Error ? error.message : String(error)); } await new Promise(resolve => this.log.end(resolve)); this.timeline.add("supervisor_stopped", { errors }); diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 7c28ff11f6c..efd17f66eb0 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -32,8 +32,9 @@ export type TaskFaultKind = FaultKind | "K13" | "K0" | ProviderFaultKind | DiskF * L1: high-output, high-accounting run for database contention (contention.ts). * F1: an absolute symlink pointing out of the repository workspace, kept at rest (staging save-back class F). * T1: a 100-second shell tool in the agent's workspace, so a fault lands while the remote Runner runs a tool (staging class R). + * W2, W2Q, W2C: the same task woken again 0–60 s after its run succeeded (rewake.ts), many tasks on one agent. */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1", "Q1", "L1", "Q2", "Q3", "Q4", "F1", "T1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1", "Q1", "L1", "Q2", "Q3", "Q4", "F1", "T1", "W2", "W2Q", "W2C"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; @@ -77,8 +78,21 @@ export interface WaveOptions { daytonaInherit?: boolean; /** `--k4-hold-max-seconds`: upper end of the seeded K4 hold (default 300 s, cap 600 s). */ k4HoldMaxMs?: number; - /** `--k15-hold-seconds`: the seeded K15 outage range (redeploy.ts); absent restarts at once. */ + /** `--k15-hold-seconds`: the seeded K15 outage range (redeploy.ts); absent restarts at once. With `--k15-overlap-seconds`, the overlap range. */ k15HoldMs?: [number, number]; + /** `--k15-overlap-seconds`: a K15 boots the next deploy beside the old server and SIGTERMs the old one after the seeded overlap (redeploy.ts). */ + k15Overlap?: boolean; + /** `--k15-fresh-home`: each deploy generation gets a new, empty instance home and no volume (redeploy.ts). */ + k15FreshHome?: boolean; + /** `--faults K0`: cells with no fault, for workloads whose shape is the fault (W2, W2Q, W2C). */ + baseline?: boolean; + /** + * `--fail-recovery-step [:]` (recovery-step-faults.ts): once the wave's first restart fault fired, the + * nth hit of that crash point in the replacement throws its step's own error, once. + */ + failRecoveryStep?: string; + /** `--long-turn-minutes`: every task's first turn does this much real tool work before its workload (long-turn.ts). */ + longTurnMinutes?: number; /** `--tm1-hold-seconds`: the seeded TM1 stall range (time-faults-config.ts); absent uses 95–150 s. */ tm1HoldMs?: [number, number]; faultAfterMs?: number; @@ -190,6 +204,8 @@ export interface TaskPlan { providerFault?: ProviderFaultRule; /** RR1–RR3: the seeded restarts after the first kill. */ restartSteps?: RestartStep[]; + /** `--long-turn-minutes`: the first turn's tool work before the workload (long-turn.ts). */ + longTurnMinutes?: number; } export interface ScheduledFault { @@ -305,6 +321,8 @@ export interface FaultReceipt { upgrade?: { fromRoot: string; fromSha: string | null; toRoot: string; toSha: string | null; switched: boolean }; /** K15: the deploy generation, emulated host name and deployment id before and after. */ redeploy?: ReturnType; + /** `--fail-recovery-step`: the step armed (fail mode) when this fault's replacement started. */ + recoveryStep?: { spec: string; token: string }; /** TM1–TM3: what was frozen and which rows were aged. */ timeFault?: TimeFaultRecord; /** RR1–RR3: every stop and replacement of the sequence, the first kill included. */ @@ -341,6 +359,10 @@ export interface OracleArtifacts { contention?: import("./contention.js").ContentionEvidence; /** Q2, Q3: what the board did and what the task, its runs and the harness's app saw. */ journey?: import("./journeys.js").JourneyArtifacts; + /** W2, W2Q, W2C: the re-wake and the task's runs (rewake.ts). */ + rewake?: import("./rewake.js").RewakeArtifacts; + /** `--long-turn-minutes`: the task's runs, for the harness's own long-turn evidence (long-turn.ts). */ + runs?: import("./rewake.js").RewakeArtifacts["runs"]; } /** What the board did for a governance task (G1, H1), and what the observer saw. */ @@ -429,6 +451,8 @@ export interface TaskRecord { question?: QuestionRecord; /** Q2, Q3: the board's journey actions (journey-driver.ts). */ journey?: import("./journeys.js").JourneyRecord; + /** W2, W2Q, W2C: the first run, the board's wake and the run that served it (rewake.ts). */ + rewake?: import("./rewake.js").RewakeRecord; /** PV with `--provider-fault-restart`: when the shim logged the task's fault (arms the `provider` trigger). */ providerFaultAppliedAt?: number; /** DF1: when the fault was armed and cleared, and what the board showed at the disk-full rest. */ diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index 73cf39825fe..e54ebda7f2f 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -1,6 +1,7 @@ import { questionHandoffWorkload, questionSequenceWorkload } from "./question-handoff.js"; import { l1Workload } from "./contention.js"; import { journeyWorkloadDefinitions } from "./journeys.js"; +import { rewakeWorkloadDefinitions } from "./rewake.js"; import type { OracleArtifacts, OracleResult, WorkloadId } from "./types.js"; /** Deterministic bench output: sum of squares 1..240. */ @@ -218,6 +219,7 @@ export const workloadDefinitions: Record = { Q4: questionSequenceWorkload, L1: l1Workload, ...journeyWorkloadDefinitions, + ...rewakeWorkloadDefinitions, G1: { id: "G1", label: "Approval-gated action across a restart", diff --git a/tests/runner-e2e/source-sdk-public-inputs.json b/tests/runner-e2e/source-sdk-public-inputs.json index 3ef601ea748..3ca35ccbe57 100644 --- a/tests/runner-e2e/source-sdk-public-inputs.json +++ b/tests/runner-e2e/source-sdk-public-inputs.json @@ -226,9 +226,9 @@ "packages/shared/src/connector-identity.test.ts": "d58357707faefa7c98bb906cdaeb4db39d31081a", "packages/shared/src/connector-identity.ts": "5ede83c9c5225dbe8af9cf3a49c74e477ef5b955", "packages/shared/src/constants.ts": "3149ee1cc5ffa34b50ecd493276b3a70fc46adc2", - "packages/shared/src/crash-point-registry.ts": "8d6f60e8ad8f90338fb3047955bb32160f50b9ae", - "packages/shared/src/crash-points.test.ts": "804a3acfd7a51010f31dbc3fb9a4dca771562c99", - "packages/shared/src/crash-points.ts": "d9f31d95c78f32290190d7728f42cd9573681fc8", + "packages/shared/src/crash-point-registry.ts": "6e9694699cb8de9c934f92170b9cfcab2eed4a1a", + "packages/shared/src/crash-points.test.ts": "3564a1f62d22f54a3c2fb0a72b3a6cde17c957d2", + "packages/shared/src/crash-points.ts": "c15259f39610e352eac5423a491548ec1bae94fe", "packages/shared/src/customer-success.ts": "a2d02aaa8223164c0eae4267b2a6da82242eaa6f", "packages/shared/src/decision-models.ts": "a07aefd87d2fae34283ce21a0334d57e2108aa7e", "packages/shared/src/decision.test.ts": "d0783d18613c73102f662e8a5da795ff6e7faed5",