diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f798686a118..1bf347ec6ba 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -79,6 +79,7 @@ import { } from "@paperclipai/paperclip-runner"; import { existsSync } from "node:fs"; import { createHash, generateKeyPairSync, randomUUID } from "node:crypto"; +import { createServer as createNetServer } from "node:net"; import { DatabaseSync } from "node:sqlite"; import { nativeSha256 } from "./canonical.js"; import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; @@ -1393,6 +1394,54 @@ describe("remote runner process supervision", () => { } }); + it("holds the runner listen port once per runner before its first launch under the chaos bind fault", async () => { + vi.stubEnv("PAPERCLIP_TEST_RUNNER_BIND_FAULT", "first_launch"); + onTestFinished(() => { vi.unstubAllEnvs(); }); + const port = await new Promise((resolvePort, reject) => { + const probe = createNetServer().listen(0, "127.0.0.1", () => { + const address = probe.address(); + probe.close(() => typeof address === "object" && address ? resolvePort(address.port) : reject(new Error("no port"))); + }); + }); + const execute = vi.fn(async (request: { args: string[] }) => { + if (request.args[2] === "paperclip-runner-test-bind-fault") { + return { exitCode: spawnSync("sh", request.args, { stdio: "ignore" }).status ?? 1, timedOut: false, stdout: "", stderr: "" }; + } + return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; + }); + const onLog = vi.fn(async () => undefined); + const launcher = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: `runner-bind-fault-${randomUUID()}`, + onLog, + }); + const spec = { command: "/controller/runnerd", args: ["--listen-address", "127.0.0.1", "--listen-port", String(port)], + cwd: "/controller", environment: {} }; + await expect(launcher(spec).completion).rejects.toThrow("runner_remote_process_launch_failed"); + const bindFailure = await new Promise(resolveBind => { + const contender = createNetServer(); + contender.once("error", (error: NodeJS.ErrnoException) => resolveBind(error.code ?? "error")); + contender.listen(port, "127.0.0.1", () => contender.close(() => resolveBind(null))); + }); + expect(bindFailure).toBe("EADDRINUSE"); + expect(onLog).toHaveBeenCalledWith("stderr", `[paperclip-runner] test bind fault armed on port ${port}\n`); + await expect(launcher(spec).completion).rejects.toThrow("runner_remote_process_launch_failed"); + expect(execute.mock.calls.filter(([request]) => request.args[2] === "paperclip-runner-test-bind-fault")).toHaveLength(1); + expect(execute.mock.calls.map(([request]) => request.args[2])).toEqual([ + "paperclip-runner-test-bind-fault", "paperclip-runner-launch", "paperclip-runner-launch"]); + const blockedLog = vi.fn(async () => undefined); + await expect(createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, + remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: `runner-bind-fault-${randomUUID()}`, + onLog: blockedLog, + })(spec).completion).rejects.toThrow("runner_remote_process_launch_failed"); + expect(blockedLog).toHaveBeenCalledWith("stderr", `[paperclip-runner] test bind fault failed on port ${port}\n`); + }); + it("confirms no replacement process exists when staging fails before the launch RPC", async () => { // A full sandbox disk rejects the runtime-context upload before dispatch. const stagingError = new Error( @@ -21059,9 +21108,11 @@ describe("runnerd provider runtime wiring", () => { ...["resumed", "replacement"].map(outcome => ({ outcome, scenario: "fresh", homeMode: "preconfigured" })), ...["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_unproven", "interrupted_bootstrap_symlink_parent", "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_unproven", "interrupted_bootstrap_unopened_other_run", - "interrupted_bootstrap_unopened_active_turn"] + "interrupted_bootstrap_unopened_active_turn", "interrupted_bootstrap_unprepared", "interrupted_bootstrap_unprepared_unproven", + "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", "interrupted_bootstrap_unprepared_backup"] .map(scenario => ({ outcome: "resumed", scenario, homeMode: "isolated" })), { outcome: "created", scenario: "interrupted_bootstrap", homeMode: "isolated" }, + { outcome: "created", scenario: "interrupted_bootstrap_unprepared", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_symlink_parent", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened", homeMode: "isolated" }, { outcome: "created", scenario: "interrupted_bootstrap_unopened_active_turn", homeMode: "isolated" }, @@ -21109,6 +21160,8 @@ describe("runnerd provider runtime wiring", () => { try { execFileSync("sh", command.args, { stdio: "pipe" }); return { exitCode: 0, timedOut: false, stdout: "", stderr: "" }; } catch { return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; } } + if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unprepared") + && command.args[1].includes("codex-provider-state.json")) return { exitCode: 1, timedOut: false, stdout: "", stderr: "" }; if (command.args?.[1]?.includes("base64") && scenario.startsWith("interrupted_bootstrap_unopened") && command.args[1].includes("codex-provider-state.json") && !command.args[1].includes("runner-state.json")) return { exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ @@ -21120,7 +21173,8 @@ describe("runnerd provider runtime wiring", () => { exitCode: 0, timedOut: false, stderr: "", stdout: Buffer.from(JSON.stringify({ runnerInstanceId: "runner-new-in-retained-workspace", normalizedSessionId: execution.session.normalizedSessionId, runId: scenario.endsWith("_other_run") ? "previous-warm-run" : execution.binding.runId, - lifecycle: scenario.startsWith("interrupted_bootstrap_unopened") ? "suspended" : "ready", + lifecycle: scenario.startsWith("interrupted_bootstrap_unopened") || scenario.endsWith("_suspended") ? "suspended" + : scenario.startsWith("interrupted_bootstrap_unprepared") ? "connecting" : "ready", outbox: [], pendingTerminalDelivery: null })).toString("base64"), }; if (command.args?.[1]?.includes("base64")) return { @@ -21208,15 +21262,17 @@ describe("runnerd provider runtime wiring", () => { if (scenario === "missing_control_plane") { await rm(join(options.stateDirectory!, "control-plane", "control-plane-state.json")); } - if (scenario === "backup_present") { + if (scenario === "backup_present" || scenario === "interrupted_bootstrap_unprepared_backup") { await mkdir(join(options.stateDirectory!, "failover-backups", "current"), { recursive: true }); await writeFile(join(options.stateDirectory!, "failover-backups", "current", "manifest.json"), "{}"); } if (["interrupted_bootstrap", "interrupted_bootstrap_other_run", "interrupted_bootstrap_symlink_parent", - "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_other_run", "interrupted_bootstrap_unopened_active_turn"].includes(scenario)) { + "interrupted_bootstrap_unopened", "interrupted_bootstrap_unopened_other_run", "interrupted_bootstrap_unopened_active_turn", + "interrupted_bootstrap_unprepared", "interrupted_bootstrap_unprepared_other_run", "interrupted_bootstrap_unprepared_suspended", + "interrupted_bootstrap_unprepared_backup"].includes(scenario)) { noteRemoteBootstrapRestartProven(execution.binding.runId, "stopped"); } - if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened") { + if (scenario === "interrupted_bootstrap" || scenario === "interrupted_bootstrap_unopened" || scenario === "interrupted_bootstrap_unprepared") { await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); const siblings = await readdir(dirname(sessionRoot)); expect(siblings.filter(name => name.startsWith(`${basename(sessionRoot)}.bootstrap-restart-`))).toHaveLength(1); @@ -21276,7 +21332,9 @@ describe("runnerd provider runtime wiring", () => { expect(remoteExecute.mock.calls.filter(([command]) => command.args?.[2] === "paperclip-runner-claim-unstarted-session")).toHaveLength(1); } else { await expect(options.prepareExternalRunnerState()).rejects.toThrow( - outcome === "replacement" ? "runner_harness_backup_unavailable" : "runner_harness_state_mismatch", + outcome === "replacement" ? "runner_harness_backup_unavailable" + : scenario.startsWith("interrupted_bootstrap_unprepared") ? "runner_harness_state_mismatch: resumed_unavailable" + : "runner_harness_state_mismatch", ); expect(remoteExecute.mock.calls.some(([command]) => command.args?.[1]?.includes("install -d"))).toBe(false); expect(syncIn).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index c6d56c6d8af..e6f6e4e729c 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -15113,6 +15113,29 @@ async function cleanupRemoteRunnerAfterIdentityFailure(input: { return result?.exitCode === 0 && result.timedOut === false; } +const injectedRunnerBindConflicts = new Set(); + +async function injectRunnerBindConflictForTest(runner: CommandManagedRuntimeRunner, runnerInstanceId: string, + args: readonly string[], onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise): Promise { + if (process.env.NODE_ENV !== "test" || process.env.PAPERCLIP_TEST_RUNNER_BIND_FAULT !== "first_launch" + || injectedRunnerBindConflicts.has(runnerInstanceId)) return; + const option = (name: string) => args.includes(name) ? args[args.indexOf(name) + 1] ?? null : null; + const port = option("--listen-port"); + const address = option("--listen-address") ?? "127.0.0.1"; + if (!port || !/^\d+$/.test(port)) return; + injectedRunnerBindConflicts.add(runnerInstanceId); + const result = await runner.execute({ + command: "sh", + args: ["-c", + 'ready=$(mktemp); rm -f "$ready"; nohup node -e "const s = require(\'net\').createServer().listen(Number(process.argv[1]), process.argv[2], () => { require(\'fs\').writeFileSync(process.argv[3], \'\'); setTimeout(() => s.close(), 15000); })" "$1" "$2" "$ready" /dev/null 2>&1 & i=0; while [ ! -e "$ready" ] && [ $i -lt 50 ]; do sleep 0.1; i=$((i + 1)); done; test -e "$ready" && rm -f "$ready"', + "paperclip-runner-test-bind-fault", port, address], + bypassSession: true, + timeoutMs: 10_000, + }).catch(() => null); + const armed = result?.exitCode === 0 && !result.timedOut; + await onLog?.("stderr", `[paperclip-runner] test bind fault ${armed ? "armed" : "failed"} on port ${port}\n`).catch(() => undefined); +} + export function createRemoteRunnerProcessLauncher(input: { target: Extract; runner: CommandManagedRuntimeRunner; @@ -15251,6 +15274,7 @@ export function createRemoteRunnerProcessLauncher(input: { // into its own session instead; its own bounded diagnostics directory and // durable PRP state remain the authorities, and the controller monitors // the exact persisted process identity below. + await injectRunnerBindConflictForTest(runner, input.runnerInstanceId, remoteArgs, input.onLog); launchDispatched = true; const launchResult = await runner.execute({ command: "sh", @@ -16525,6 +16549,18 @@ async function createRunnerdBackendWithinSessionClaim( const restartableBootstrapState = (state: { incompleteReason: string | null; notSuspendedRunId?: string | null }) => (state.incompleteReason === "not_suspended" || state.incompleteReason === "provider_unopened") && provenOwnBootstrapRestart(state.notSuspendedRunId); + const inspectOwnUnpreparedBootstrap = async (): Promise => { + if (!remoteCommandRunner || !remoteStateDirectory || archivedRotationRunnerStatePath + || !provenOwnBootstrapRestart(input.execution.binding.runId) + || harnessBackupCandidates(root).some(candidate => existsSync(resolve(candidate, "manifest.json")))) return null; + const runnerState = await readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory }) + .then(record, () => null); + if (!runnerState || runnerState.runnerInstanceId !== input.runnerInstanceId + || runnerState.normalizedSessionId !== nativeSessionKey(input.execution) + || typeof runnerState.lifecycle !== "string" || runnerState.lifecycle === "suspended" + || !provenOwnBootstrapRestart(typeof runnerState.runId === "string" ? runnerState.runId : null)) return null; + return input.execution.binding.runId; + }; const inspectRemoteHarnessState = async ( source: "live" | "host_preparation" = "live", ): Promise<{ @@ -16569,6 +16605,16 @@ async function createRunnerdBackendWithinSessionClaim( timeoutMs: 10_000, }); if (inspected.exitCode !== 0 || inspected.timedOut) { + const unpreparedBootstrapRunId = source === "host_preparation" ? await inspectOwnUnpreparedBootstrap() : null; + if (unpreparedBootstrapRunId) { + return { + complete: false, + runnerState: null, + providerSessionIdentity: null, + incompleteReason: "not_suspended", + notSuspendedRunId: unpreparedBootstrapRunId, + }; + } return { complete: false, runnerState: null, diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index d270b74ba1a..f3302cc4f4f 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -26,7 +26,7 @@ Options: `--triggers time,comment,retry,queued,tool,result` (fault timing, below default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance), `--k4-hold-max-seconds` (raise the seeded K4 hold upper end from 300 s, up to the 600 s outage cap). `--faults K0` plans cells with no fault, for workloads whose shape is the fault (W2, below). `--fail-recovery-step`, -`--long-turn-minutes`, `--k15-overlap-seconds`, `--k15-fresh-home` and `--ephemeral-server` are described in their own sections. +`--long-turn-minutes`, `--k15-overlap-seconds`, `--k15-fresh-home`, `--ephemeral-server` and `--runner-bind-fault` are described in their own sections. `--fault-after-seconds ` fires every planned fault `n` seconds after its task launches (5–3600; with `--triggers time`), keeping the rest of the seeded plan, for example to restart a Daytona R1 task after its workspace preparation. `--matrix` crosses every selected workload with every fault kind (below), and `--server-from ` @@ -518,6 +518,24 @@ workloads and refused when listed. loses the agents' instruction bundles, which staging keeps in Postgres and object storage. Every restart replays the server's startup eligibility check, so a replacement that refuses to start fails the wave. +### Runner exits before session.open (`--runner-bind-fault`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona --daytona-image $IMG --daytona-warm --storage s3 \ + --ephemeral-server --adapters claude --workloads W2 --faults K1,K2,K15 --k15-fresh-home --runner-bind-fault --seed 6402 +``` + +Staging's volume-free shape (wave `w-20261111T230537-s6402`) showed a reused warm sandbox whose Runner listen port +was still held by another Runner. The new Runner wrote `runner-state.json` (`lifecycle: connecting`), failed with +`runner_ingress_bind_conflict` and exited before `session.open`, so it never wrote its provider state. The same-run +bootstrap retry then found that partial session root and failed with `runner_harness_state_mismatch: resumed_unavailable`. +`--runner-bind-fault` gives every server of the wave `PAPERCLIP_TEST_RUNNER_BIND_FAULT=first_launch` (honoured only +with `NODE_ENV=test`): before a run's first Runner launch on that server, a short-lived process in the sandbox holds +the Runner's listen port for 15 s. The server waits until that listener has bound the port and writes +`[paperclip-runner] test bind fault armed on port ` (or `failed`) to the run log, so a wave can count the runs that +really took this path. Every run then takes it once per server generation, and the run must still finish without +Retry. It needs `--env daytona`. + ## Absolute symlink in the workspace (F1) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index f52210945b2..84ccf192008 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -145,6 +145,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "k15-overlap-seconds": { type: "string" }, "k15-fresh-home": { type: "boolean" }, "ephemeral-server": { type: "boolean" }, + "runner-bind-fault": { type: "boolean" }, "tm1-hold-seconds": { type: "string" }, "fail-recovery-step": { type: "string" }, "long-turn-minutes": { type: "string" }, @@ -158,7 +159,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-pack-mismatch", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "k15-overlap-seconds", "k15-fresh-home", "ephemeral-server", "tm1-hold-seconds", "fail-recovery-step", "long-turn-minutes", "cr1-hold-seconds", "cr2-hold-seconds", "cr3-stall-seconds", "sandbox-watch-seconds"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-pack-mismatch", "daytona-cpu", "daytona-reuse-lease", "daytona-warm", "daytona-inherit", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from", "workspace-repo", "storage", "storage-faults", "storage-fault-seconds", "storage-trigger", "native-trigger", "native-trigger-outage", "fault-after-seconds", "provider-fault-message", "provider-fault-delays", "db-pressure", "matrix", "server-from", "time-age-hours", "provider-fault-restart", "provider-fault-restart-after-seconds", "rr-gap-seconds", "rr-stop-on", "k15-hold-seconds", "k15-overlap-seconds", "k15-fresh-home", "ephemeral-server", "runner-bind-fault", "tm1-hold-seconds", "fail-recovery-step", "long-turn-minutes", "cr1-hold-seconds", "cr2-hold-seconds", "cr3-stall-seconds", "sandbox-watch-seconds"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -396,6 +397,10 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC if (repository.length) throw new Error(`${flag} cannot run ${repository.join(", ")}: the fixture repository is a local_path project source, which a volume-free server refuses; pick other --workloads`); options.ephemeralServer = true; } + if (values["runner-bind-fault"]) { + if (env !== "daytona") throw new Error("--runner-bind-fault holds a port inside a remote sandbox: run it with --env daytona"); + options.runnerBindFault = true; + } if (values["storage-faults"] !== undefined) { if (storage !== "s3") throw new Error("--storage-faults needs --storage s3"); options.storageFaults = list(values["storage-faults"], STORAGE_FAULT_KINDS, "storage-faults", []); diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index ded4a0efb10..16dc725ab9d 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -52,7 +52,7 @@ import type { RestartStep } from "./repeated-restarts.js"; import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timeline.js"; import { guardedRunnerOutputs, worktreeFingerprint } from "./wave-guard.js"; import { checkCrossServer, readCrossServerEvidence } from "./multi-server.js"; -import { sessionFaultServerEnvironment } from "./session-faults.js"; +import { runnerBindFaultServerEnvironment, sessionFaultServerEnvironment } from "./session-faults.js"; import { controllerLogNames, gradeWarmShutdown, isWarmWave, warmShutdownMarkdown, type WarmShutdownCleanup } from "./warm-shutdown.js"; import type { FaultKind, FaultReceipt, OracleArtifacts, TriggerId, WavePlan } from "./types.js"; @@ -427,6 +427,7 @@ async function main(): Promise { if (plan.options.timeAgeMs) supervisor.timeAgeMs = plan.options.timeAgeMs; if (plan.options.rrStopOn) supervisor.rrStopOn = plan.options.rrStopOn; Object.assign(serverEnvironment, sessionFaultServerEnvironment(plan.options.sessionFaults)); + Object.assign(serverEnvironment, runnerBindFaultServerEnvironment(plan.options.runnerBindFault)); await supervisor.start(serverEnvironment); // TM2/TM3 age the database; the observer and grader read run times back at their real times. const api: ChaosApi = new TimeShiftedApi(supervisor.baseUrl, supervisor.agingShifts); diff --git a/tests/restart-chaos/runner-bind-fault.test.ts b/tests/restart-chaos/runner-bind-fault.test.ts new file mode 100644 index 00000000000..b9ba331d225 --- /dev/null +++ b/tests/restart-chaos/runner-bind-fault.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { parseCommand } from "./config.js"; +import { runnerBindFaultServerEnvironment } from "./session-faults.js"; + +const image = `img@sha256:${"a".repeat(64)}`; + +describe("--runner-bind-fault", () => { + it("arms the server's first-launch bind fault on the s6402 volume-free shape", () => { + const command = parseCommand(["--env", "daytona", "--daytona-image", image, "--daytona-warm", "--storage", "s3", + "--ephemeral-server", "--adapters", "claude", "--workloads", "W2", "--faults", "K1,K2,K15", "--k15-fresh-home", + "--seed", "6402", "--runner-bind-fault"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.runnerBindFault).toBe(true); + expect(runnerBindFaultServerEnvironment(command.options.runnerBindFault)) + .toEqual({ PAPERCLIP_TEST_RUNNER_BIND_FAULT: "first_launch" }); + }); + + it("leaves the server unarmed without the flag and refuses a local wave", () => { + const command = parseCommand(["--env", "daytona", "--daytona-image", image, "--workloads", "W2", "--faults", "K2"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.runnerBindFault).toBeUndefined(); + expect(runnerBindFaultServerEnvironment(command.options.runnerBindFault)).toEqual({}); + expect(() => parseCommand(["--workloads", "W2", "--faults", "K2", "--runner-bind-fault"])).toThrow("--env daytona"); + }); +}); diff --git a/tests/restart-chaos/session-faults.ts b/tests/restart-chaos/session-faults.ts index 4f51f9dca75..923607d3c4b 100644 --- a/tests/restart-chaos/session-faults.ts +++ b/tests/restart-chaos/session-faults.ts @@ -36,6 +36,10 @@ export function sessionFaultServerEnvironment(faults: readonly SessionFaultKind[ return faults?.includes("SR1U") ? { PAPERCLIP_TEST_WARM_RETIREMENT_FAULT: "unconfirmed" } : {}; } +export function runnerBindFaultServerEnvironment(armed: boolean | undefined): Record { + return armed ? { PAPERCLIP_TEST_RUNNER_BIND_FAULT: "first_launch" } : {}; +} + /** What the board changed, recorded on the task's question. */ export interface SessionRotation { at: number; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 8ad98fd558b..be413df9d46 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -87,6 +87,7 @@ export interface WaveOptions { /** `--k15-fresh-home`: each deploy generation gets a new, empty instance home and no volume (redeploy.ts). */ k15FreshHome?: boolean; ephemeralServer?: boolean; + runnerBindFault?: boolean; /** `--faults K0`: cells with no fault, for workloads whose shape is the fault (W2, W2Q, W2C). */ baseline?: boolean; /**