diff --git a/packages/shared/src/crash-point-registry.ts b/packages/shared/src/crash-point-registry.ts new file mode 100644 index 00000000000..b796ac92cfc --- /dev/null +++ b/packages/shared/src/crash-point-registry.ts @@ -0,0 +1,155 @@ +/** + * Registry of named crash points (durable ownership plan, "Testing: enumerate + * crash points"). Browser-safe data only; the helper is in `crash-points.ts`. + * + * Each point marks one durable write boundary. Add a point by adding an entry + * here and one `crashPoint("")` call at the boundary; see + * tests/restart-chaos/README.md, "Crash-point suite". + * + * Name: `.._`, lower snake case. + * `kinds` are ownable kind ids from server/src/services/recovery/ownership-registry.ts + * (the Phase 1 registry of doc/plans/2026-11-07-ownership-inventory.md). `invariants` are the plan's + * I1–I8 that a crash here puts at risk. `environments` are where a suite task + * reaches the point (a local run never creates a sandbox, for example). + */ + +export type CrashPointInvariant = "I1" | "I2" | "I3" | "I4" | "I5" | "I6" | "I7" | "I8"; +export type CrashPointEnvironment = "local" | "daytona"; + +export type CrashPointDefinition = { + description: string; + environments: readonly CrashPointEnvironment[]; + kinds: readonly string[]; + invariants: readonly CrashPointInvariant[]; +}; + +export const CRASH_POINT_NAME_PATTERN = /^[a-z][a-z0-9_]*\.[a-z][a-z0-9_]*\.(before|after)_[a-z0-9_]+$/; + +export const CRASH_POINTS = { + "sandbox.create.after_intent_before_provider": { + description: "The sandbox_creation_operations intent is committed; the provider create call has not been made. Recovery must settle the intent without a sandbox to find.", + environments: ["daytona"], + kinds: ["sandbox_creation_intent"], + invariants: ["I1", "I2", "I7"], + }, + "sandbox.create.after_provider_before_lease": { + description: "The provider created the sandbox; no environment lease row records it yet. Recovery must adopt or destroy the sandbox from the intent, never leak it.", + environments: ["daytona"], + kinds: ["sandbox_creation_intent", "environment_run_lease_active"], + invariants: ["I2", "I7"], + }, + "native.process_start.after_requested_before_spawn": { + description: "native.process_start_requested is in the run log; runnerd has not been spawned. No provider work started, so a resumed attempt must not be charged.", + environments: ["local", "daytona"], + kinds: ["native_startup_context", "native_executor_claim"], + invariants: ["I2", "I5", "I7"], + }, + "runner.process.after_spawn_before_identity": { + description: "runnerd is spawned (own process group, survives the server); its pid and process identity are not yet recorded on the run. Recovery must find or stop the orphan by identity.", + environments: ["local", "daytona"], + kinds: ["native_executor_claim", "detached_process_marker"], + invariants: ["I1", "I2", "I5"], + }, + "runner.turn_start.after_runner_accept_before_controller_checkpoint": { + description: "runnerd returned the turn.start result; the controller's command journal still says pending. Recovery must adopt the accepted turn, not start a second one.", + environments: ["local", "daytona"], + kinds: ["native_executor_claim", "native_restart_recovery_claim"], + invariants: ["I4", "I5", "I6"], + }, + "runner.turn_start.after_controller_journal_before_db_checkpoint": { + description: "The controller journal recorded the turn.start result; the next send and (remote runs) the database checkpoint have not happened.", + environments: ["local", "daytona"], + kinds: ["native_restart_recovery_claim"], + invariants: ["I4", "I6"], + }, + "native.result.after_accept_before_finalize": { + description: "The run's result and its workspace_finalizing finalization row are committed; finalization has not started. The accepted result must be finalized exactly once.", + environments: ["local", "daytona"], + kinds: ["native_accepted_result_lease", "native_finalizer_lease"], + invariants: ["I2", "I4", "I7"], + }, + "native.finalize.after_decision_before_projection": { + description: "The status decision is committed; the run and issue projection is not. Recovery must finish the projection, not decide again.", + environments: ["local", "daytona"], + kinds: ["native_finalizer_lease"], + invariants: ["I2", "I7"], + }, + "workspace.finalize.after_start_before_copyback": { + description: "A workspace_finalize operation row is running; copyback or export has not started. Recovery must settle the operation and keep the workspace.", + environments: ["local", "daytona"], + kinds: ["native_workspace_finalize_attempt", "copyback_owner_hold"], + invariants: ["I1", "I2", "I7"], + }, + "workspace.finalize.after_copyback_before_done": { + description: "Copyback or export finished; the workspace operation row still says running. Recovery must settle it without copying again over newer work.", + environments: ["local", "daytona"], + kinds: ["native_workspace_finalize_attempt", "copyback_owner_hold"], + invariants: ["I2", "I7"], + }, + "environment.lease_release.after_provider_before_lease": { + description: "The provider released the sandbox; the environment lease row is still active.", + environments: ["daytona"], + kinds: ["environment_run_lease_active"], + invariants: ["I2"], + }, + "environment.teardown.after_claim_before_destroy": { + description: "A pending_cleanup teardown claim is held; the provider destroy has not been called. Proof of the dead claimant must release the claim, not a TTL.", + environments: ["daytona"], + kinds: ["environment_teardown_claim", "environment_pending_cleanup"], + invariants: ["I2", "I7"], + }, + "environment.teardown.after_destroy_before_lease": { + description: "A reusable sandbox was destroyed at the provider; its lease row is not released.", + environments: ["daytona"], + kinds: ["environment_run_lease_active", "environment_retained_hold"], + invariants: ["I2"], + }, + "environment.stop_retain.after_claim_before_stop": { + description: "A stop-and-retain intent and its claim are committed; the provider stop has not been retried.", + environments: ["daytona"], + kinds: ["environment_stop_and_retain_intent"], + invariants: ["I2", "I7"], + }, + "environment.lease.after_release_before_runtime_services": { + description: "The run's environment leases are released; its workspace runtime services are not.", + environments: ["local", "daytona"], + kinds: ["execution_workspace_runtime_lease", "workspace_runtime_service"], + invariants: ["I2"], + }, + "wake.claim.after_run_running_before_wakeup_claimed": { + description: "The claimed run is running; its wake request still says queued. The wake must be delivered exactly once.", + environments: ["local", "daytona"], + kinds: ["wake_request_claimed"], + invariants: ["I4", "I7"], + }, + "budget.reservation.after_hold_before_dispatch": { + description: "The run's budget reservation hold is committed; the run has not been dispatched. The hold must not gate admission or outlive the run.", + environments: ["local", "daytona"], + kinds: ["run_accounting_debt"], + invariants: ["I2", "I3", "I5"], + }, + "accounting.usage_receipt.after_spool_before_db": { + description: "A usage receipt is fsynced to the local spool; its database row is not written.", + environments: ["local", "daytona"], + kinds: ["usage_receipt_spool"], + invariants: ["I3", "I4"], + }, + "accounting.usage_receipt.after_db_before_unlink": { + description: "The usage receipt row is written; its spool file is not removed. Replaying the spool must not double-count.", + environments: ["local", "daytona"], + kinds: ["usage_receipt_spool"], + invariants: ["I4"], + }, + "accounting.closure.after_commit_before_spool_unlink": { + description: "Run accounting is closed (reservation settled, cost recorded); the recovered spool files are not removed.", + environments: ["local", "daytona"], + kinds: ["run_accounting_debt", "usage_receipt_spool"], + invariants: ["I3", "I4"], + }, +} as const satisfies Record; + +export type CrashPointName = keyof typeof CRASH_POINTS; + +export function isCrashPointName(value: string): value is CrashPointName { + return Object.prototype.hasOwnProperty.call(CRASH_POINTS, value); +} diff --git a/packages/shared/src/crash-points.test.ts b/packages/shared/src/crash-points.test.ts new file mode 100644 index 00000000000..9d745e28f5e --- /dev/null +++ b/packages/shared/src/crash-points.test.ts @@ -0,0 +1,191 @@ +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { + CRASH_POINTS, + CRASH_POINT_NAME_PATTERN, + CrashPointReached, + armCrashPointForTest, + armedCrashPoint, + crashPoint, + parseCrashPointSpec, + type CrashPointName, +} from "./crash-points.js"; + +const names = Object.keys(CRASH_POINTS) as CrashPointName[]; +const first = names[0]!; +const second = names[1]!; +let disarm: (() => void) | null = null; + +afterEach(() => { disarm?.(); disarm = null; }); + +describe("crash point registry", () => { + it("names every point .._ with a description", () => { + expect(names.length).toBeGreaterThan(0); + for (const name of names) { + expect(name).toMatch(CRASH_POINT_NAME_PATTERN); + const definition = CRASH_POINTS[name]; + expect(definition.description.length, name).toBeGreaterThan(20); + expect(definition.invariants.length, name).toBeGreaterThan(0); + expect(definition.environments.length, name).toBeGreaterThan(0); + } + }); +}); + +describe("parseCrashPointSpec", () => { + it("reads a name and an optional hit count", () => { + expect(parseCrashPointSpec(undefined)).toBeNull(); + expect(parseCrashPointSpec(" ")).toBeNull(); + expect(parseCrashPointSpec(first)).toEqual({ name: first, nth: 1 }); + expect(parseCrashPointSpec(`${first}:3`)).toEqual({ name: first, nth: 3 }); + }); + + it("refuses unknown names and bad hit counts", () => { + expect(() => parseCrashPointSpec("sandbox.create.after_typo")).toThrow(/unknown crash point/); + expect(() => parseCrashPointSpec(`${first}:0`)).toThrow(/positive integer/); + expect(() => parseCrashPointSpec(`${first}:x`)).toThrow(/positive integer/); + }); +}); + +describe("crashPoint in a test process", () => { + it("does nothing while disarmed", () => { + expect(armedCrashPoint()).toBeNull(); + expect(() => crashPoint(first)).not.toThrow(); + }); + + it("throws at the nth hit of the armed name only", () => { + disarm = armCrashPointForTest(`${first}:2`); + crashPoint(second); + crashPoint(first); + expect(armedCrashPoint()).toEqual({ name: first, nth: 2, hits: 1 }); + expect(() => crashPoint(first)).toThrow(CrashPointReached); + expect(() => crashPoint(first)).not.toThrow(); + }); + + it("refuses an object spec with a count that could never fire", () => { + for (const nth of [0, -1, 2.5, Number.NaN]) { + expect(() => armCrashPointForTest({ name: first, nth })).toThrow(/positive integer/); + } + expect(armedCrashPoint()).toBeNull(); + }); + + it("restores the previous state on disarm", () => { + const off = armCrashPointForTest(first); + off(); + expect(armedCrashPoint()).toBeNull(); + }); +}); + +// Each case starts a child Node process with the tsx loader; allow for a busy host. +describe("crashPoint armed by the environment", { timeout: 60_000 }, () => { + const script = (name: string) => [ + `const { crashPoint } = await import(${JSON.stringify(path.join(import.meta.dirname, "crash-points.ts"))});`, + `crashPoint(${JSON.stringify(name)}); process.stdout.write("one\\n");`, + `crashPoint(${JSON.stringify(name)}); process.stdout.write("two\\n");`, + ].join("\n"); + // The loader, not the tsx CLI: the CLI runs the script in a child process, so + // the SIGKILL would land on a grandchild. + const tsxLoader = pathToFileURL(path.resolve(import.meta.dirname, "../../../cli/node_modules/tsx/dist/loader.mjs")).href; + const runChild = (env: Record, source = script(first)) => { + const dir = mkdtempSync(path.join(os.tmpdir(), "crash-point-script-")); + try { + const file = path.join(dir, "child.mts"); + writeFileSync(file, source); + return spawnSync(process.execPath, ["--import", tsxLoader, file], { + env: { ...process.env, NODE_ENV: "test", ...env }, encoding: "utf8", timeout: 60_000, + }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }; + + it("SIGKILLs the process at the nth hit and appends a receipt", () => { + const dir = mkdtempSync(path.join(os.tmpdir(), "crash-point-")); + try { + const receipt = path.join(dir, "receipt.jsonl"); + const result = runChild({ PAPERCLIP_CRASH_POINT: `${first}:2`, PAPERCLIP_CRASH_POINT_RECEIPT: receipt }); + expect(result.signal).toBe("SIGKILL"); + expect(result.stdout).toBe("one\n"); + expect(result.stderr).toContain(`[crash-point] armed ${first}:2`); + expect(JSON.parse(readFileSync(receipt, "utf8").trim())).toMatchObject({ point: first, hit: 2, pid: result.pid }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("ignores hits until the gate file exists", () => { + const dir = mkdtempSync(path.join(os.tmpdir(), "crash-point-gate-")); + try { + const result = runChild({ PAPERCLIP_CRASH_POINT: first, PAPERCLIP_CRASH_POINT_GATE: path.join(dir, "open") }); + expect(result.signal).toBeNull(); + expect(result.stdout).toBe("one\ntwo\n"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("counts hits only after the gate file appears", () => { + const dir = mkdtempSync(path.join(os.tmpdir(), "crash-point-gate-")); + try { + const gate = path.join(dir, "open"); + const receipt = path.join(dir, "receipt.jsonl"); + const source = [ + `const { crashPoint } = await import(${JSON.stringify(path.join(import.meta.dirname, "crash-points.ts"))});`, + `const { writeFileSync } = await import("node:fs");`, + `crashPoint(${JSON.stringify(first)}); crashPoint(${JSON.stringify(first)}); process.stdout.write("closed\\n");`, + `writeFileSync(${JSON.stringify(gate)}, "");`, + `crashPoint(${JSON.stringify(first)}); process.stdout.write("one\\n");`, + `crashPoint(${JSON.stringify(first)}); process.stdout.write("two\\n");`, + ].join("\n"); + const result = runChild({ PAPERCLIP_CRASH_POINT: `${first}:2`, PAPERCLIP_CRASH_POINT_GATE: gate, + PAPERCLIP_CRASH_POINT_RECEIPT: receipt }, source); + expect(result.signal).toBe("SIGKILL"); + expect(result.stdout).toBe("closed\none\n"); + expect(JSON.parse(readFileSync(receipt, "utf8").trim())).toMatchObject({ point: first, hit: 2 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("stays disarmed in production", () => { + const result = runChild({ PAPERCLIP_CRASH_POINT: first, NODE_ENV: "production" }); + expect(result.signal).toBeNull(); + expect(result.status).toBe(0); + expect(result.stdout).toBe("one\ntwo\n"); + }); + + it("refuses an unknown name at import", () => { + const result = runChild({ PAPERCLIP_CRASH_POINT: "sandbox.create.after_typo" }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("unknown crash point"); + }); +}); + +describe("disarmed cost", () => { + // TTFT is a hard constraint (plan invariant 8): a disarmed point must cost + // about as much as an empty call. A regression that matters (a syscall or + // I/O on the disarmed path) costs microseconds, so the bound is 100 ns over + // an empty call: far above timer and load noise, far below any real cost. + // The measured figure is printed for the PR record. + it("stays within 100 ns of an empty call", () => { + const iterations = 2_000_000; + const empty = (_name: CrashPointName) => {}; + const time = (fn: (name: CrashPointName) => void) => { + for (let i = 0; i < 100_000; i += 1) fn(first); + const start = process.hrtime.bigint(); + for (let i = 0; i < iterations; i += 1) fn(i & 1 ? first : second); + return Number(process.hrtime.bigint() - start) / iterations; + }; + let baseline = Number.POSITIVE_INFINITY; + let measured = Number.POSITIVE_INFINITY; + for (let round = 0; round < 7; round += 1) { + baseline = Math.min(baseline, time(empty)); + measured = Math.min(measured, time(crashPoint)); + } + console.info(`crashPoint disarmed: ${measured.toFixed(2)} ns/call (empty call ${baseline.toFixed(2)} ns)`); + expect(measured).toBeLessThan(baseline + 100); + }); +}); diff --git a/packages/shared/src/crash-points.ts b/packages/shared/src/crash-points.ts new file mode 100644 index 00000000000..06d430e50f6 --- /dev/null +++ b/packages/shared/src/crash-points.ts @@ -0,0 +1,146 @@ +/** + * Named crash points (durable ownership plan, "Testing: enumerate crash points"). + * + * `crashPoint(name)` marks a durable write boundary: the gap between two durable + * writes, or between an outside effect and the record of its outcome. A restart + * that lands in that gap leaves an in-progress record owned by a dead process. + * The crash-point suite (`pnpm crash-points`) kills the control plane at each + * named gap on purpose instead of hoping random chaos timing finds it. + * + * Arming (test mode only): + * + * PAPERCLIP_CRASH_POINT=[:] SIGKILL this process at the nth hit (default 1) + * PAPERCLIP_CRASH_POINT_RECEIPT= optional; one JSON line is appended before the kill + * PAPERCLIP_CRASH_POINT_GATE= optional; hits before this file exists are not counted, + * so setup work (seeding, probes) cannot fire the point + * + * The variable is read once, when this module is first imported. It is ignored + * when `NODE_ENV=production`, and an unknown name throws at import so a typo + * cannot silently arm nothing. Disarmed, `crashPoint` is a single comparison + * against a module constant (see crash-points.test.ts for the cost check). + * + * In-process tests use `armCrashPointForTest`, which throws `CrashPointReached` + * at the hit instead of killing the test runner. A throw still runs `finally` + * blocks and rejects the caller's promise, so it only approximates a crash: a + * test that needs real process death runs the suite instead. + * + * This is test instrumentation, not Telemetry or Observability (AGENTS.md §5.7): + * nothing is recorded or sent anywhere unless a test arms a point. + */ +import { appendFileSync, existsSync } from "node:fs"; +import { CRASH_POINTS, isCrashPointName, type CrashPointName } from "./crash-point-registry.js"; + +export { CRASH_POINTS, CRASH_POINT_NAME_PATTERN, isCrashPointName } from "./crash-point-registry.js"; +export type { CrashPointDefinition, CrashPointInvariant, CrashPointName } from "./crash-point-registry.js"; + +export const CRASH_POINT_ENV = "PAPERCLIP_CRASH_POINT"; +export const CRASH_POINT_RECEIPT_ENV = "PAPERCLIP_CRASH_POINT_RECEIPT"; +export const CRASH_POINT_GATE_ENV = "PAPERCLIP_CRASH_POINT_GATE"; +/** Printed to stderr just before the kill; the suite greps server logs for it. */ +export const CRASH_POINT_LOG_PREFIX = "[crash-point]"; + +export type CrashPointSpec = { name: CrashPointName; nth: number }; + +export type CrashPointReceipt = { + point: CrashPointName; + hit: number; + pid: number; + at: string; +}; + +/** Thrown by a point armed with `armCrashPointForTest`. */ +export class CrashPointReached extends Error { + constructor(readonly point: CrashPointName, readonly hit: number) { + super(`crash point ${point} reached (hit ${hit})`); + this.name = "CrashPointReached"; + } +} + +/** Parses `[:]`. Returns null for an empty value; throws on a bad one. */ +export function parseCrashPointSpec(raw: string | undefined): CrashPointSpec | null { + const value = raw?.trim(); + if (!value) return null; + const separator = value.lastIndexOf(":"); + const name = separator === -1 ? value : value.slice(0, separator); + const nthText = separator === -1 ? "1" : value.slice(separator + 1); + if (!/^[1-9]\d{0,5}$/.test(nthText)) { + throw new Error(`${CRASH_POINT_ENV}=${value}: the hit count after ':' must be a positive integer`); + } + if (!isCrashPointName(name)) { + throw new Error(`${CRASH_POINT_ENV}=${value}: unknown crash point "${name}" (see packages/shared/src/crash-point-registry.ts)`); + } + return { name, nth: Number(nthText) }; +} + +type Armed = { + spec: CrashPointSpec; + hits: number; + mode: "kill" | "throw"; + receiptPath: string | null; + /** Hits count only once this file exists; null when ungated. */ + gatePath: string | null; +}; + +function armFromEnvironment(): Armed | null { + const env = typeof process === "undefined" ? undefined : process.env; + if (!env || env.NODE_ENV === "production") return null; + const spec = parseCrashPointSpec(env[CRASH_POINT_ENV]); + if (!spec) return null; + process.stderr.write(`${CRASH_POINT_LOG_PREFIX} armed ${spec.name}:${spec.nth} pid=${process.pid}\n`); + return { + spec, hits: 0, mode: "kill", + receiptPath: env[CRASH_POINT_RECEIPT_ENV]?.trim() || null, + gatePath: env[CRASH_POINT_GATE_ENV]?.trim() || null, + }; +} + +let armed: Armed | null = armFromEnvironment(); + +/** + * Marks a durable write boundary. Disarmed (always in production) it returns + * immediately. Armed for this name, the nth hit kills the process with SIGKILL + * (or throws `CrashPointReached` under `armCrashPointForTest`). + */ +export function crashPoint(name: CrashPointName): void { + if (armed === null) return; + if (armed.spec.name !== name) return; + reach(armed); +} + +function reach(current: Armed): void { + if (current.gatePath !== null) { + if (!existsSync(current.gatePath)) return; + current.gatePath = null; + } + current.hits += 1; + if (current.hits !== current.spec.nth) return; + if (current.mode === "throw") throw new CrashPointReached(current.spec.name, current.hits); + const receipt: CrashPointReceipt = { + point: current.spec.name, hit: current.hits, pid: process.pid, at: new Date().toISOString(), + }; + if (current.receiptPath) { + try { appendFileSync(current.receiptPath, `${JSON.stringify(receipt)}\n`); } catch { /* the log line below still records it */ } + } + try { process.stderr.write(`${CRASH_POINT_LOG_PREFIX} SIGKILL ${JSON.stringify(receipt)}\n`); } catch { /* ignore */ } + process.kill(process.pid, "SIGKILL"); +} + +/** + * Arms one point inside a test process: the nth hit throws `CrashPointReached`. + * Returns a disarm function; call it in `afterEach`. Replaces any armed point. + */ +export function armCrashPointForTest(spec: string | CrashPointSpec): () => void { + // One parser for both forms, so an object with a bad name or count is refused + // instead of arming a point that can never fire. + const parsed = parseCrashPointSpec(typeof spec === "string" ? spec : `${spec.name}:${spec.nth}`); + if (!parsed) throw new Error("armCrashPointForTest needs a crash point name"); + const previous = armed; + const current: Armed = { spec: parsed, hits: 0, mode: "throw", receiptPath: null, gatePath: null }; + armed = current; + return () => { if (armed === current) armed = previous; }; +} + +/** The armed point and its hit count so far, or null. For tests and diagnostics. */ +export function armedCrashPoint(): { name: CrashPointName; nth: number; hits: number } | null { + return armed ? { name: armed.spec.name, nth: armed.spec.nth, hits: armed.hits } : null; +}