diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index caa29c30eb2..1a0c42eba5c 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -24,8 +24,79 @@ node scripts/typecheck-test-projects.mjs tests/restart-chaos # typecheck Options: `--triggers time,comment,retry,queued,tool` (fault timing, below), `--workloads A1,B1,B5,C1`, `--budget-cents` (company hard cap, default 3000), `--poll-ms` (2000–5000), `--keep` (keep the temporary instance). -`--env daytona` is reserved; it fails until the Daytona image and environment -are wired (see the TODO in `seed.ts`). +## Daytona (`--env daytona`) + +The control plane stays local; agent runs execute in Daytona sandboxes from an +immutable runner image. No Docker is needed. + +```sh +set -a; source ~/.secrets; set +a # DAYTONA_API_KEY plus provider keys, process env only +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env daytona \ + --daytona-image ghcr.io/paperclipai/paperclip-daytona-runner@sha256: \ + --adapters codex,claude --tasks 3 --concurrency 2 --faults K1 --seed 11 +``` + +`PAPERCLIP_E2E_DAYTONA_IMAGE` can replace `--daytona-image`. B5 is local-only +(its oracle reads the live workspace, which is inside the sandbox). + +**vCPU budget.** The Daytona org is shared with the live Cloud fleet. Chaos +sandboxes use `--daytona-cpu` vCPU (default 4, with 4 GiB memory and 10 GiB +disk), and `--concurrency × --daytona-cpu` must stay within 40 vCPU +(`RESTART_CHAOS_DAYTONA_VCPU_BUDGET`). On a small org, set the budget below the +org limit with room for verification probes: a 10 vCPU org fails creates with +HTTP 400 "Total CPU limit exceeded", so use 8 there and `--daytona-cpu 2`. + +**Fleet safety.** The harness never stops, archives or deletes a Daytona +sandbox. Its Daytona client is a frozen wrapper that exposes only a +label-filtered `list`. `assertHarnessMayMutate` refuses `paperclip-create-*` +names (every plugin-created sandbox has one) and `fleet-sbx-*` snapshots, so a +future mutating path cannot act on fleet sandboxes either. Cleanup goes only +through Paperclip, by deleting the wave's own environment. + +**Image matching.** The controller needs the image's own Linux amd64 +`/usr/local/bin/paperclip-runnerd` (`PAPERCLIP_RUNNER_REMOTE_BINARY_PATH`) and +`/opt/paperclip-runner/provider-pack/` (`PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH`). +`daytona-image.ts` gets them without a daemon: `skopeo copy --override-os linux +--override-arch amd64 docker:// dir:`, then walks the layers from +top to bottom (honouring `.wh.` and `.wh..wh..opq` whiteouts), extracts only +those two paths, checks the ELF header, the pack manifest and the image labels, +records their digests in `receipt.json`, and deletes the layer blobs. The result +is cached in `results/daytona-images//`, so it runs once per image. +Run it alone with `node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/daytona-image.ts `. + +Before starting, the harness compares the image's `io.paperclip.runner.content-id` +label with the content ID of this checkout (`pnpm test:e2e:runner:image-id`, +which hashes every runner input) and refuses to run on a mismatch. To get an +image for the current master, dispatch a narrow image build and reject its paid +gate: + +```sh +gh workflow run runner-full-stack-e2e.yml --ref master -f cadence=manual -f all=false \ + -f id=core-compatibility.runner-codex.daytona.message-marker +# when the run waits on the paid environment: +gh api -X POST repos/paperclipai/paperclip/actions/runs//pending_deployments \ + -F "environment_ids[]=20786721735" -f state=rejected -f comment="Image build only" +``` + +The digest is in the "Publish verified Daytona image" job log (`pushing manifest +for …@sha256:…`), or check out the image's `org.opencontainers.image.revision`. + +**Setup** reuses runner-e2e: the bundled Daytona plugin (`installFixtureDaytonaProvider`, +prepared outside the root workspace as CI does), the `daytona` environment +fixture from `catalog.ts`, and the native Codex platform-qualification probe. + +**R6 sweep.** After the wave, `daytona-sweep.ts` lists (read-only) the +sandboxes whose `paperclip-company-id` and `paperclip-environment-id` labels +both equal the UUIDs this wave's isolated instance minted, re-checks every row, +and reconciles them with the environment's leases. A sandbox archived after its +lease was released is expected (the plugin archives on release and Daytona +deletes it after 60 minutes). Any other live sandbox without a held lease, or +an `active`/`pending_cleanup` lease, is a leak: it fails R6 for the +task whose run owns it, or the wave when no task does. The harness then deletes +the wave's environment through Paperclip and lists again: a wave sandbox that +is still neither archived nor destroyed after 90 seconds fails the wave and is +reported for a person to act on (the environment's 60-minute ttl also expires +it). The result is in `/sandbox-sweep.json`. ## Fault timing (`--triggers`) @@ -59,6 +130,8 @@ re-arms the task's own triggered faults instead of using their offsets. | `grader.ts` | Pure R1–R6 grading over the samples. | | `workloads.ts` | A1 brief document, B1 python CLI with tests, B5 long bench, C1 comment while running, with their T-oracles. | | `evidence.ts`, `ledger.ts` | Evidence bundle for each failure and the JSONL ledger plus `SUMMARY.md`. | +| `daytona-image.ts` | Docker-free extraction of the image's runnerd and provider pack, cached per digest, and the controller/image content-ID check. | +| `daytona-sweep.ts` | Daytona R6: read-only reconciliation of this wave's labelled sandboxes with leases; cleanup only through Paperclip. | ## Output (gitignored) diff --git a/tests/restart-chaos/api.ts b/tests/restart-chaos/api.ts index b72d5b900b4..249e59cef15 100644 --- a/tests/restart-chaos/api.ts +++ b/tests/restart-chaos/api.ts @@ -33,4 +33,8 @@ export class ChaosApi { post(route: string, body?: unknown, timeoutMs?: number) { return this.request("POST", route, body ?? {}, timeoutMs); } put(route: string, body?: unknown) { return this.request("PUT", route, body ?? {}); } patch(route: string, body?: unknown) { return this.request("PATCH", route, body ?? {}); } + async delete(route: string, options?: { allowNotFound?: boolean; timeout?: number }): Promise { + try { await this.request("DELETE", route, undefined, options?.timeout); } + catch (error) { if (!(options?.allowNotFound && error instanceof ChaosApiError && error.status === 404)) throw error; } + } } diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index a8e675feff8..bf5b45bd43c 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -17,6 +17,17 @@ export const PROFILE_FOR_ADAPTER: Record = { opencode: "runner-opencode", }; +/** + * Chaos sandboxes run in the company Daytona org next to the live Cloud fleet. + * They use the runner-e2e fixture's 4 vCPU by default, and task sandboxes may + * use at most 40 vCPU in total, a small fraction of the fleet's capacity. Set + * RESTART_CHAOS_DAYTONA_VCPU_BUDGET lower for a smaller org (a 10 vCPU org + * fails creates with HTTP 400 "Total CPU limit exceeded"; use 8 there so + * verification probes still fit). + */ +export const DAYTONA_DEFAULT_CPU = 4; +export const DAYTONA_VCPU_BUDGET = 40; + export type ParsedCommand = | { mode: "wave"; options: WaveOptions; keep: boolean } | { mode: "replay"; ref: ReplayRef; keep: boolean; overrides: Partial> }; @@ -70,6 +81,8 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "budget-cents": { type: "string" }, "poll-ms": { type: "string" }, replay: { type: "string" }, + "daytona-image": { type: "string" }, + "daytona-cpu": { type: "string" }, keep: { type: "boolean" }, }, }); @@ -77,7 +90,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -85,10 +98,18 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC } const env = (values.env ?? "local") as EnvironmentKind; if (!ENVIRONMENT_KINDS.includes(env)) throw new Error(`Unknown env "${env}"; allowed: ${ENVIRONMENT_KINDS.join(", ")}`); + const daytonaImage = values["daytona-image"] ?? (env === "daytona" ? process.env.PAPERCLIP_E2E_DAYTONA_IMAGE : undefined); + if (env === "daytona" && !/^.+@sha256:[0-9a-f]{64}$/.test(daytonaImage ?? "")) { + throw new Error("--env daytona needs an immutable runner image: --daytona-image or PAPERCLIP_E2E_DAYTONA_IMAGE"); + } + if (env !== "daytona" && values["daytona-image"] !== undefined) throw new Error("--daytona-image applies only to --env daytona"); + // B5 grades bench.sh progress read from the live workspace while it runs. A + // Daytona run's live workspace is inside the sandbox, so B5 is local-only. + const workloadChoices = env === "daytona" ? WORKLOAD_IDS.filter(id => id !== "B5") : WORKLOAD_IDS; const options: WaveOptions = { env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), - workloads: list(values.workloads, WORKLOAD_IDS, "workloads", WORKLOAD_IDS), + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices), faults: list(values.faults, FAULT_KINDS, "faults", ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), @@ -96,7 +117,16 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC seed: integer(values.seed, "seed", now % 2_147_483_647, 0, 2_147_483_647), budgetCents, pollMs, + ...(env === "daytona" ? { daytonaImage, daytonaCpu: integer(values["daytona-cpu"], "daytona-cpu", DAYTONA_DEFAULT_CPU, 1, 4) } : {}), }; + if (env !== "daytona" && values["daytona-cpu"] !== undefined) throw new Error("--daytona-cpu applies only to --env daytona"); + if (options.daytonaCpu !== undefined) { + const budget = integer(process.env.RESTART_CHAOS_DAYTONA_VCPU_BUDGET, "RESTART_CHAOS_DAYTONA_VCPU_BUDGET", DAYTONA_VCPU_BUDGET, 1, 1_000); + if (options.concurrency * options.daytonaCpu > budget) { + throw new Error(`--concurrency ${options.concurrency} × --daytona-cpu ${options.daytonaCpu} vCPU exceeds the Daytona vCPU budget of ${budget} ` + + "for chaos tasks. Lower one of them (RESTART_CHAOS_DAYTONA_VCPU_BUDGET sets the budget)."); + } + } return { mode: "wave", options, keep }; } diff --git a/tests/restart-chaos/daytona-image.test.ts b/tests/restart-chaos/daytona-image.test.ts new file mode 100644 index 00000000000..3b26cff187c --- /dev/null +++ b/tests/restart-chaos/daytona-image.test.ts @@ -0,0 +1,193 @@ +import { mkdtemp, readFile, readlink, rm, stat } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { assertControllerMatchesImage, extractFromLayers, normalizeLayerPath, walkTar, type LayerSource } from "./daytona-image.js"; + +type Entry = + | { path: string; type: "file"; content: string; mode?: number } + | { path: string; type: "dir" } + | { path: string; type: "symlink" | "hardlink"; target: string }; + +function header(name: string, typeflag: string, size: number, mode: number, link = ""): Buffer { + const block = Buffer.alloc(512); + const field = (value: string, offset: number, length: number) => block.write(value, offset, length, "utf8"); + const octal = (value: number, offset: number, length: number) => field(`${value.toString(8).padStart(length - 1, "0")}\0`, offset, length); + field(name, 0, 100); + octal(mode, 100, 8); octal(0, 108, 8); octal(0, 116, 8); octal(size, 124, 12); octal(0, 136, 12); + block.write(typeflag, 156, 1, "latin1"); + field(link, 157, 100); + block.write("ustar\0", 257, 6, "latin1"); + block.write("00", 263, 2, "latin1"); + block.fill(0x20, 148, 156); + let sum = 0; + for (const byte of block) sum += byte; + field(`${sum.toString(8).padStart(6, "0")}\0 `, 148, 8); + return block; +} + +const pad = (body: Buffer) => Buffer.concat([body, Buffer.alloc((512 - (body.length % 512)) % 512)]); + +/** Minimal ustar writer; names over 100 bytes use a pax `path` record. */ +function tar(entries: Entry[]): Buffer { + const blocks: Buffer[] = []; + for (const entry of entries) { + if (Buffer.byteLength(entry.path) > 100) { + const record = (value: string) => { + let length = value.length + 3; + while (`${length} ${value}\n`.length !== length) length = `${length} ${value}\n`.length; + return `${length} ${value}\n`; + }; + const body = Buffer.from(record(`path=${entry.path}`)); + blocks.push(header("PaxHeader", "x", body.length, 0o644), pad(body)); + } + const name = Buffer.byteLength(entry.path) > 100 ? "long" : entry.path; + if (entry.type === "file") { + const body = Buffer.from(entry.content); + blocks.push(header(name, "0", body.length, entry.mode ?? 0o644), pad(body)); + } else if (entry.type === "dir") blocks.push(header(name, "5", 0, 0o755)); + else blocks.push(header(name, entry.type === "symlink" ? "2" : "1", 0, 0o777, entry.target)); + } + blocks.push(Buffer.alloc(1024)); + return Buffer.concat(blocks); +} + +/** Chunk the archive oddly to exercise header/body boundaries. */ +function layer(digest: string, entries: Entry[]): LayerSource { + const bytes = tar(entries); + return { digest, open: async function* () { for (let offset = 0; offset < bytes.length; offset += 333) yield bytes.subarray(offset, offset + 333); } }; +} + +const RUNNERD = "usr/local/bin/paperclip-runnerd"; +const PACK = "opt/paperclip-runner/provider-pack"; +const directories: string[] = []; +async function outDir() { const directory = await mkdtemp(path.join(os.tmpdir(), "chaos-layers-")); directories.push(directory); return directory; } +afterEach(async () => { await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))); }); +const read = (root: string, relative: string) => readFile(path.join(root, relative), "utf8"); +const exists = (root: string, relative: string) => stat(path.join(root, relative)).then(() => true, () => false); + +describe("walkTar", () => { + it("reads ustar, pax long paths, links and normalizes ./ prefixes", async () => { + const long = `${PACK}/${"x".repeat(120)}/file.txt`; + const seen: Array<[string, string, string]> = []; + await walkTar(layer("l", [ + { path: "./usr/", type: "dir" }, + { path: long, type: "file", content: "deep" }, + { path: "a/link", type: "symlink", target: "../b" }, + ]).open(), async (entry, body) => { + let content = ""; + if (entry.type === "file") await body.forward(chunk => { content += chunk.toString(); }); + seen.push([entry.path, entry.type, content || entry.linkTarget]); + }); + expect(seen).toEqual([["usr", "directory", ""], [long, "file", "deep"], ["a/link", "symlink", "../b"]]); + }); + + it("rejects parent-directory traversal", () => { + expect(() => normalizeLayerPath("opt/../../etc/passwd")).toThrow(/\.\./); + }); +}); + +describe("extractFromLayers", () => { + it("takes the topmost version of each file and merges lower-only files", async () => { + const out = await outDir(); + const report = await extractFromLayers([ + layer("base", [{ path: "usr/local/bin/other", type: "file", content: "x" }, + { path: RUNNERD, type: "file", content: "old", mode: 0o755 }, + { path: `${PACK}/provider-pack.json`, type: "file", content: "{\"v\":1}" }, + { path: `${PACK}/lib/keep.js`, type: "file", content: "keep" }]), + layer("top", [{ path: RUNNERD, type: "file", content: "new", mode: 0o755 }, + { path: `${PACK}/provider-pack.json`, type: "file", content: "{\"v\":2}" }]), + ], [RUNNERD, PACK], out); + expect(await read(out, RUNNERD)).toBe("new"); + expect((await stat(path.join(out, RUNNERD))).mode & 0o111).not.toBe(0); + expect(await read(out, `${PACK}/provider-pack.json`)).toBe("{\"v\":2}"); + expect(await read(out, `${PACK}/lib/keep.js`)).toBe("keep"); + expect(await exists(out, "usr/local/bin/other")).toBe(false); + expect(report.layersWithMatches).toEqual(["top", "base"]); + }); + + it("honours .wh. whiteouts for files and whole subtrees", async () => { + const out = await outDir(); + await extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "r" }, + { path: `${PACK}/provider-pack.json`, type: "file", content: "{}" }, + { path: `${PACK}/stale.js`, type: "file", content: "stale" }, + { path: `${PACK}/old-dir/a.js`, type: "file", content: "a" }]), + layer("top", [{ path: `${PACK}/.wh.stale.js`, type: "file", content: "" }, + { path: `${PACK}/.wh.old-dir`, type: "file", content: "" }]), + ], [RUNNERD, PACK], out); + expect(await exists(out, `${PACK}/stale.js`)).toBe(false); + expect(await exists(out, `${PACK}/old-dir`)).toBe(false); + expect(await exists(out, `${PACK}/provider-pack.json`)).toBe(true); + }); + + it("honours opaque directories: only the opaque layer and above contribute children", async () => { + const out = await outDir(); + await extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "r" }, { path: `${PACK}/old.js`, type: "file", content: "old" }]), + layer("middle", [{ path: PACK, type: "dir" }, { path: `${PACK}/.wh..wh..opq`, type: "file", content: "" }, + { path: `${PACK}/provider-pack.json`, type: "file", content: "{}" }]), + layer("top", [{ path: `${PACK}/added.js`, type: "file", content: "added" }]), + ], [RUNNERD, PACK], out); + expect(await exists(out, `${PACK}/old.js`)).toBe(false); + expect(await read(out, `${PACK}/provider-pack.json`)).toBe("{}"); + expect(await read(out, `${PACK}/added.js`)).toBe("added"); + }); + + it("deleting a target in a higher layer means it is missing", async () => { + const out = await outDir(); + await expect(extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "r" }, { path: `${PACK}/provider-pack.json`, type: "file", content: "{}" }]), + layer("top", [{ path: "usr/local/bin/.wh.paperclip-runnerd", type: "file", content: "" }]), + ], [RUNNERD, PACK], out)).rejects.toThrow(/paperclip-runnerd was not found/); + }); + + it("a whiteout does not hide entries of its own layer", async () => { + const out = await outDir(); + await extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "old" }, { path: `${PACK}/provider-pack.json`, type: "file", content: "{}" }]), + layer("top", [{ path: `${PACK}/.wh..wh..opq`, type: "file", content: "" }, { path: `${PACK}/provider-pack.json`, type: "file", content: "{\"top\":1}" }]), + ], [RUNNERD, PACK], out); + expect(await read(out, `${PACK}/provider-pack.json`)).toBe("{\"top\":1}"); + }); + + it("a higher-layer file replacing a directory hides the lower tree", async () => { + const out = await outDir(); + await extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "r" }, { path: `${PACK}/sub/a.js`, type: "file", content: "a" }]), + layer("top", [{ path: `${PACK}/sub`, type: "file", content: "now a file" }]), + ], [RUNNERD, PACK], out); + expect(await read(out, `${PACK}/sub`)).toBe("now a file"); + }); + + it("resolves hard links and keeps symlinks inside the pack", async () => { + const out = await outDir(); + await extractFromLayers([layer("only", [ + { path: RUNNERD, type: "file", content: "r", mode: 0o755 }, + { path: `${PACK}/node_modules/tool/cli.js`, type: "file", content: "cli", mode: 0o755 }, + { path: `${PACK}/node_modules/tool/cli-copy.js`, type: "hardlink", target: `${PACK}/node_modules/tool/cli.js` }, + { path: `${PACK}/node_modules/.bin/tool`, type: "symlink", target: "../tool/cli.js" }, + ])], [RUNNERD, PACK], out); + expect(await read(out, `${PACK}/node_modules/tool/cli-copy.js`)).toBe("cli"); + expect(await readlink(path.join(out, PACK, "node_modules/.bin/tool"))).toBe("../tool/cli.js"); + expect(await read(out, `${PACK}/node_modules/.bin/tool`)).toBe("cli"); + }); + + it("refuses a parent directory replaced by a symlink", async () => { + const out = await outDir(); + await expect(extractFromLayers([ + layer("base", [{ path: RUNNERD, type: "file", content: "r" }, { path: `${PACK}/provider-pack.json`, type: "file", content: "{}" }]), + layer("top", [{ path: "opt/paperclip-runner", type: "symlink", target: "/elsewhere" }]), + ], [RUNNERD, PACK], out)).rejects.toThrow(/replaces parent directory \/opt\/paperclip-runner/); + }); +}); + +describe("assertControllerMatchesImage", () => { + const image = { image: "ghcr.io/x/y@sha256:" + "a".repeat(64), contentId: "c".repeat(64), sourceRevision: "d".repeat(40) }; + it("passes when content IDs match", async () => { + await expect(assertControllerMatchesImage(image, "/unused", "c".repeat(64))).resolves.toEqual({ localContentId: "c".repeat(64) }); + }); + it("names both content IDs and the image revision when they differ", async () => { + await expect(assertControllerMatchesImage(image, "/unused", "e".repeat(64))).rejects.toThrow(/does not match the Daytona image[\s\S]*d{40}[\s\S]*e{64}/); + }); +}); diff --git a/tests/restart-chaos/daytona-image.ts b/tests/restart-chaos/daytona-image.ts new file mode 100644 index 00000000000..c7e1a33c20a --- /dev/null +++ b/tests/restart-chaos/daytona-image.ts @@ -0,0 +1,441 @@ +/** + * Extracts the two controller inputs that must match a Daytona runner image + * exactly, without Docker: + * + * - `/usr/local/bin/paperclip-runnerd` → PAPERCLIP_RUNNER_REMOTE_BINARY_PATH + * - `/opt/paperclip-runner/provider-pack/` → PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH + * + * `skopeo copy` writes the linux/amd64 manifest, config and layer blobs to a + * directory (no daemon). The layers are then walked from top to bottom, + * honouring OCI whiteouts, and only those two paths are written out. The + * result is cached by image digest and the layer blobs are deleted. + * + * node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/daytona-image.ts + */ +import { execFile } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { chmod, copyFile, mkdir, open, readFile, rename, rm, stat, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { createGunzip, createZstdDecompress } from "node:zlib"; +import type { Readable } from "node:stream"; + +const execFileAsync = promisify(execFile); + +export const RUNNERD_PATH = "usr/local/bin/paperclip-runnerd"; +export const PROVIDER_PACK_PATH = "opt/paperclip-runner/provider-pack"; +export const CONTENT_ID_LABEL = "io.paperclip.runner.content-id"; +export const REVISION_LABEL = "org.opencontainers.image.revision"; +const RECEIPT_SCHEMA = "paperclip-restart-chaos/daytona-image-artifacts/v1"; +const IMMUTABLE_IMAGE = /^[a-z0-9.-]+(?::[0-9]+)?\/[a-z0-9._/-]+@sha256:([0-9a-f]{64})$/; + +// ---------------------------------------------------------------- tar reader + +export type TarEntryType = "file" | "directory" | "symlink" | "hardlink" | "other"; +export interface TarEntry { path: string; type: TarEntryType; mode: number; size: number; linkTarget: string } + +/** Buffered reader over an async byte stream. */ +class ByteReader { + private chunks: Buffer[] = []; + private length = 0; + private ended = false; + constructor(private readonly source: AsyncIterator) {} + + private async fill(bytes: number) { + while (this.length < bytes && !this.ended) { + const next = await this.source.next(); + if (next.done) this.ended = true; + else if (next.value.length) { this.chunks.push(next.value); this.length += next.value.length; } + } + } + + /** Exactly `bytes` bytes, or null at a clean end of stream. */ + async read(bytes: number): Promise { + await this.fill(bytes); + if (this.length === 0) return null; + if (this.length < bytes) throw new Error("Truncated tar stream"); + const parts: Buffer[] = []; + let needed = bytes; + while (needed > 0) { + const head = this.chunks[0]!; + if (head.length <= needed) { parts.push(head); this.chunks.shift(); needed -= head.length; } + else { parts.push(head.subarray(0, needed)); this.chunks[0] = head.subarray(needed); needed = 0; } + } + this.length -= bytes; + return parts.length === 1 ? parts[0]! : Buffer.concat(parts); + } + + /** Passes the next `bytes` bytes to `sink` without holding them all in memory. */ + async forward(bytes: number, sink: (chunk: Buffer) => Promise | void): Promise { + let remaining = bytes; + while (remaining > 0) { + if (this.length === 0) await this.fill(1); + if (this.length === 0) throw new Error("Truncated tar entry"); + const head = this.chunks[0]!; + const take = Math.min(head.length, remaining); + const part = take === head.length ? head : head.subarray(0, take); + if (take === head.length) this.chunks.shift(); else this.chunks[0] = head.subarray(take); + this.length -= take; + remaining -= take; + await sink(part); + } + } +} + +const cString = (block: Buffer, start: number, end: number) => { + const field = block.subarray(start, end); + const nul = field.indexOf(0); + return (nul === -1 ? field : field.subarray(0, nul)).toString("utf8"); +}; + +function numeric(block: Buffer, start: number, end: number): number { + const field = block.subarray(start, end); + if (field[0]! & 0x80) { + // GNU base-256: big-endian, high bit set on the first byte. + let value = field[0]! & 0x7f; + for (let index = 1; index < field.length; index += 1) value = value * 256 + field[index]!; + return value; + } + const text = cString(block, start, end).trim(); + return text ? Number.parseInt(text, 8) : 0; +} + +function parsePax(body: Buffer): Record { + const out: Record = {}; + let offset = 0; + while (offset < body.length) { + const space = body.indexOf(0x20, offset); + if (space === -1) break; + const length = Number.parseInt(body.subarray(offset, space).toString("utf8"), 10); + if (!Number.isSafeInteger(length) || length <= 0) throw new Error("Malformed pax header"); + const record = body.subarray(space + 1, offset + length - 1).toString("utf8"); + const equals = record.indexOf("="); + if (equals > 0) out[record.slice(0, equals)] = record.slice(equals + 1); + offset += length; + } + return out; +} + +/** Normalized layer path: no leading `./` or `/`, no trailing `/`, no `..`. */ +export function normalizeLayerPath(raw: string): string { + const parts = raw.split("/").filter(part => part !== "" && part !== "."); + if (parts.some(part => part === "..")) throw new Error(`Refusing layer path with "..": ${raw}`); + return parts.join("/"); +} + +/** + * Streams a tar archive. `onEntry` must either call `body.forward` once or + * leave the body unread (it is skipped). + */ +export async function walkTar(source: AsyncIterable, + onEntry: (entry: TarEntry, body: { forward(sink: (chunk: Buffer) => Promise | void): Promise }) => Promise): Promise { + const reader = new ByteReader(source[Symbol.asyncIterator]()); + let pax: Record = {}; + let longName: string | null = null; + let longLink: string | null = null; + for (;;) { + const header = await reader.read(512); + if (!header || header.every(byte => byte === 0)) return; + const stored = numeric(header, 148, 156); + let sum = 0; + for (let index = 0; index < 512; index += 1) sum += index >= 148 && index < 156 ? 0x20 : header[index]!; + if (sum !== stored) throw new Error("Tar header checksum mismatch"); + const typeflag = String.fromCharCode(header[156]!); + const size = pax.size !== undefined ? Number(pax.size) : numeric(header, 124, 136); + const padded = Math.ceil(size / 512) * 512; + if (typeflag === "x" || typeflag === "g" || typeflag === "L" || typeflag === "K") { + const body = (await reader.read(padded)) ?? Buffer.alloc(0); + const content = body.subarray(0, size); + if (typeflag === "x") pax = { ...pax, ...parsePax(content) }; + else if (typeflag === "L") longName = cString(content, 0, content.length); + else if (typeflag === "K") longLink = cString(content, 0, content.length); + continue; + } + const magic = header.subarray(257, 262).toString("latin1"); + const prefix = magic === "ustar" ? cString(header, 345, 500) : ""; + const name = cString(header, 0, 100); + const rawPath = pax.path ?? longName ?? (prefix ? `${prefix}/${name}` : name); + const linkTarget = pax.linkpath ?? longLink ?? cString(header, 157, 257); + pax = {}; longName = null; longLink = null; + const type: TarEntryType = typeflag === "0" || typeflag === "\0" || typeflag === "7" ? "file" + : typeflag === "5" ? "directory" : typeflag === "2" ? "symlink" : typeflag === "1" ? "hardlink" : "other"; + const entry: TarEntry = { path: normalizeLayerPath(rawPath), type, mode: numeric(header, 100, 108) & 0o7777, size: type === "file" ? size : 0, linkTarget }; + let consumed = false; + await onEntry(entry, { async forward(sink) { + if (consumed) throw new Error("Tar entry body already read"); + consumed = true; + await reader.forward(size, sink); + } }); + if (!consumed) await reader.forward(size, () => undefined); + if (padded > size) await reader.forward(padded - size, () => undefined); + } +} + +// ---------------------------------------------------------------- layer walk + +export interface LayerSource { digest: string; open(): AsyncIterable } +export interface ExtractionReport { extracted: Record; layersWithMatches: string[] } + +const within = (candidate: string, root: string) => candidate === root || candidate.startsWith(`${root}/`); +const ancestors = (value: string) => value.split("/").slice(0, -1).map((_, index, parts) => parts.slice(0, index + 1).join("/")); + +/** + * Resolves `roots` (files or directory trees) through an ordered layer stack, + * walking from the top layer down, and writes the effective result to `outDir`. + * + * Whiteouts follow the OCI image spec: `.wh.` deletes `` (and its + * subtree) in lower layers, `.wh..wh..opq` hides every lower-layer child of its + * directory. A higher-layer non-directory at a path also hides lower content + * under that path. Whiteouts affect only lower layers, never their own layer. + */ +export async function extractFromLayers(layersBottomToTop: readonly LayerSource[], roots: readonly string[], outDir: string): Promise { + const normalizedRoots = roots.map(normalizeLayerPath); + const rootAncestors = new Set(normalizedRoots.flatMap(ancestors)); + const deleted = new Set(); + const opaque = new Set(); + const resolved = new Map(); + const symlinks: Array<{ path: string; target: string }> = []; + const directoryModes = new Map(); + const layersWithMatches: string[] = []; + + const hidden = (candidate: string) => { + if (deleted.has(candidate)) return true; + for (const ancestor of ancestors(candidate)) if (deleted.has(ancestor) || opaque.has(ancestor)) return true; + return false; + }; + + await mkdir(outDir, { recursive: true }); + for (const layer of [...layersBottomToTop].reverse()) { + const pendingDeleted = new Set(); + const pendingOpaque = new Set(); + const stagedFiles = new Map(); + let matched = false; + await walkTar(layer.open(), async (entry, body) => { + const base = path.posix.basename(entry.path); + const directory = path.posix.dirname(entry.path); + const parent = directory === "." ? "" : directory; + if (base === ".wh..wh..opq") { pendingOpaque.add(parent); return; } + if (base.startsWith(".wh.")) { pendingDeleted.add(parent ? `${parent}/${base.slice(4)}` : base.slice(4)); return; } + if (!entry.path || hidden(entry.path) || resolved.has(entry.path)) return; + const relevant = normalizedRoots.some(root => within(entry.path, root)); + if (!relevant) { + // A file or symlink replacing a parent of a root would redirect the root. + if (rootAncestors.has(entry.path) && entry.type !== "directory") { + throw new Error(`Layer ${layer.digest} replaces parent directory /${entry.path} with a ${entry.type}; extraction is unsupported`); + } + if (rootAncestors.has(entry.path)) resolved.set(entry.path, entry.type); + return; + } + matched = true; + resolved.set(entry.path, entry.type); + if (entry.type !== "directory") pendingDeleted.add(entry.path); + const target = path.join(outDir, ...entry.path.split("/")); + if (entry.type === "directory") { + await mkdir(target, { recursive: true }); + directoryModes.set(target, entry.mode); + } else if (entry.type === "file") { + await mkdir(path.dirname(target), { recursive: true }); + const handle = await open(target, "wx", 0o600); + try { await body.forward(async chunk => { await handle.write(chunk); }); } finally { await handle.close(); } + await chmod(target, entry.mode & 0o755); + stagedFiles.set(entry.path, target); + } else if (entry.type === "hardlink") { + const source = stagedFiles.get(normalizeLayerPath(entry.linkTarget)); + if (!source) throw new Error(`Hard link /${entry.path} → /${entry.linkTarget} points outside the extracted paths of layer ${layer.digest}`); + await mkdir(path.dirname(target), { recursive: true }); + await copyFile(source, target); + await chmod(target, (await stat(source)).mode & 0o755); + resolved.set(entry.path, "file"); + stagedFiles.set(entry.path, target); + } else if (entry.type === "symlink") { + symlinks.push({ path: target, target: entry.linkTarget }); + } else { + throw new Error(`Unsupported tar entry type at /${entry.path} in layer ${layer.digest}`); + } + }); + for (const value of pendingDeleted) deleted.add(value); + for (const value of pendingOpaque) opaque.add(value); + if (matched) layersWithMatches.push(layer.digest); + } + // Symlinks last, so no file is ever written through one. + for (const link of symlinks) { + await mkdir(path.dirname(link.path), { recursive: true }); + await symlink(link.target, link.path); + } + for (const [directory, mode] of directoryModes) await chmod(directory, (mode & 0o755) | 0o700); + for (const root of normalizedRoots) if (![...resolved.keys()].some(key => within(key, root))) throw new Error(`/${root} was not found in the image layers`); + const extracted = Object.fromEntries([...resolved].filter(([key]) => normalizedRoots.some(root => within(key, root)))); + return { extracted, layersWithMatches }; +} + +// ---------------------------------------------------------------- image + +export interface DaytonaImageArtifacts { + schema: typeof RECEIPT_SCHEMA; + image: string; + platformManifestDigest: string; + contentId: string; + sourceRevision: string; + runnerdPath: string; + runnerdSha256: string; + providerPackPath: string; + providerPackManifestSha256: string; + providerPackDigest: string; + providerPackRunnerSourceRevision: string; + extractedAt: string; +} + +const sha256 = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); + +async function fileSha256(file: string): Promise { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(file)) hash.update(chunk as Buffer); + return hash.digest("hex"); +} + +/** Decompresses a layer blob. skopeo verified the blob digests when it copied them. */ +function layerStream(file: string, mediaType: string): AsyncIterable { + if (!/tar/.test(mediaType)) throw new Error(`Unsupported layer media type ${mediaType}`); + const raw = createReadStream(file); + const decoder: Readable = /zstd/.test(mediaType) ? raw.pipe(createZstdDecompress()) + : /gzip/.test(mediaType) ? raw.pipe(createGunzip()) : raw; + return decoder as AsyncIterable; +} + +/** ELF64, little-endian, x86-64 executable or shared object. */ +export async function assertLinuxAmd64Executable(file: string): Promise { + const handle = await open(file, "r"); + try { + const header = Buffer.alloc(20); + await handle.read(header, 0, 20, 0); + if (header.readUInt32BE(0) !== 0x7f454c46) throw new Error(`${file} is not an ELF binary`); + if (header[4] !== 2 || header[5] !== 1) throw new Error(`${file} is not a 64-bit little-endian ELF binary`); + if (header.readUInt16LE(18) !== 62) throw new Error(`${file} is not an x86-64 binary`); + } finally { await handle.close(); } + if (((await stat(file)).mode & 0o111) === 0) throw new Error(`${file} is not executable`); +} + +async function readReceipt(directory: string): Promise { + const receipt = await readFile(path.join(directory, "receipt.json"), "utf8").then(text => JSON.parse(text) as DaytonaImageArtifacts).catch(() => null); + if (!receipt || receipt.schema !== RECEIPT_SCHEMA) return null; + // A cache hit must still be the exact bytes that were verified. + if (await fileSha256(receipt.runnerdPath).catch(() => null) !== receipt.runnerdSha256) return null; + const pack = await readFile(path.join(receipt.providerPackPath, "provider-pack.json")).catch(() => null); + if (!pack || sha256(pack) !== receipt.providerPackManifestSha256) return null; + return receipt; +} + +/** + * Returns the verified runnerd and provider pack of `image`, extracting them + * once per image digest into `cacheRoot//`. + */ +export async function prepareDaytonaImageArtifacts(input: { image: string; cacheRoot: string; skopeo?: string; log?: (line: string) => void }): Promise { + const match = IMMUTABLE_IMAGE.exec(input.image); + if (!match) throw new Error(`Daytona image must be an immutable registry/repository@sha256: reference, got "${input.image}"`); + const log = input.log ?? (() => undefined); + const finalDir = path.join(path.resolve(input.cacheRoot), match[1]!); + const cached = await readReceipt(finalDir); + if (cached) { log(`Daytona image artifacts cached at ${finalDir}`); return cached; } + await rm(finalDir, { recursive: true, force: true }); + + const staging = path.join(path.resolve(input.cacheRoot), `.staging-${match[1]!.slice(0, 12)}-${randomBytes(4).toString("hex")}`); + const blobs = path.join(staging, "blobs"); + const rootfs = path.join(staging, "rootfs"); + try { + await mkdir(blobs, { recursive: true, mode: 0o700 }); + log(`skopeo copy ${input.image} (linux/amd64) → ${blobs}`); + await execFileAsync(input.skopeo ?? process.env.SKOPEO ?? "skopeo", ["copy", "--retry-times", "3", + "--override-os", "linux", "--override-arch", "amd64", `docker://${input.image}`, `dir:${blobs}`], + { maxBuffer: 16 * 1024 * 1024 }); + const manifestBytes = await readFile(path.join(blobs, "manifest.json")); + const manifest = JSON.parse(manifestBytes.toString("utf8")) as { config: { digest: string }; layers: Array<{ digest: string; mediaType: string }> }; + if (!manifest.config?.digest || !Array.isArray(manifest.layers)) throw new Error("skopeo did not produce a single-platform image manifest"); + const blobPath = (digest: string) => { + if (!/^sha256:[0-9a-f]{64}$/.test(digest)) throw new Error(`Unexpected blob digest ${digest}`); + return path.join(blobs, digest.slice("sha256:".length)); + }; + for (const layer of manifest.layers) { + if (`sha256:${await fileSha256(blobPath(layer.digest))}` !== layer.digest) throw new Error(`Layer blob digest mismatch for ${layer.digest}`); + } + const configBytes = await readFile(blobPath(manifest.config.digest)); + if (`sha256:${sha256(configBytes)}` !== manifest.config.digest) throw new Error("Image config digest mismatch"); + const config = JSON.parse(configBytes.toString("utf8")) as { os?: string; architecture?: string; config?: { Labels?: Record } }; + if (config.os !== "linux" || config.architecture !== "amd64") throw new Error(`Image platform is ${config.os}/${config.architecture}, expected linux/amd64`); + const contentId = config.config?.Labels?.[CONTENT_ID_LABEL] ?? ""; + const sourceRevision = config.config?.Labels?.[REVISION_LABEL] ?? ""; + if (!/^[0-9a-f]{64}$/.test(contentId)) throw new Error(`Image has no ${CONTENT_ID_LABEL} label`); + if (!/^[0-9a-f]{40}$/.test(sourceRevision)) throw new Error(`Image has no ${REVISION_LABEL} label`); + + log(`Walking ${manifest.layers.length} layers for /${RUNNERD_PATH} and /${PROVIDER_PACK_PATH}`); + await extractFromLayers(manifest.layers.map(layer => ({ digest: layer.digest, open: () => layerStream(blobPath(layer.digest), layer.mediaType) })), + [RUNNERD_PATH, PROVIDER_PACK_PATH], rootfs); + await rm(blobs, { recursive: true, force: true }); + + const runnerd = path.join(rootfs, ...RUNNERD_PATH.split("/")); + await assertLinuxAmd64Executable(runnerd); + const packDir = path.join(rootfs, ...PROVIDER_PACK_PATH.split("/")); + const packBytes = await readFile(path.join(packDir, "provider-pack.json")); + const pack = JSON.parse(packBytes.toString("utf8")) as { schema?: string; digest?: string; payload?: { runnerSourceRevision?: string } }; + if (pack.schema !== "paperclip-runner/remote-provider-pack/v1" || !/^sha256:[0-9a-f]{64}$/.test(pack.digest ?? "")) { + throw new Error("Image provider-pack.json is not a paperclip-runner/remote-provider-pack/v1 manifest"); + } + if (pack.payload?.runnerSourceRevision !== sourceRevision) { + throw new Error(`Provider pack was built from ${pack.payload?.runnerSourceRevision ?? "an unknown revision"}, but the image revision label is ${sourceRevision}`); + } + const receiptFor = (root: string): DaytonaImageArtifacts => ({ + schema: RECEIPT_SCHEMA, image: input.image, platformManifestDigest: `sha256:${sha256(manifestBytes)}`, contentId, sourceRevision, + runnerdPath: path.join(root, "rootfs", ...RUNNERD_PATH.split("/")), runnerdSha256: "", + providerPackPath: path.join(root, "rootfs", ...PROVIDER_PACK_PATH.split("/")), providerPackManifestSha256: sha256(packBytes), + providerPackDigest: pack.digest!, providerPackRunnerSourceRevision: sourceRevision, extractedAt: new Date().toISOString(), + }); + const receipt = { ...receiptFor(finalDir), runnerdSha256: await fileSha256(runnerd) }; + await writeFile(path.join(staging, "receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + await rename(staging, finalDir); + log(`Daytona image artifacts ready at ${finalDir}`); + return receipt; + } catch (error) { + await rm(staging, { recursive: true, force: true }).catch(() => undefined); + throw error; + } +} + +/** + * The controller and the image must come from the same runner source: the + * image's content-ID label must equal the content ID of this checkout, which + * hashes every runner input (Rust crates, runner src, provider-pack scripts, + * lockfile, Dockerfile). + */ +export async function assertControllerMatchesImage(artifacts: Pick, + repositoryRoot: string, localContentId?: string): Promise<{ localContentId: string }> { + const computed = localContentId ?? await (await import("../runner-e2e/daytona-image-content.js")).computeDaytonaImageContentId({ repositoryRoot }); + if (computed !== artifacts.contentId) { + throw new Error([ + "Controller runner source does not match the Daytona image.", + ` image ${artifacts.image}`, + ` image content ID ${artifacts.contentId} (source revision ${artifacts.sourceRevision})`, + ` this checkout's content ID ${computed}`, + "Check out the image's source revision, or build a new image for this checkout", + "(gh workflow run runner-full-stack-e2e.yml, see tests/restart-chaos/README.md).", + ].join("\n")); + } + return { localContentId: computed }; +} + +const invoked = process.argv[1] ? path.resolve(process.argv[1]) : null; +if (invoked && import.meta.url === pathToFileURL(invoked).href) { + const image = process.argv[2]; + if (!image) { console.error("Usage: daytona-image.ts "); process.exit(2); } + const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + const cacheRoot = process.env.RESTART_CHAOS_IMAGE_CACHE ?? path.join(import.meta.dirname, "results", "daytona-images"); + try { + const artifacts = await prepareDaytonaImageArtifacts({ image, cacheRoot, log: line => console.error(line) }); + const check = await assertControllerMatchesImage(artifacts, repositoryRoot).then(() => "match", (error: Error) => error.message); + console.info(JSON.stringify({ ...artifacts, controllerMatch: check }, null, 2)); + process.exit(check === "match" ? 0 : 1); + } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exit(2); + } +} diff --git a/tests/restart-chaos/daytona-sweep.test.ts b/tests/restart-chaos/daytona-sweep.test.ts new file mode 100644 index 00000000000..5edcac7c29a --- /dev/null +++ b/tests/restart-chaos/daytona-sweep.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { + assertHarnessMayMutate, DaytonaWaveSweeper, isProtectedSandbox, leaksForTask, readOnlyDaytona, reconcileSandboxes, selectWaveSandboxes, + sweepHasLeaks, unattributedLeaks, type DaytonaSandboxRow, type DaytonaSweepResult, +} from "./daytona-sweep.js"; + +const COMPANY = "11111111-1111-4111-8111-111111111111"; +const ENVIRONMENT = "22222222-2222-4222-8222-222222222222"; +const OTHER = "33333333-3333-4333-8333-333333333333"; +const wave = { companyId: COMPANY, environmentId: ENVIRONMENT }; +const labelled = (id: string, labels: Record = {}, extra: Partial = {}): DaytonaSandboxRow => + ({ id, state: "started", labels, ...extra }); + +describe("wave sandbox selection is read-only and exact", () => { + const rows = [ + labelled("mine", { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT }), + labelled("unlabelled"), + labelled("other-environment", { "paperclip-company-id": COMPANY, "paperclip-environment-id": OTHER }), + labelled("other-company", { "paperclip-company-id": OTHER, "paperclip-environment-id": ENVIRONMENT }), + labelled("company-only", { "paperclip-company-id": COMPANY }), + labelled("fleet", { "paperclip-company-id": OTHER }, { name: "paperclip-create-abc", snapshot: "fleet-sbx-1" }), + ]; + + it("selects only sandboxes carrying both of the wave's UUID labels", () => { + expect(selectWaveSandboxes(rows, wave).map(row => row.id)).toEqual(["mine"]); + expect(() => selectWaveSandboxes(rows, { companyId: "", environmentId: "" })).toThrow(/UUIDs/); + }); + + it("re-checks rows even when the provider ignores the label filter", async () => { + const queries: unknown[] = []; + const sweeper = new DaytonaWaveSweeper({ apiKey: "unused", ...wave, api: {} as never }); + (sweeper as unknown as { client: unknown }).client = readOnlyDaytona({ list: query => { + queries.push(query); + return (async function* () { yield* rows; })(); + } }); + expect((await sweeper.listWaveSandboxes()).map(row => row.id)).toEqual(["mine"]); + expect(queries).toEqual([{ labels: { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT } }]); + }); + + it("exposes no mutating Daytona operation", () => { + const client = readOnlyDaytona({ list: () => (async function* () {})(), delete: () => undefined, stop: () => undefined } as never); + expect(Object.keys(client)).toEqual(["list"]); + expect(Object.isFrozen(client)).toBe(true); + expect(() => readOnlyDaytona({} as never)).toThrow(/no list\(\)/); + }); + + it("hard-refuses fleet sandboxes and anything not labelled for the wave, even with matching labels", () => { + const waveLabels = { "paperclip-company-id": COMPANY, "paperclip-environment-id": ENVIRONMENT }; + expect(isProtectedSandbox({ name: "paperclip-create-1" })).toBe(true); + expect(isProtectedSandbox({ name: "x", snapshot: "fleet-sbx-2" })).toBe(true); + expect(() => assertHarnessMayMutate(labelled("a", waveLabels, { name: "paperclip-create-1" }), wave)).toThrow(/protected/); + expect(() => assertHarnessMayMutate(labelled("b", waveLabels, { name: "probe", snapshot: "fleet-sbx-2" }), wave)).toThrow(/protected/); + expect(() => assertHarnessMayMutate(labelled("c", {}, { name: "probe" }), wave)).toThrow(/not labelled for this wave/); + expect(() => assertHarnessMayMutate(labelled("d", waveLabels, { name: "probe" }), wave)).not.toThrow(); + }); +}); + +describe("Daytona R6 reconciliation", () => { + const base = { companyId: "c", environmentId: "e", checkedAt: "now", error: null }; + + it("accepts live sandboxes backed by held leases and ignores destroyed ones", () => { + const result = reconcileSandboxes( + [{ id: "s1", state: "started" }, { id: "s2", state: "destroyed" }, { id: "s3", state: "stopped" }], + [{ id: "l1", status: "retained", providerLeaseId: "s1", heartbeatRunId: "r1" }, + { id: "l3", status: "released", providerLeaseId: "s3", heartbeatRunId: "r3" }], + ); + expect(result.liveSandboxes).toEqual(["s1", "s3"]); + expect(result.orphanSandboxes).toEqual([expect.objectContaining({ sandboxId: "s3", leaseStatus: "released" })]); + expect(result.heldLeases).toEqual([]); + expect(result.retainedLeases.map(lease => lease.leaseId)).toEqual(["l1"]); + }); + + it("treats an archive behind a released lease as expected, but not a running one or one without a lease", () => { + const result = reconcileSandboxes( + [{ id: "a1", state: "archived" }, { id: "a2", state: "started" }, { id: "a3", state: "archived" }], + [{ id: "l1", status: "released", providerLeaseId: "a1" }, { id: "l2", status: "released", providerLeaseId: "a2" }], + ); + expect(result.archivedSandboxes).toEqual(["a1"]); + expect(result.orphanSandboxes.map(leak => leak.sandboxId)).toEqual(["a2", "a3"]); + }); + + + it("reports sandboxes with no lease and leases still active at rest", () => { + const result = reconcileSandboxes( + [{ id: "s9", state: "started", labels: { "paperclip-run-id": "r9" } }, { id: "s1", state: "started" }], + [{ id: "l1", status: "active", providerLeaseId: "s1", heartbeatRunId: "r1", issueId: "i1" }], + ); + expect(result.orphanSandboxes).toEqual([expect.objectContaining({ sandboxId: "s9", runId: "r9", reason: "no lease references this sandbox" })]); + expect(result.heldLeases).toEqual([expect.objectContaining({ leaseId: "l1", status: "active" })]); + }); + + it("attributes leaks to the task that owns the run or issue; the rest stay wave-level", () => { + const sweep: DaytonaSweepResult = { ...base, liveSandboxes: ["s9", "s1", "s5"], retainedLeases: [], archivedSandboxes: [], + orphanSandboxes: [{ sandboxId: "s9", state: "started", runId: "r9", leaseId: null, leaseStatus: null, reason: "x" }, + { sandboxId: "s5", state: "started", runId: null, leaseId: null, leaseStatus: null, reason: "x" }], + heldLeases: [{ leaseId: "l1", status: "active", providerLeaseId: "s1", runId: null, issueId: "i1" }] }; + const task = leaksForTask(sweep, new Set(["r9"]), "i1"); + expect(task).toEqual(["orphan_sandbox=s9", "active_lease=l1"]); + expect(unattributedLeaks(sweep, new Set(task))).toEqual(["orphan_sandbox=s5"]); + expect(sweepHasLeaks(sweep)).toBe(true); + expect(sweepHasLeaks({ ...sweep, orphanSandboxes: [], heldLeases: [] })).toBe(false); + }); +}); diff --git a/tests/restart-chaos/daytona-sweep.ts b/tests/restart-chaos/daytona-sweep.ts new file mode 100644 index 00000000000..208526064fd --- /dev/null +++ b/tests/restart-chaos/daytona-sweep.ts @@ -0,0 +1,205 @@ +/** + * R6 for `--env daytona`: reconcile the Daytona sandboxes this wave created + * against the instance's environment leases and report leaks. + * + * The Daytona org is shared with the live Cloud fleet, so this module NEVER + * changes a sandbox. Its client exposes only `list`; stop, archive and delete + * do not exist here. Cleanup goes only through Paperclip: deleting the wave's + * own environment, which acts on the leases in the wave's isolated database. + * Leaks are reported for a person, and expire through the environment's ttl. + * + * Wave ownership comes from the labels the Daytona plugin puts on every + * sandbox it creates. Each wave's isolated instance mints a fresh company and + * environment UUID, and a row is selected only when both labels equal them. + */ +import { setTimeout as delay } from "node:timers/promises"; +import { createRemoteFixtureClient } from "../runner-e2e/remote-native-bootstrap.js"; +import { deleteDaytonaEnvironment } from "../runner-e2e/live-fixtures.js"; +import type { ChaosApi } from "./api.js"; + +export interface DaytonaSandboxRow { id: string; name?: string; state?: string; snapshot?: string; labels?: Record } +export interface LeaseRow { id: string; status: string; providerLeaseId?: string | null; heartbeatRunId?: string | null; issueId?: string | null } + +/** Lease statuses where Paperclip still owns a provider resource. */ +const HELD_LEASE_STATUSES = new Set(["active", "retained", "pending_cleanup"]); +/** Sandbox states that no longer hold anything. */ +const GONE_SANDBOX_STATES = new Set(["destroyed", "destroying"]); +/** + * The Daytona plugin may archive a sandbox when it releases the lease + * (`archiveOnRelease`, auto-deleted after 60 min) and Daytona auto-archives + * stopped ones. An archive behind a lease Paperclip released holds no compute. + */ +const ARCHIVED_SANDBOX_STATES = new Set(["archived", "archiving"]); + +export interface SandboxLeak { sandboxId: string; state: string | null; runId: string | null; leaseId: string | null; leaseStatus: string | null; reason: string } +export interface LeaseLeak { leaseId: string; status: string; providerLeaseId: string | null; runId: string | null; issueId: string | null } + +export interface DaytonaSweepResult { + companyId: string; + environmentId: string; + checkedAt: string; + liveSandboxes: string[]; + /** Live sandboxes no held lease accounts for, or whose lease Paperclip already released. */ + orphanSandboxes: SandboxLeak[]; + /** Leases still `active` or `pending_cleanup` with every task at rest. */ + heldLeases: LeaseLeak[]; + /** Leases kept on purpose (`retained`), reported but not a leak. */ + retainedLeases: LeaseLeak[]; + /** Archived sandboxes behind a released lease: expected and reported; Daytona deletes them. */ + archivedSandboxes: string[]; + error: string | null; +} + +/** Pure reconciliation of one listing. */ +export function reconcileSandboxes(sandboxes: readonly DaytonaSandboxRow[], leases: readonly LeaseRow[]): + Pick { + const live = sandboxes.filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())); + const byProviderId = new Map(leases.filter(lease => lease.providerLeaseId).map(lease => [lease.providerLeaseId!, lease])); + const orphanSandboxes: SandboxLeak[] = []; + const archivedSandboxes: string[] = []; + for (const sandbox of live) { + const lease = byProviderId.get(sandbox.id) ?? (sandbox.name ? byProviderId.get(sandbox.name) : undefined); + if (lease && HELD_LEASE_STATUSES.has(lease.status)) continue; + if (lease && ARCHIVED_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())) { archivedSandboxes.push(sandbox.id); continue; } + orphanSandboxes.push({ sandboxId: sandbox.id, state: sandbox.state ?? null, runId: sandbox.labels?.["paperclip-run-id"] ?? null, + leaseId: lease?.id ?? null, leaseStatus: lease?.status ?? null, + reason: lease ? `sandbox alive after its lease became ${lease.status}` : "no lease references this sandbox" }); + } + const toLeak = (lease: LeaseRow): LeaseLeak => ({ leaseId: lease.id, status: lease.status, providerLeaseId: lease.providerLeaseId ?? null, + runId: lease.heartbeatRunId ?? null, issueId: lease.issueId ?? null }); + return { + liveSandboxes: live.map(sandbox => sandbox.id), + orphanSandboxes, + heldLeases: leases.filter(lease => lease.status === "active" || lease.status === "pending_cleanup").map(toLeak), + retainedLeases: leases.filter(lease => lease.status === "retained").map(toLeak), + archivedSandboxes, + }; +} + +export const sweepHasLeaks = (result: DaytonaSweepResult) => Boolean(result.error || result.orphanSandboxes.length || result.heldLeases.length); + +/** Leak descriptions for one task, matched by its run ids and issue id. */ +export function leaksForTask(result: DaytonaSweepResult, runIds: ReadonlySet, issueId: string | null): string[] { + return [ + ...result.orphanSandboxes.filter(leak => leak.runId && runIds.has(leak.runId)).map(leak => `orphan_sandbox=${leak.sandboxId}`), + ...result.heldLeases.filter(lease => (lease.runId && runIds.has(lease.runId)) || (issueId && lease.issueId === issueId)) + .map(lease => `${lease.status}_lease=${lease.leaseId}`), + ]; +} + +/** Leaks no task claimed (setup probes, sandboxes without a run label). */ +export function unattributedLeaks(result: DaytonaSweepResult, attributed: ReadonlySet): string[] { + return [ + ...result.orphanSandboxes.map(leak => `orphan_sandbox=${leak.sandboxId}`), + ...result.heldLeases.map(lease => `${lease.status}_lease=${lease.leaseId}`), + ].filter(leak => !attributed.has(leak)); +} + + +/** The only Daytona operation the harness may perform. */ +export interface ReadOnlyDaytona { list(query: { labels: Record }): AsyncIterable } + +/** + * Wraps the SDK so nothing but a label-filtered `list` is reachable: the + * returned object has no stop, archive, delete or label-setting method. + */ +export function readOnlyDaytona(client: { list(query?: { labels?: Record }): AsyncIterable }): ReadOnlyDaytona { + // The SDK is untyped here (it lives in the plugin's own install), so check its shape at runtime. + if (typeof client?.list !== "function") throw new Error("Daytona SDK client has no list(); the pinned SDK version changed"); + return Object.freeze({ list: (query: { labels: Record }) => client.list(query) }); +} + +/** Fleet sandboxes the harness must never act on, even if their labels matched. */ +export function isProtectedSandbox(sandbox: Pick): boolean { + return (sandbox.name ?? "").startsWith("paperclip-create-") || (sandbox.snapshot ?? "").startsWith("fleet-sbx-"); +} + +/** + * Hard guard for any future mutating path. It refuses `paperclip-create-*` + * names and `fleet-sbx-*` snapshots outright, and anything not carrying both + * of the wave's UUID labels. Every plugin-created sandbox is + * `paperclip-create-*`, so the harness can never mutate one directly. + */ +export function assertHarnessMayMutate(sandbox: DaytonaSandboxRow, wave: { companyId: string; environmentId: string }): void { + if (isProtectedSandbox(sandbox)) throw new Error(`Refusing to change protected Daytona sandbox ${sandbox.id} (${sandbox.name ?? sandbox.snapshot})`); + if (selectWaveSandboxes([sandbox], wave).length !== 1) throw new Error(`Refusing to change Daytona sandbox ${sandbox.id}: it is not labelled for this wave`); +} + +const UUID = /^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i; + +/** Keeps only rows whose company AND environment labels equal this wave's UUIDs. */ +export function selectWaveSandboxes(rows: readonly DaytonaSandboxRow[], wave: { companyId: string; environmentId: string }): DaytonaSandboxRow[] { + if (!UUID.test(wave.companyId) || !UUID.test(wave.environmentId)) throw new Error("Wave sandbox selection needs the wave's company and environment UUIDs"); + return rows.filter(row => row.labels?.["paperclip-company-id"] === wave.companyId + && row.labels?.["paperclip-environment-id"] === wave.environmentId); +} + +export class DaytonaWaveSweeper { + private client: ReadOnlyDaytona | null = null; + constructor(private readonly input: { apiKey: string; companyId: string; environmentId: string; api: ChaosApi }) {} + + private async daytona(): Promise { + this.client ??= readOnlyDaytona(await createRemoteFixtureClient(this.input.apiKey) as unknown as Parameters[0]); + return this.client; + } + + /** Server-side label filter, then the same check again on every row. */ + async listWaveSandboxes(): Promise { + const wave = { companyId: this.input.companyId, environmentId: this.input.environmentId }; + const rows: DaytonaSandboxRow[] = []; + for await (const sandbox of (await this.daytona()).list({ labels: { + "paperclip-company-id": wave.companyId, "paperclip-environment-id": wave.environmentId } })) rows.push(sandbox); + return selectWaveSandboxes(rows, wave); + } + + /** Reconcile, retrying for `settleMs` so normal per-turn teardown can finish. */ + async sweep(settleMs = 120_000): Promise { + const deadline = Date.now() + settleMs; + for (;;) { + const result: DaytonaSweepResult = { companyId: this.input.companyId, environmentId: this.input.environmentId, + checkedAt: new Date().toISOString(), liveSandboxes: [], orphanSandboxes: [], heldLeases: [], retainedLeases: [], archivedSandboxes: [], error: null }; + try { + const [sandboxes, leases] = await Promise.all([ + this.listWaveSandboxes(), + this.input.api.get(`/api/environments/${this.input.environmentId}/leases`), + ]); + Object.assign(result, reconcileSandboxes(sandboxes, leases)); + } catch (error) { + result.error = error instanceof Error ? error.message : String(error); + } + if (!sweepHasLeaks(result) || Date.now() >= deadline) return result; + await delay(10_000); + } + } + + /** + * Cleanup through Paperclip only: delete the wave's own environment (with its + * reusable leases). Then list, read-only, what is still not destroyed or + * archived, for the report. No sandbox is changed by the harness itself. + */ + async cleanUp(settleMs = 90_000): Promise<{ environmentDeleted: boolean; remainingSandboxes: string[]; errors: string[] }> { + const errors: string[] = []; + let environmentDeleted = false; + try { + await deleteDaytonaEnvironment(this.input.api, this.input.environmentId); + environmentDeleted = true; + } catch (error) { errors.push(`environment delete: ${error instanceof Error ? error.message : String(error)}`); } + let remainingSandboxes: string[] = []; + // Provider teardown is asynchronous; give it a bounded settle before reporting. + const deadline = Date.now() + settleMs; + for (;;) { + try { + remainingSandboxes = (await this.listWaveSandboxes()) + .filter(sandbox => !GONE_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase()) + && !ARCHIVED_SANDBOX_STATES.has(String(sandbox.state ?? "").toLowerCase())) + .map(sandbox => `${sandbox.id}:${sandbox.state}`); + } catch (error) { + errors.push(`sandbox listing: ${error instanceof Error ? error.message : String(error)}`); + break; + } + if (!remainingSandboxes.length || Date.now() >= deadline) break; + await delay(10_000); + } + return { environmentDeleted, remainingSandboxes, errors }; + } +} diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 51b29bbcdb6..c173f86ed5f 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -179,6 +179,7 @@ export function gradeR6(record: TaskRecord): RuleResult { ...(sweep.activeRuns.length ? [`active_runs=${sweep.activeRuns.join(",")}`] : []), ...(sweep.runningWorkspaceOperations.length ? [`running_workspace_operations=${sweep.runningWorkspaceOperations.join(",")}`] : []), ...(sweep.activeRecoveryAction ? ["active_recovery_action"] : []), + ...(sweep.daytonaLeaks ?? []), ]; return leaks.length ? { verdict: "fail", detail: `leak: ${leaks.join("; ")}` } : { verdict: "pass", detail: "clean rest" }; } diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 89623d437cc..60cea2129e7 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -14,6 +14,8 @@ import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApi, isSocketTypeOfServiceRace } from "./api.js"; import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; +import { assertControllerMatchesImage, prepareDaytonaImageArtifacts } from "./daytona-image.js"; +import { DaytonaWaveSweeper, leaksForTask, unattributedLeaks, type DaytonaSweepResult } from "./daytona-sweep.js"; import { knownSecretValues, writeEvidenceBundle } from "./evidence.js"; import { gradeTask } from "./grader.js"; import { appendLedger, ledgerLine, writeSummary, type LedgerLine } from "./ledger.js"; @@ -48,8 +50,41 @@ async function loadPlan(): Promise<{ plan: WavePlan; keep: boolean }> { return { plan: planWave(waveIdFor(command.options.seed), command.options), keep: command.keep }; } +function run(command: string, args: string[], cwd: string, label: string) { + const result = spawnSync(command, args, { cwd, stdio: "inherit" }); + if (result.status !== 0) throw new Error(`${label} failed (${command} ${args.join(" ")})`); +} + +/** The bundled Daytona plugin is outside the root workspace; prepare it as CI does. */ +function ensureDaytonaPlugin() { + const root = path.join(repositoryRoot, "packages/plugins/sandbox-providers/daytona"); + if (["dist/manifest.js", "dist/worker.js", "node_modules/@daytonaio/sdk"].every(file => existsSync(path.join(root, file)))) return; + run("pnpm", ["install", "--ignore-workspace", "--frozen-lockfile", "--ignore-scripts"], root, "Daytona plugin install"); + run(process.execPath, [path.join(repositoryRoot, "scripts/link-plugin-dev-sdk.mjs")], repositoryRoot, "Plugin SDK link"); + run("pnpm", ["--dir", root, "build"], repositoryRoot, "Daytona plugin build"); +} + +/** + * `--env daytona`: extract the image's Linux runnerd and provider pack (cached + * per image digest, no Docker) and refuse to run a controller whose runner + * source differs from the image. + */ +async function prepareDaytona(plan: WavePlan, timeline: Timeline): Promise> { + const image = plan.options.daytonaImage!; + const artifacts = await prepareDaytonaImageArtifacts({ image, cacheRoot: path.join(resultsRoot, "daytona-images"), + log: line => console.info(`[daytona-image] ${line}`) }); + const { localContentId } = await assertControllerMatchesImage(artifacts, repositoryRoot); + timeline.add("daytona_image_matched", { image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision, + runnerdSha256: artifacts.runnerdSha256, providerPackDigest: artifacts.providerPackDigest, platformManifestDigest: artifacts.platformManifestDigest }); + ensureDaytonaPlugin(); + timeline.add("daytona_vcpu_plan", { cpuPerSandbox: plan.options.daytonaCpu, concurrency: plan.options.concurrency, + taskVcpu: plan.options.concurrency * (plan.options.daytonaCpu ?? 0) }); + return { PAPERCLIP_RUNNER_REMOTE_BINARY_PATH: artifacts.runnerdPath, PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: artifacts.providerPackPath }; +} + function preflight(plan: WavePlan) { - const missing = [...new Set(plan.tasks.map(task => CREDENTIAL_FOR_ADAPTER[task.adapter]))].filter(key => !process.env[key]); + const missing = [...new Set([...plan.tasks.map(task => CREDENTIAL_FOR_ADAPTER[task.adapter]), + ...(plan.options.env === "daytona" ? ["DAYTONA_API_KEY"] : [])])].filter(key => !process.env[key]); if (missing.length) throw new Error(`Missing credentials ${missing.join(", ")}. Run: set -a; source ~/.secrets; set +a`); const runner = process.env.PAPERCLIP_RUNNER_BINARY ?? path.join(repositoryRoot, "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"); @@ -71,8 +106,11 @@ async function main(): Promise { timeline.add("wave_planned", { waveId: plan.waveId, seed: plan.seed, gitSha, gitDirty, replayOf: plan.replayOf ?? null, tasks: plan.tasks.map(task => `${task.taskId}:${task.adapter}/${task.workload}/${task.fault}${task.trigger ? `^${task.trigger}+${task.triggerDelayMs ?? 0}ms` : ""}@${task.fireAfterMs}ms${task.holdMs ? ` hold ${task.holdMs}ms` : ""}`) }); preflight(plan); + const serverEnvironment = plan.options.env === "daytona" ? await prepareDaytona(plan, timeline) : {}; const supervisor = await ChaosSupervisor.create({ repositoryRoot, waveDir, timeline }); + let sweeper: DaytonaWaveSweeper | null = null; + let sandboxSweep: DaytonaSweepResult | null = null; let interrupted = false; const onSignal = (signal: NodeJS.Signals) => { if (!interrupted) { interrupted = true; timeline.add("harness_interrupted", { signal }); } }; process.on("SIGINT", onSignal); process.on("SIGTERM", onSignal); @@ -90,11 +128,15 @@ async function main(): Promise { let faultFailures = 0; let faultsQueued = 0; try { - await supervisor.start(); + await supervisor.start(serverEnvironment); const api = new ChaosApi(supervisor.baseUrl); - const lab = await seedChaosLab({ api, env: plan.options.env, adapters: [...new Set(plan.tasks.map(task => task.adapter))], + const lab = await seedChaosLab({ api, env: plan.options.env, daytonaImage: plan.options.daytonaImage, daytonaCpu: plan.options.daytonaCpu, + adapters: [...new Set(plan.tasks.map(task => task.adapter))], concurrency: plan.options.concurrency, budgetCents: plan.options.budgetCents, - fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline }); + fixtureRoot: path.join(supervisor.paths.temporaryRoot, "fixtures"), waveId: plan.waveId, timeline, + onEnvironment: owned => { + sweeper = new DaytonaWaveSweeper({ apiKey: process.env.DAYTONA_API_KEY!, companyId: owned.companyId, environmentId: owned.environmentId, api }); + } }); // The instance root holds the secrets master key and the database: oracle code may not read it. const protectedRoots = [os.homedir(), supervisor.paths.temporaryRoot, resultsRoot]; const observers = plan.tasks.map(task => new TaskObserver(task, api, lab, timeline, supervisor, protectedRoots)); @@ -152,9 +194,20 @@ async function main(): Promise { const quietFor = Date.now() - lastDoneAt; if (quietFor < QUIET_BEFORE_SWEEP_MS) await delay(QUIET_BEFORE_SWEEP_MS - quietFor); timeline.add("wave_sweep", {}); + if (sweeper) { + sandboxSweep = await (sweeper as DaytonaWaveSweeper).sweep(); + timeline.add("daytona_sweep", { ...sandboxSweep }); + } + const attributed = new Set(); const secrets = knownSecretValues(); for (const observer of observers) { observer.record.sweep = await observer.sweep(); + if (sandboxSweep) { + const runIds = new Set(observer.record.samples.flatMap(sample => (sample.runs ?? []).map(run => run.id))); + const leaks = leaksForTask(sandboxSweep, runIds, observer.record.issueId); + for (const leak of leaks) attributed.add(leak); + observer.record.sweep.daytonaLeaks = leaks; + } let artifacts: OracleArtifacts | null = null; if (observer.record.issueId) artifacts = await observer.collectArtifacts().catch(() => null); const oracle = artifacts ? observer.definition.grade(observer.plan.nonce, artifacts) @@ -173,14 +226,43 @@ async function main(): Promise { rules: Object.fromEntries(Object.entries(grade.rules).map(([id, rule]) => [id, rule.verdict])), T: grade.T.verdict }); } await writeFile(path.join(waveDir, "faults.json"), `${JSON.stringify(faults, null, 2)}\n`, { mode: 0o600 }); + // R6 at wave level: a leak no task's runs account for (e.g. the setup probe) still fails the wave. + if (sandboxSweep) { + const unattributed = unattributedLeaks(sandboxSweep, attributed); + if (sandboxSweep.error || unattributed.length) { + anyFailure = true; + timeline.add("daytona_sweep_wave_leak", { error: sandboxSweep.error, unattributed }); + } + } } catch (error) { harnessError = error; timeline.add("harness_error", { error: error instanceof Error ? error.stack ?? error.message : String(error) }); } finally { - try { await supervisor.stop(); } catch (error) { - // Incomplete cleanup is a harness failure; keep the instance for inspection. + // A failure here (for example writing the report) must not skip stopping the controller and Postgres. + try { if (sweeper) try { + const owned = sweeper as DaytonaWaveSweeper; + // An interrupted wave still reports what it leaked before cleanup. + sandboxSweep ??= await owned.sweep(0); + // Cleanup only through Paperclip; the harness never changes a sandbox. + const cleanup = await owned.cleanUp(); + timeline.add("daytona_cleanup", cleanup); + console.info(`Daytona R6 sweep: ${sandboxSweep.liveSandboxes.length} live, ${sandboxSweep.archivedSandboxes.length} archived after release, ` + + `${sandboxSweep.orphanSandboxes.length} orphaned, ` + + `${sandboxSweep.heldLeases.length} held leases${sandboxSweep.error ? `, error ${sandboxSweep.error}` : ""}; ` + + `environment deleted: ${cleanup.environmentDeleted}; still running after cleanup: ${cleanup.remainingSandboxes.join(", ") || "none"}`); + // A sandbox neither archived nor destroyed after cleanup is a leak for a person to act on. + if (cleanup.remainingSandboxes.length) anyFailure = true; + if (cleanup.errors.length) harnessError ??= new Error(`Daytona cleanup incomplete: ${cleanup.errors.join("; ")}`); + await writeFile(path.join(waveDir, "sandbox-sweep.json"), `${JSON.stringify({ sweep: sandboxSweep, cleanup }, null, 2)}\n`, { mode: 0o600 }); + } catch (error) { harnessError ??= error; - timeline.add("supervisor_stop_failed", { error: String(error) }); + timeline.add("daytona_cleanup_failed", { error: String(error) }); + } } finally { + try { await supervisor.stop(); } catch (error) { + // Incomplete cleanup is a harness failure; keep the instance for inspection. + harnessError ??= error; + timeline.add("supervisor_stop_failed", { error: String(error) }); + } } if (lines.length) { await appendLedger(resultsRoot, lines); diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 9af1752ed34..3094909e8a9 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -11,7 +11,7 @@ import type { TaskPlan } from "./types.js"; import { BENCH_EXPECTED, parseBenchRecord } from "./workloads.js"; process.env.RESTART_CHAOS_QUIET = "1"; -const lab: ChaosLab = { companyId: "c1", environmentId: "e1", projectId: "p1", fixtureRepositoryPath: "/tmp/none", +const lab: ChaosLab = { companyId: "c1", environmentId: "e1", environmentKind: "local", projectId: "p1", fixtureRepositoryPath: "/tmp/none", agents: { codex: { id: "a1", name: "Chaos codex" }, claude: { id: "a2", name: "Chaos claude" }, opencode: { id: "a3", name: "Chaos opencode" } } }; const plan: TaskPlan = { taskId: "t01", index: 0, workload: "C1", adapter: "codex", fault: "K1", fireAfterMs: 999_999, holdMs: 0, followUpAfterMs: 1_000, nonce: "abcd1234" }; diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index 491d689760d..5445c805e79 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -114,3 +114,17 @@ describe("CLI parsing", () => { expect(() => parseCommand(["--poll-ms", "500"])).toThrow(/between 2000 and 5000/); }); }); + +describe("Daytona options", () => { + const image = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"a".repeat(64)}`; + it("defaults to 4 vCPU sandboxes, excludes B5, and keeps the image in the plan", () => { + const parsed = parseCommand(["--env", "daytona", "--daytona-image", image, "--concurrency", "10"]); + expect(parsed).toMatchObject({ mode: "wave", options: { daytonaImage: image, daytonaCpu: 4, concurrency: 10 } }); + if (parsed.mode === "wave") expect(parsed.options.workloads).not.toContain("B5"); + }); + it("refuses waves over the 40 vCPU chaos budget, mutable images, and Daytona flags without Daytona", () => { + expect(() => parseCommand(["--env", "daytona", "--daytona-image", image, "--concurrency", "11"])).toThrow(/exceeds the Daytona vCPU budget of 40/); + expect(() => parseCommand(["--env", "daytona", "--daytona-image", "ghcr.io/x/y:latest"])).toThrow(/immutable runner image/); + expect(() => parseCommand(["--env", "local", "--daytona-cpu", "2"])).toThrow(/only to --env daytona/); + }); +}); diff --git a/tests/restart-chaos/seed.ts b/tests/restart-chaos/seed.ts index 2799fb2384f..8b608026414 100644 --- a/tests/restart-chaos/seed.ts +++ b/tests/restart-chaos/seed.ts @@ -1,8 +1,10 @@ import { execFileSync } from "node:child_process"; import { chmod, mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; -import { runnerProfiles } from "../runner-e2e/catalog.js"; -import type { SecretReference, SecretReferenceMap } from "../runner-e2e/types.js"; +import type { RunnerApi } from "../runner-e2e/api.js"; +import { runnerEnvironments, runnerProfiles } from "../runner-e2e/catalog.js"; +import { installFixtureDaytonaProvider, waitForFixtureDaytonaProvider } from "../runner-e2e/live-fixtures.js"; +import type { MatrixExecution, SecretReference, SecretReferenceMap } from "../runner-e2e/types.js"; import type { ChaosApi } from "./api.js"; import { CREDENTIAL_FOR_ADAPTER, PROFILE_FOR_ADAPTER } from "./config.js"; import type { Timeline } from "./timeline.js"; @@ -12,6 +14,7 @@ import { fixtureRepositoryFiles } from "./workloads.js"; export interface ChaosLab { companyId: string; environmentId: string; + environmentKind: EnvironmentKind; projectId: string; fixtureRepositoryPath: string; agents: Record; @@ -35,6 +38,55 @@ export async function createFixtureRepository(directory: string, benchSeconds?: return directory; } +/** The runner-e2e fixture helpers take RunnerApi's `{ timeout }` options. */ +function runnerApiView(api: ChaosApi) { + return { + get: (route: string, options?: { timeout?: number }) => api.get(route, options?.timeout), + post: (route: string, body?: unknown, options?: { timeout?: number }) => api.post(route, body, options?.timeout), + } as unknown as Pick; +} + +/** + * Daytona environment exactly as runner-e2e creates it: the bundled Daytona + * plugin (live-fixtures.ts installFixtureDaytonaProvider) and the `daytona` + * fixture from catalog.ts runnerEnvironments, pinned to the immutable image. + */ +export function chaosDaytonaShape(cpu: number) { + // The runner-e2e fixture's 4 GiB memory and 10 GiB disk, with the vCPU count + // chosen by --daytona-cpu (4 by default, like the fixture). 2 vCPU with this + // memory and disk passed a 3-task wave on a 10 vCPU org. + return { cpu, memory: 4, disk: 10 }; +} + +async function createDaytonaEnvironment(input: { api: ChaosApi; companyId: string; image: string; cpu: number; waveId: string; + secretRefs: SecretReferenceMap; timeline: Timeline }): Promise { + const fixture = runnerEnvironments.find(candidate => candidate.id === "daytona"); + if (!fixture) throw new Error("runner-e2e catalog has no daytona environment fixture"); + const view = runnerApiView(input.api); + await waitForFixtureDaytonaProvider(view, { companyId: input.companyId, deadlineAt: Date.now() + 120_000 }); + const payload = fixture.buildEnvironment({ secretRefs: input.secretRefs, daytonaImage: input.image, executionId: input.waveId }); + const shape = chaosDaytonaShape(input.cpu); + const environment = await input.api.post<{ id: string; driver: string }>(`/api/companies/${input.companyId}/environments`, + { ...payload, config: { ...(payload.config as Record), ...shape } }); + input.timeline.add("seed_environment", { environmentId: environment.id, driver: environment.driver, image: input.image, ...shape }); + return environment.id; +} + +/** + * Native Codex on a sandbox needs the image's platform qualification recorded + * first, as in live-fixtures.ts (the agent is not created otherwise). + */ +async function qualifyNativeCodex(input: { api: ChaosApi; companyId: string; environmentId: string; timeline: Timeline }) { + const startedAt = Date.now(); + const probe = await input.api.post<{ ok: boolean; details?: { platformQualificationRecorded?: boolean; leaseId?: string } }>( + `/api/environments/${input.environmentId}/probe?companyId=${encodeURIComponent(input.companyId)}`, {}, 330_000); + input.timeline.add("seed_codex_qualification", { ok: probe.ok, recorded: probe.details?.platformQualificationRecorded ?? false, + leaseId: probe.details?.leaseId ?? null, durationMs: Date.now() - startedAt }); + if (probe.ok !== true || probe.details?.platformQualificationRecorded !== true) { + throw new Error("Native Codex image qualification was not recorded; the Codex agent was not created."); + } +} + /** * Seeds the isolated instance. Credentials cross into the server only as * encrypted company secrets, as in tests/runner-e2e/live-fixtures.ts, and the @@ -43,20 +95,23 @@ export async function createFixtureRepository(directory: string, benchSeconds?: export async function seedChaosLab(input: { api: ChaosApi; env: EnvironmentKind; + daytonaImage?: string; + daytonaCpu?: number; adapters: readonly AdapterId[]; concurrency: number; budgetCents: number; fixtureRoot: string; waveId: string; timeline: Timeline; + /** Called as soon as the wave owns external resources that need cleanup. */ + onEnvironment?: (lab: Pick) => void; }): Promise { const { api, timeline } = input; if (input.env === "daytona") { - // TODO(restart-chaos phase 1b): create the Daytona sandbox environment from - // the prepared image (see tests/runner-e2e/catalog.ts runnerEnvironments - // and live-fixtures.ts installFixtureDaytonaProvider) and add the R6 - // sandbox sweep. The image and env are being prepared separately. - throw new Error("--env daytona is not wired yet; use --env local"); + if (!input.daytonaImage) throw new Error("--env daytona needs --daytona-image"); + // Installed before the company exists so the provider worker is warm. + await installFixtureDaytonaProvider(runnerApiView(input.api), { environment: { id: "daytona" } } as MatrixExecution); + timeline.add("seed_daytona_plugin", {}); } const fixtureRepositoryPath = await createFixtureRepository(path.join(input.fixtureRoot, "chaos-sandbox")); const company = await api.post<{ id: string }>("/api/companies", { @@ -67,27 +122,37 @@ export async function seedChaosLab(input: { timeline.add("seed_company", { companyId: company.id }); const secretRefs: SecretReferenceMap = {}; - for (const adapter of input.adapters) { - const key = CREDENTIAL_FOR_ADAPTER[adapter]; - if (secretRefs[key]) continue; + const credentialKeys = [...new Set([...input.adapters.map(adapter => CREDENTIAL_FOR_ADAPTER[adapter]), + ...(input.env === "daytona" ? ["DAYTONA_API_KEY" as const] : [])])]; + for (const key of credentialKeys) { const value = process.env[key]; - if (!value) throw new Error(`Missing ${key} for adapter ${adapter}; load ~/.secrets into the environment first`); + if (!value) throw new Error(`Missing ${key}; load ~/.secrets into the environment first`); const created = await api.post<{ id: string }>(`/api/companies/${company.id}/secrets`, { - name: `Chaos ${key}`, key, value, description: "Restart chaos harness provider credential", + name: `Chaos ${key}`, key, value, description: "Restart chaos harness credential", }); secretRefs[key] = { type: "secret_ref", secretId: created.id, version: "latest" } satisfies SecretReference; } timeline.add("seed_secrets", { keys: Object.keys(secretRefs) }); - const environments = await api.get>(`/api/companies/${company.id}/environments?driver=local`); - const environment = environments.find(candidate => candidate.driver === "local"); - if (!environment) throw new Error("Isolated instance has no managed local environment"); + let environmentId: string; + if (input.env === "daytona") { + environmentId = await createDaytonaEnvironment({ api, companyId: company.id, image: input.daytonaImage!, cpu: input.daytonaCpu ?? 4, + waveId: input.waveId, secretRefs, timeline }); + input.onEnvironment?.({ companyId: company.id, environmentId, environmentKind: "daytona" }); + if (input.adapters.includes("codex")) await qualifyNativeCodex({ api, companyId: company.id, environmentId, timeline }); + } else { + const environments = await api.get>(`/api/companies/${company.id}/environments?driver=local`); + const environment = environments.find(candidate => candidate.driver === "local"); + if (!environment) throw new Error("Isolated instance has no managed local environment"); + environmentId = environment.id; + } const agents = {} as ChaosLab["agents"]; for (const adapter of input.adapters) { const profile = runnerProfiles.find(candidate => candidate.id === PROFILE_FOR_ADAPTER[adapter]); if (!profile) throw new Error(`runner-e2e catalog has no profile ${PROFILE_FOR_ADAPTER[adapter]}`); - const payload = profile.buildAgent({ environmentId: environment.id, environmentFixtureId: "local", + if (!profile.supportedEnvironments.includes(input.env)) throw new Error(`Profile ${profile.id} does not support ${input.env}`); + const payload = profile.buildAgent({ environmentId, environmentFixtureId: input.env, workspacePath: fixtureRepositoryPath, secretRefs, executionId: input.waveId }); const name = `Chaos ${adapter}`; const runtimeConfig = (payload.runtimeConfig ?? {}) as Record; @@ -99,7 +164,7 @@ export async function seedChaosLab(input: { enabled: false, maxConcurrentRuns: Math.max(1, input.concurrency) } }, }); agents[adapter] = { id: created.id, name }; - timeline.add("seed_agent", { adapter, agentId: created.id, profile: profile.id, model: profile.model }); + timeline.add("seed_agent", { adapter, agentId: created.id, profile: profile.id, model: profile.model, environmentId }); } // The project's git_worktree policy only takes effect when the instance has @@ -119,11 +184,11 @@ export async function seedChaosLab(input: { enabled: true, defaultMode: "isolated_workspace", allowIssueOverride: false, - environmentId: environment.id, + environmentId, workspaceStrategy: { type: "git_worktree" }, }, workspace: { name: "Primary", sourceType: "local_path", cwd: fixtureRepositoryPath, isPrimary: true }, }); timeline.add("seed_project", { projectId: project.id, fixtureRepositoryPath }); - return { companyId: company.id, environmentId: environment.id, projectId: project.id, fixtureRepositoryPath, agents }; + return { companyId: company.id, environmentId, environmentKind: input.env, projectId: project.id, fixtureRepositoryPath, agents }; } diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index e309a16a343..c27ab268a17 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -112,7 +112,8 @@ export class ChaosSupervisor { try { return statSync(this.paths.serverLogPath).size + this.log.writableLength; } catch { return 0; } } - async start(): Promise { + /** `extraEnvironment` is kept for every restart (e.g. the Daytona image's runnerd and provider pack paths). */ + async start(extraEnvironment: Record = {}): Promise { const { temporaryRoot, home, configPath } = this.paths; const reservation = await prepareRunnerE2EServerConfig({ temporaryRoot, configPath, serverPort: this.port }); await reservation?.close(); @@ -138,6 +139,7 @@ export class ChaosSupervisor { PAPERCLIP_STORAGE_LOCAL_DIR: path.join(temporaryRoot, "storage"), PAPERCLIP_SECRETS_PROVIDER: "local_encrypted", PAPERCLIP_SECRETS_STRICT_MODE: "true", PAPERCLIP_DB_BACKUP_ENABLED: "false", PAPERCLIP_DB_BACKUP_DIR: path.join(temporaryRoot, "backups"), PAPERCLIP_OPEN_ON_LISTEN: "false", + ...extraEnvironment, }; const environment = buildPaperclipServerEnvironment(source, overrides); assertIsolatedServerEnvironment(environment, { temporaryRoot, paperclipHome: home, configPath }); diff --git a/tests/restart-chaos/triggers.test.ts b/tests/restart-chaos/triggers.test.ts index 8a8c4e2978b..b9a06db2517 100644 --- a/tests/restart-chaos/triggers.test.ts +++ b/tests/restart-chaos/triggers.test.ts @@ -15,7 +15,7 @@ import { ledgerLine } from "./ledger.js"; import type { TaskGrade, TaskPlan, TaskRecord, TaskSample, WaveOptions, WavePlan } from "./types.js"; process.env.RESTART_CHAOS_QUIET = "1"; -const lab: ChaosLab = { companyId: "c1", environmentId: "e1", projectId: "p1", fixtureRepositoryPath: "/tmp/none", +const lab: ChaosLab = { companyId: "c1", environmentId: "e1", environmentKind: "local", projectId: "p1", fixtureRepositoryPath: "/tmp/none", agents: { codex: { id: "a1", name: "Chaos codex" }, claude: { id: "a2", name: "Chaos claude" }, opencode: { id: "a3", name: "Chaos opencode" } } }; const ready = { serverReady: true, faultInProgress: false }; const base: TaskPlan = { taskId: "t01", index: 0, workload: "A1", adapter: "codex", fault: "K1", fireAfterMs: 999_999, diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 451e044ca98..1efdf67acd0 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -28,6 +28,10 @@ export interface WaveOptions { budgetCents: number; /** Poll interval for the observer, 2–5 s per plan §5. */ pollMs: number; + /** `--env daytona`: the immutable runner image (registry/repository@sha256:…). */ + daytonaImage?: string; + /** `--env daytona`: vCPU per sandbox (see seed.ts chaosDaytonaShape). */ + daytonaCpu?: number; } export interface TaskPlan { @@ -161,6 +165,8 @@ export interface RestSweep { activeRuns: string[]; runningWorkspaceOperations: string[]; activeRecoveryAction: boolean; + /** `--env daytona`: orphaned sandboxes and held leases belonging to this task's runs. */ + daytonaLeaks?: string[]; error: string | null; }