diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index f0e8a0e65fb..5c14c5b3418 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -92,7 +92,9 @@ through Paperclip, by deleting the wave's own environment. top to bottom (honouring `.wh.` and `.wh..wh..opq` whiteouts), extracts only those two paths, checks the ELF header, the pack manifest and the image labels, records their digests in `receipt.json`, and deletes the layer blobs. The result -is cached in `results/daytona-images//`, so it runs once per image. +is cached in `tests/restart-chaos/results/daytona-images//` (or +`$RESTART_CHAOS_IMAGE_CACHE`), shared by every wave and crash-point run, so it +runs once per image. A copy that fails part-way retries from an empty directory. Run it alone with `node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/daytona-image.ts `. Before starting, the harness compares the image's `io.paperclip.runner.content-id` diff --git a/tests/restart-chaos/daytona-image.test.ts b/tests/restart-chaos/daytona-image.test.ts index 3b26cff187c..d850e952ff1 100644 --- a/tests/restart-chaos/daytona-image.test.ts +++ b/tests/restart-chaos/daytona-image.test.ts @@ -1,8 +1,9 @@ -import { mkdtemp, readFile, readlink, rm, stat } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, mkdtemp, readFile, readlink, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { assertControllerMatchesImage, extractFromLayers, normalizeLayerPath, walkTar, type LayerSource } from "./daytona-image.js"; +import { assertControllerMatchesImage, copyImageToDir, extractFromLayers, prepareDaytonaImageArtifacts, normalizeLayerPath, walkTar, type LayerSource } from "./daytona-image.js"; type Entry = | { path: string; type: "file"; content: string; mode?: number } @@ -191,3 +192,117 @@ describe("assertControllerMatchesImage", () => { await expect(assertControllerMatchesImage(image, "/unused", "e".repeat(64))).rejects.toThrow(/does not match the Daytona image[\s\S]*d{40}[\s\S]*e{64}/); }); }); + +describe("copyImageToDir", () => { + async function fakeSkopeo(root: string, failures: number): Promise { + const script = path.join(root, "skopeo"); + await writeFile(script, `#!/bin/sh +dest="\${9#dir:}" +if [ -n "$(ls -A "$dest")" ]; then echo "not a containers image directory, don't want to overwrite" >&2; exit 1; fi +n=$(cat "${root}/count" 2>/dev/null || echo 0); n=$((n+1)); echo $n > "${root}/count" +if [ $n -le ${failures} ]; then echo partial > "$dest/blob"; echo "read: operation timed out" >&2; exit 1; fi +echo '{}' > "$dest/manifest.json" +`); + await chmod(script, 0o755); + return script; + } + + it("retries a copy that failed part-way from an empty directory", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daytona-skopeo-")); + try { + const blobs = path.join(root, "blobs"); + await copyImageToDir({ skopeo: await fakeSkopeo(root, 2), image: "ghcr.io/x/y@sha256:" + "a".repeat(64), blobs }); + expect(await readFile(path.join(root, "count"), "utf8")).toBe("3\n"); + expect(await readFile(path.join(blobs, "manifest.json"), "utf8")).toBe("{}\n"); + await expect(stat(path.join(blobs, "blob"))).rejects.toThrow(); + } finally { await rm(root, { recursive: true, force: true }); } + }); + + it("gives up after the last attempt with skopeo's error", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daytona-skopeo-")); + try { + await expect(copyImageToDir({ skopeo: await fakeSkopeo(root, 5), image: "i", blobs: path.join(root, "blobs"), attempts: 2 })) + .rejects.toThrow(/operation timed out/); + expect(await readFile(path.join(root, "count"), "utf8")).toBe("2\n"); + } finally { await rm(root, { recursive: true, force: true }); } + }); +}); + +describe("prepareDaytonaImageArtifacts with a shared cache", () => { + const sha = (bytes: Buffer | string) => createHash("sha256").update(bytes).digest("hex"); + const revision = "a".repeat(40); + + async function fakeImage(root: string): Promise { + const source = path.join(root, "image"); + await mkdir(source, { recursive: true }); + const elf = Buffer.alloc(64); elf.writeUInt32BE(0x7f454c46, 0); elf[4] = 2; elf[5] = 1; elf.writeUInt16LE(62, 18); + const pack = JSON.stringify({ schema: "paperclip-runner/remote-provider-pack/v1", digest: `sha256:${"b".repeat(64)}`, + payload: { runnerSourceRevision: revision } }); + const layerBytes = tar([ + { path: RUNNERD, type: "file", content: elf.toString("latin1"), mode: 0o755 }, + { path: `${PACK}/provider-pack.json`, type: "file", content: pack }, + ]); + const config = Buffer.from(JSON.stringify({ os: "linux", architecture: "amd64", + config: { Labels: { "io.paperclip.runner.content-id": "c".repeat(64), "org.opencontainers.image.revision": revision } } })); + await writeFile(path.join(source, sha(layerBytes)), layerBytes); + await writeFile(path.join(source, sha(config)), config); + await writeFile(path.join(source, "manifest.json"), JSON.stringify({ config: { digest: `sha256:${sha(config)}` }, + layers: [{ digest: `sha256:${sha(layerBytes)}`, mediaType: "application/vnd.oci.image.layer.v1.tar" }] })); + const barrier = path.join(root, "barrier"); + await mkdir(barrier); + const script = path.join(root, "skopeo"); + await writeFile(script, `#!/bin/sh +dest="\${9#dir:}" +touch "${barrier}/$$" +while [ "$(ls "${barrier}" | wc -l)" -lt 2 ]; do sleep 0.05; done +cp -R "${source}/." "$dest/" +`); + await chmod(script, 0o755); + return script; + } + + it("lets two runs extract the same image at once; both get a usable receipt", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daytona-cache-")); + directories.push(root); + const skopeo = await fakeImage(root); + const image = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"d".repeat(64)}`; + const cacheRoot = path.join(root, "cache"); + const [first, second] = await Promise.all([ + prepareDaytonaImageArtifacts({ image, cacheRoot, skopeo }), + prepareDaytonaImageArtifacts({ image, cacheRoot, skopeo }), + ]); + expect(second.runnerdPath).toBe(first.runnerdPath); + for (const receipt of [first, second]) { + expect((await stat(receipt.runnerdPath)).isFile()).toBe(true); + expect(await readFile(path.join(receipt.providerPackPath, "provider-pack.json"), "utf8")).toContain(revision); + } + }); + + it("leaves a corrupt leftover in place and publishes beside it", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daytona-cache-")); + directories.push(root); + const skopeo = await fakeImage(root); + await writeFile(path.join(root, "barrier", "other"), ""); + const image = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"e".repeat(64)}`; + const cacheRoot = path.join(root, "cache"); + await mkdir(path.join(cacheRoot, "e".repeat(64)), { recursive: true }); + await writeFile(path.join(cacheRoot, "e".repeat(64), "junk"), "partial"); + const receipt = await prepareDaytonaImageArtifacts({ image, cacheRoot, skopeo }); + expect((await stat(receipt.runnerdPath)).isFile()).toBe(true); + expect(receipt.runnerdPath.startsWith(path.join(cacheRoot, `${"e".repeat(64)}.`))).toBe(true); + expect(await exists(cacheRoot, `${"e".repeat(64)}/junk`)).toBe(true); + }); + + it("two runs replacing a corrupt leftover at once both keep a usable cache", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "daytona-cache-")); + directories.push(root); + const skopeo = await fakeImage(root); + const image = `ghcr.io/paperclipai/paperclip-daytona-runner@sha256:${"f".repeat(64)}`; + const cacheRoot = path.join(root, "cache"); + await mkdir(path.join(cacheRoot, "f".repeat(64)), { recursive: true }); + await writeFile(path.join(cacheRoot, "f".repeat(64), "junk"), "partial"); + const receipts = await Promise.all([1, 2].map(() => prepareDaytonaImageArtifacts({ image, cacheRoot, skopeo }))); + for (const receipt of receipts) expect((await stat(receipt.runnerdPath)).isFile()).toBe(true); + expect(await exists(cacheRoot, `${"f".repeat(64)}/junk`)).toBe(true); + }); +}); diff --git a/tests/restart-chaos/daytona-image.ts b/tests/restart-chaos/daytona-image.ts index c7e1a33c20a..a70d45212cb 100644 --- a/tests/restart-chaos/daytona-image.ts +++ b/tests/restart-chaos/daytona-image.ts @@ -327,6 +327,29 @@ async function readReceipt(directory: string): Promise void }): Promise { + const attempts = input.attempts ?? 3; + const log = input.log ?? (() => undefined); + for (let attempt = 1; ; attempt += 1) { + await rm(input.blobs, { recursive: true, force: true }); + await mkdir(input.blobs, { recursive: true, mode: 0o700 }); + log(`skopeo copy ${input.image} (linux/amd64) → ${input.blobs} (attempt ${attempt}/${attempts})`); + try { + await execFileAsync(input.skopeo, ["copy", "--retry-times", "3", "--override-os", "linux", "--override-arch", "amd64", + `docker://${input.image}`, `dir:${input.blobs}`], { maxBuffer: 16 * 1024 * 1024 }); + return; + } catch (error) { + if (attempt >= attempts) throw error; + log(`skopeo copy failed (${error instanceof Error ? error.message.split("\n")[0] : String(error)}); retrying into an empty directory`); + } + } +} + /** * Returns the verified runnerd and provider pack of `image`, extracting them * once per image digest into `cacheRoot//`. @@ -338,17 +361,12 @@ export async function prepareDaytonaImageArtifacts(input: { image: string; cache const finalDir = path.join(path.resolve(input.cacheRoot), match[1]!); const cached = await readReceipt(finalDir); if (cached) { log(`Daytona image artifacts cached at ${finalDir}`); return cached; } - await rm(finalDir, { recursive: true, force: true }); const staging = path.join(path.resolve(input.cacheRoot), `.staging-${match[1]!.slice(0, 12)}-${randomBytes(4).toString("hex")}`); const blobs = path.join(staging, "blobs"); const rootfs = path.join(staging, "rootfs"); try { - await mkdir(blobs, { recursive: true, mode: 0o700 }); - log(`skopeo copy ${input.image} (linux/amd64) → ${blobs}`); - await execFileAsync(input.skopeo ?? process.env.SKOPEO ?? "skopeo", ["copy", "--retry-times", "3", - "--override-os", "linux", "--override-arch", "amd64", `docker://${input.image}`, `dir:${blobs}`], - { maxBuffer: 16 * 1024 * 1024 }); + await copyImageToDir({ skopeo: input.skopeo ?? process.env.SKOPEO ?? "skopeo", image: input.image, blobs, log }); const manifestBytes = await readFile(path.join(blobs, "manifest.json")); const manifest = JSON.parse(manifestBytes.toString("utf8")) as { config: { digest: string }; layers: Array<{ digest: string; mediaType: string }> }; if (!manifest.config?.digest || !Array.isArray(manifest.layers)) throw new Error("skopeo did not produce a single-platform image manifest"); @@ -390,11 +408,27 @@ export async function prepareDaytonaImageArtifacts(input: { image: string; cache providerPackPath: path.join(root, "rootfs", ...PROVIDER_PACK_PATH.split("/")), providerPackManifestSha256: sha256(packBytes), providerPackDigest: pack.digest!, providerPackRunnerSourceRevision: sourceRevision, extractedAt: new Date().toISOString(), }); - const receipt = { ...receiptFor(finalDir), runnerdSha256: await fileSha256(runnerd) }; - await writeFile(path.join(staging, "receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); - await rename(staging, finalDir); - log(`Daytona image artifacts ready at ${finalDir}`); - return receipt; + const runnerdSha256 = await fileSha256(runnerd); + const publishTo = async (root: string) => { + const receipt = { ...receiptFor(root), runnerdSha256 }; + await writeFile(path.join(staging, "receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + await rename(staging, root); + log(`Daytona image artifacts ready at ${root}`); + return receipt; + }; + try { + return await publishTo(finalDir); + } catch (error) { + const concurrent = await readReceipt(finalDir); + if (concurrent) { + await rm(staging, { recursive: true, force: true }); + log(`Daytona image artifacts extracted concurrently at ${finalDir}`); + return concurrent; + } + if (!(await stat(finalDir).then(() => true, () => false))) throw error; + log(`Invalid cache entry at ${finalDir} left in place; publishing this extraction beside it`); + return await publishTo(`${finalDir}.${randomBytes(4).toString("hex")}`); + } } catch (error) { await rm(staging, { recursive: true, force: true }).catch(() => undefined); throw error; @@ -428,7 +462,7 @@ if (invoked && import.meta.url === pathToFileURL(invoked).href) { const image = process.argv[2]; if (!image) { console.error("Usage: daytona-image.ts "); process.exit(2); } const repositoryRoot = path.resolve(import.meta.dirname, "../.."); - const cacheRoot = process.env.RESTART_CHAOS_IMAGE_CACHE ?? path.join(import.meta.dirname, "results", "daytona-images"); + const cacheRoot = defaultDaytonaImageCacheRoot(); try { const artifacts = await prepareDaytonaImageArtifacts({ image, cacheRoot, log: line => console.error(line) }); const check = await assertControllerMatchesImage(artifacts, repositoryRoot).then(() => "match", (error: Error) => error.message); diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index 9d9d83581bf..2fdfec95e80 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -14,7 +14,7 @@ import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; import { ChaosApi, isSocketTypeOfServiceRace } from "./api.js"; import { CREDENTIAL_FOR_ADAPTER, parseCommand, replayCommand, waveIdFor } from "./config.js"; -import { assertControllerMatchesImage, prepareDaytonaImageArtifacts } from "./daytona-image.js"; +import { assertControllerMatchesImage, defaultDaytonaImageCacheRoot, prepareDaytonaImageArtifacts } from "./daytona-image.js"; import { PLUGIN_BUILD_STAMP, pluginBuildState } from "./plugin-build.js"; import { DaytonaWaveSweeper, leaksForTask, unattributedLeaks, type DaytonaSweepResult } from "./daytona-sweep.js"; import { knownSecretValues, redactKnownSecrets, writeEvidenceBundle } from "./evidence.js"; @@ -91,7 +91,7 @@ function ensureDaytonaPlugin(timeline: Timeline) { */ async function prepareDaytona(plan: WavePlan, timeline: Timeline): Promise> { const image = plan.options.daytonaImage!; - const artifacts = await prepareDaytonaImageArtifacts({ image, cacheRoot: path.join(resultsRoot, "daytona-images"), + const artifacts = await prepareDaytonaImageArtifacts({ image, cacheRoot: defaultDaytonaImageCacheRoot(), log: line => console.info(`[daytona-image] ${line}`) }); const { localContentId } = await assertControllerMatchesImage(artifacts, repositoryRoot); timeline.add("daytona_image_matched", { image, contentId: artifacts.contentId, localContentId, sourceRevision: artifacts.sourceRevision,