import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createServer } from "node:http"; import { promises as fs } from "node:fs"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { pathToFileURL } from "node:url"; import { gzipSync, zstdCompressSync } from "node:zlib"; import tar from "tar-stream"; import { afterEach, expect, it, vi } from "vitest"; import { canonicalProviderPackJson } from "./remote-provider-pack-verification.js"; import { uploadVerifiedProviderPack } from "./provider-pack-upload.js"; import * as archiveOwner from "./provider-pack-archive.js"; import { resolveProviderPackUploadSource } from "./retained-provider-pack-source.js"; const archiveScript = pathToFileURL(path.resolve(import.meta.dirname, "../../../../scripts/create-provider-pack-archive.mjs")).href; const { createProviderPackLayers } = await import(archiveScript); const exec = promisify(execFile); const hash = (value: string | Buffer) => createHash("sha256").update(value).digest("hex"); const roots: string[] = []; const servers: ReturnType[] = []; const actualFetch = globalThis.fetch; afterEach(async () => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); vi.restoreAllMocks(); await Promise.all(servers.splice(0).map(server => new Promise((done, reject) => { server.close(error => error ? reject(error) : done()); server.closeAllConnections(); }))); await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); }); async function fixture(options: { wrongCode?: boolean; unsafeOverlay?: boolean; changedFileLayer?: boolean; hardlink?: "safe" | "escape" | "unknown" } = {}) { const root = await fs.mkdtemp(path.join(os.tmpdir(), "retained-pack-test-")); roots.push(root); const temporary = path.join(root, "tmp"); await fs.mkdir(temporary); vi.stubEnv("TMPDIR", temporary); const makePack = async (name: string, tool: string, source: string) => { const pack = path.join(root, name); await fs.mkdir(path.join(pack, "bin"), { recursive: true }); const artifact = { path: "bin/tool", sha256: `sha256:${hash(tool)}` }; const payload = { pins: { opencode: "1.18.34" }, target: { platform: "linux", architecture: "x64" }, runnerSourceRevision: source, contractDigest: `sha256:${hash(tool)}`, artifacts: { nodeCommand: artifact, opencodeCommand: artifact, opencodeProxy: artifact } }; const manifest = { schema: "paperclip-runner/remote-provider-pack/v1", payload, digest: `sha256:${hash(canonicalProviderPackJson(payload))}` }; await fs.writeFile(path.join(pack, "provider-pack.json"), JSON.stringify(manifest)); await fs.writeFile(path.join(pack, "bin/tool"), tool, { mode: 0o755 }); await fs.writeFile(path.join(pack, "bin/large"), Buffer.alloc(1024 * 1024, 42), { mode: 0o755 }); await createProviderPackLayers(pack); return { pack, manifest }; }; const selected = await makePack("old", "retained provider code", "a".repeat(40)); const current = await makePack("current", "current incompatible code", "b".repeat(40)); if (options.changedFileLayer) { const unsafe = tar.pack(), chunks: Buffer[] = []; const collected = (async () => { for await (const chunk of unsafe) chunks.push(Buffer.from(chunk)); })(); const body = Buffer.alloc(1024 * 1024, 43); await new Promise((done, reject) => unsafe.entry({ name: "bin/large", type: "file", size: body.length, mode: 0o755 }, body, error => error ? reject(error) : done())); unsafe.finalize(); await collected; const bytes = zstdCompressSync(Buffer.concat(chunks)); await fs.writeFile(selected.pack + ".archives/0.tar.zst", bytes); const receipt = JSON.parse(await fs.readFile(selected.pack + ".archive.json", "utf8")); receipt.files[0].archive = { bytes: bytes.length, sha256: hash(bytes) }; await fs.writeFile(selected.pack + ".archive.json", JSON.stringify(receipt)); } if (options.unsafeOverlay || options.hardlink) { const unsafe = tar.pack(), chunks: Buffer[] = []; const collected = (async () => { for await (const chunk of unsafe) chunks.push(Buffer.from(chunk)); })(); for (const [name, body] of [["provider-pack.json", JSON.stringify(selected.manifest)], ["bin/tool", "retained provider code"], ...(options.unsafeOverlay ? [["../outside", "must not be written"]] : [])]) { await new Promise((done, reject) => unsafe.entry({ name: name!, type: "file", size: Buffer.byteLength(body!) }, body!, error => error ? reject(error) : done())); } if (options.hardlink) await new Promise((done, reject) => unsafe.entry({ name: "bin/tool-copy", type: "link", linkname: options.hardlink === "safe" ? "./bin/tool" : options.hardlink === "escape" ? "../outside" : "bin/missing", mode: 0o644 }, error => error ? reject(error) : done())); unsafe.finalize(); await collected; const bytes = zstdCompressSync(Buffer.concat(chunks)); await fs.writeFile(selected.pack + ".archives/overlay.tar.zst", bytes); const receipt = JSON.parse(await fs.readFile(selected.pack + ".archive.json", "utf8")); receipt.overlay = { bytes: bytes.length, sha256: hash(bytes) }; await fs.writeFile(selected.pack + ".archive.json", JSON.stringify(receipt)); } const transportPack = options.wrongCode ? current.pack : selected.pack; const archive = tar.pack(); const chunks: Buffer[] = []; const collected = (async () => { for await (const chunk of archive) chunks.push(Buffer.from(chunk)); })(); for (const name of ["provider-pack.archive.json", ...await fs.readdir(transportPack + ".archives").then(names => names.map(name => `provider-pack.archives/${name}`))]) { const bytes = await fs.readFile(path.join(path.dirname(transportPack), name.replace("provider-pack", path.basename(transportPack)))); await new Promise((done, reject) => archive.entry({ name: `opt/paperclip-runner/${name}`, type: "file", size: bytes.length }, bytes, error => error ? reject(error) : done())); } archive.finalize(); await collected; const layer = gzipSync(Buffer.concat(chunks)); const config = Buffer.from(JSON.stringify({ architecture: "amd64", os: "linux", config: { Labels: { "org.opencontainers.image.revision": selected.manifest.payload.runnerSourceRevision, "org.opencontainers.image.source": "https://github.com/paperclipai/paperclip", } }, history: [{ created_by: "COPY /provider-pack-transport/ /opt/paperclip-runner/ # buildkit" }] })); const manifest = Buffer.from(JSON.stringify({ schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", config: { digest: `sha256:${hash(config)}`, size: config.length }, layers: [{ digest: `sha256:${hash(layer)}`, size: layer.length, mediaType: "application/vnd.oci.image.layer.v1.tar+gzip" }] })); const index = Buffer.from(JSON.stringify({ schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", manifests: [{ digest: `sha256:${hash(manifest)}`, size: manifest.length, platform: { os: "linux", architecture: "amd64" } }] })); const routes = new Map([ ["/token", Buffer.from(JSON.stringify({ token: "public-code-fixture-token" }))], [`/v2/paperclipai/paperclip/manifests/sha-${selected.manifest.payload.runnerSourceRevision}`, index], [`/v2/paperclipai/paperclip/manifests/sha256:${hash(manifest)}`, manifest], [`/v2/paperclipai/paperclip/blobs/sha256:${hash(config)}`, config], [`/v2/paperclipai/paperclip/blobs/sha256:${hash(layer)}`, layer], ]); const requests: string[] = []; const stalled = new Set(); const observers = new Map void>(); const server = createServer((request, response) => { const route = new URL(request.url!, "http://fixture").pathname; requests.push(route); const bytes = routes.get(route); response.writeHead(bytes ? 200 : 404, { "content-length": bytes?.length ?? 0 }); response.flushHeaders(); observers.get(route)?.(); if (stalled.has(route)) return; response.end(bytes); }); servers.push(server); await new Promise(done => server.listen(0, "127.0.0.1", done)); const address = server.address(); if (!address || typeof address === "string") throw Error("fixture server unavailable"); vi.stubGlobal("fetch", (input: string | URL | Request, init?: RequestInit) => { const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url); expect(url.hostname).toBe("ghcr.io"); return actualFetch(`http://127.0.0.1:${address.port}${url.pathname}${url.search}`, init); }); const bin = path.join(root, "commands"); await fs.mkdir(bin); const tarCommand = (await exec("sh", ["-c", "command -v gtar || command -v tar"])).stdout.trim(); await fs.symlink(tarCommand, path.join(bin, "tar")); const execute = vi.fn(async (input: { command: string; args?: string[] }) => { try { const result = await exec(input.command, input.args ?? [], { env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } }); return { exitCode: 0, timedOut: false, stdout: result.stdout, stderr: result.stderr }; } catch (error) { return { exitCode: 1, timedOut: false, stdout: "", stderr: String(error) }; } }); const syncIn = vi.fn(async (operations: Array<{ files: Array<{ sourcePath: string; targetPath: string }> }>) => { for (const operation of operations) for (const file of operation.files) { await fs.mkdir(path.dirname(file.targetPath), { recursive: true }); await fs.copyFile(file.sourcePath, file.targetPath); } return { operations: [] }; }); return { root, selected, current, requests, routes, runner: { execute, syncIn }, targetPath: path.join(root, "target"), temporary, layerRoute: `/v2/paperclipai/paperclip/blobs/sha256:${hash(layer)}`, stall: (route: string) => { stalled.add(route); }, observed: (route: string) => new Promise(done => { if (requests.includes(route)) done(); else observers.set(route, done); }), }; } function upload(input: Awaited>, extra: Partial[0]> = {}) { return uploadVerifiedProviderPack({ root: input.current.pack, sourceDigest: input.current.manifest.digest, expectedDigest: input.selected.manifest.digest, manifest: input.selected.manifest, runner: input.runner, cwd: input.root, targetPath: input.targetPath, reuseRoots: [], verifyDirectory: () => { throw Error("current_directory_fallback_forbidden"); }, stageDirectory: async () => { throw Error("current_directory_fallback_forbidden"); }, verifyUploaded: async () => {}, ...extra } as Parameters[0]); } it("installs the exact retained pack through the real uploader on an empty host instead of the current image archive", async () => { const input = await fixture(); const verifyUploaded = vi.fn(async () => { expect(await fs.readFile(path.join(input.targetPath, "bin/tool"), "utf8")).toBe("retained provider code"); expect(JSON.parse(await fs.readFile(path.join(input.targetPath, "provider-pack.json"), "utf8"))).toEqual(input.selected.manifest); }); await upload(input, { verifyUploaded }); expect(verifyUploaded).toHaveBeenCalledOnce(); expect(input.requests.some(route => route.includes("/blobs/"))).toBe(true); }); it("keeps a matching current pack on its existing path without registry or cache reads", async () => { const input = await fixture(); await upload(input, { root: input.selected.pack, sourceDigest: input.selected.manifest.digest }); expect(input.requests).toEqual([]); await expect(fs.stat(path.join(input.temporary, `paperclip-provider-pack-sources-${process.getuid?.()}`))).rejects.toMatchObject({ code: "ENOENT" }); expect(await fs.readFile(path.join(input.targetPath, "bin/tool"), "utf8")).toBe("retained provider code"); }); it("refuses a different historical code pack even when OCI source labels and all transport hashes are valid", async () => { const input = await fixture({ wrongCode: true }); await expect(upload(input)).rejects.toThrow("runner_remote_provider_archive_invalid"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); expect(await fs.readFile(path.join(input.current.pack, "bin/tool"), "utf8")).toBe("current incompatible code"); }); it("does not mask a corrupt matching current archive by looking up a different code source", async () => { const input = await fixture(); await fs.writeFile(input.selected.pack + ".archive.json", "{"); await expect(upload(input, { root: input.selected.pack, sourceDigest: input.selected.manifest.digest })).rejects.toThrow("runner_remote_provider_archive_invalid"); expect(input.requests).toEqual([]); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("rejects a changed controller code identity before looking up a historical source", async () => { const input = await fixture(); await expect(upload(input, { sourceDigest: input.selected.manifest.digest })).rejects.toThrow("runner_remote_provider_source_invalid"); expect(input.requests).toEqual([]); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("refuses an unknown historical source without clearing or falling back to the current image", async () => { const input = await fixture(); input.routes.delete(`/v2/paperclipai/paperclip/manifests/sha-${input.selected.manifest.payload.runnerSourceRevision}`); await expect(upload(input)).rejects.toThrow("runner_remote_provider_source_unavailable: HTTP 404"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("rejects escaping inner tar paths despite valid OCI and layer digests before any remote effects", async () => { const input = await fixture({ unsafeOverlay: true }); await expect(upload(input)).rejects.toThrow("runner_remote_provider_source_invalid"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); await expect(fs.stat(path.join(input.temporary, "outside"))).rejects.toMatchObject({ code: "ENOENT" }); }); it("checks the file layer's actual plaintext against its declared code digest before any remote effects", async () => { const input = await fixture({ changedFileLayer: true }); await expect(upload(input)).rejects.toThrow("runner_remote_provider_source_invalid"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("preserves historical code hardlinks only to earlier confined regular files", async () => { const input = await fixture({ hardlink: "safe" }); await upload(input); const original = await fs.stat(path.join(input.targetPath, "bin/tool")), copy = await fs.stat(path.join(input.targetPath, "bin/tool-copy")); expect(copy.ino).toBe(original.ino); expect(await fs.readFile(path.join(input.targetPath, "bin/tool-copy"), "utf8")).toBe("retained provider code"); }); it.each(["escape", "unknown"] as const)("refuses a historical code hardlink with an %s target before remote effects", async hardlink => { const input = await fixture({ hardlink }); await expect(upload(input)).rejects.toThrow("runner_remote_provider_source_invalid"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("cancels a stalled source body and removes only the unfinished private import", async () => { const input = await fixture(); input.stall(input.layerRoute); const controller = new AbortController(); const operation = upload(input, { signal: controller.signal }); const rejected = expect(operation).rejects.toThrow("owned caller stopped"); await input.observed(input.layerRoute); controller.abort(new Error("owned caller stopped")); await rejected; expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); const cache = path.join(input.temporary, `paperclip-provider-pack-sources-${process.getuid?.()}`); await vi.waitFor(async () => expect(await fs.readdir(cache)).toEqual([])); }); it("enforces the source deadline while the registry is stalled", async () => { const input = await fixture(); input.stall(input.layerRoute); const nativeTimeout = AbortSignal.timeout; vi.spyOn(AbortSignal, "timeout").mockImplementation(milliseconds => { expect(milliseconds).toBe(180_000); return nativeTimeout(80); }); await expect(upload(input)).rejects.toMatchObject({ name: "TimeoutError" }); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); }); it("reuses a verified derived cache without registry reads and reconstructs it after an empty-host reset", async () => { const input = await fixture(); await upload(input); const first = input.requests.length; await upload(input, { targetPath: input.targetPath + "-cached" }); expect(input.requests).toHaveLength(first); await fs.rm(path.join(input.temporary, `paperclip-provider-pack-sources-${process.getuid?.()}`), { recursive: true }); await upload(input, { targetPath: input.targetPath + "-empty" }); expect(input.requests.length).toBeGreaterThan(first); expect(await fs.readFile(path.join(input.targetPath + "-empty", "bin/tool"), "utf8")).toBe("retained provider code"); }); it("shares warm-cache verification while a stopped caller leaves another waiter active", async () => { const input = await fixture(); await upload(input); const requests = input.requests.length; const firstController = new AbortController(), secondController = new AbortController(); let enter!: () => void, release!: () => void, joined!: () => void, duplicate!: () => void; const entered = new Promise(done => { enter = done; }); const gate = new Promise(done => { release = done; }); const secondJoined = new Promise(done => { joined = done; }); const repeated = new Promise(done => { duplicate = done; }); const originalRead = archiveOwner.readProviderPackArchive; const verification = vi.spyOn(archiveOwner, "readProviderPackArchive").mockImplementation(async (...args) => { enter(); if (verification.mock.calls.length > 1) duplicate(); await gate; return originalRead(...args); }); const originalAny = AbortSignal.any; vi.spyOn(AbortSignal, "any").mockImplementation(signals => { const combined = originalAny(signals); if (signals.includes(secondController.signal)) { const add = combined.addEventListener.bind(combined); vi.spyOn(combined, "addEventListener").mockImplementation((type, listener, options) => { const result = add(type, listener, options); if (type === "abort") joined(); return result; }); } return combined; }); const source = { root: input.current.pack, sourceDigest: input.current.manifest.digest, manifest: input.selected.manifest }; const first = resolveProviderPackUploadSource({ ...source, signal: firstController.signal }); const firstStopped = expect(first).rejects.toThrow("owned shared-cache waiter stopped"); await entered; const second = resolveProviderPackUploadSource({ ...source, signal: secondController.signal }); try { await Promise.race([secondJoined, repeated.then(() => { throw Error("warm-cache verification repeated"); })]); expect(verification).toHaveBeenCalledOnce(); firstController.abort(new Error("owned shared-cache waiter stopped")); await firstStopped; expect(verification.mock.calls[0]?.[2]?.aborted).toBe(false); release(); expect(await second).toMatchObject({ archiveOnly: true }); expect(input.requests).toHaveLength(requests); } finally { firstController.abort(new Error("owned shared-cache waiter stopped")); secondController.abort(new Error("owned shared-cache fixture cleanup")); release(); await Promise.allSettled([first, second, firstStopped]); } }); it("rejects a changed cached archive before any remote command or transfer", async () => { const input = await fixture(); await upload(input); const cache = path.join(input.temporary, `paperclip-provider-pack-sources-${process.getuid?.()}`); const publication = JSON.parse(await fs.readFile(path.join(cache, input.selected.manifest.digest.slice(7) + ".json"), "utf8")); await fs.appendFile(path.join(cache, publication.entry, "provider-pack.archives/overlay.tar.zst"), "changed"); input.runner.execute.mockClear(); input.runner.syncIn.mockClear(); await expect(upload(input, { targetPath: input.targetPath + "-new" })).rejects.toThrow("runner_remote_provider_archive_invalid"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); expect(await fs.readFile(path.join(input.targetPath, "bin/tool"), "utf8")).toBe("retained provider code"); }); it("refuses an unsafe cache directory instead of following it", async () => { const input = await fixture(); const outside = path.join(input.root, "outside"); await fs.mkdir(outside); await fs.symlink(outside, path.join(input.temporary, `paperclip-provider-pack-sources-${process.getuid?.()}`)); await expect(upload(input)).rejects.toThrow("runner_remote_provider_source_invalid"); expect(await fs.readdir(outside)).toEqual([]); expect(input.requests).toEqual([]); expect(input.runner.execute).not.toHaveBeenCalled(); }); it("leaves a target intact when its archive transport cannot install the verified source", async () => { const input = await fixture(); await fs.mkdir(input.targetPath); await fs.writeFile(path.join(input.targetPath, "marker"), "untouched"); input.runner.execute.mockResolvedValue({ exitCode: 127, timedOut: false, stdout: "", stderr: "unavailable" }); await expect(upload(input)).rejects.toThrow("runner_remote_provider_archive_transport_unavailable"); expect(input.runner.execute).toHaveBeenCalledOnce(); expect(input.runner.execute.mock.calls[0]?.[0].args?.join(" ")).not.toContain("rm -rf"); expect(input.runner.syncIn).not.toHaveBeenCalled(); expect(await fs.readFile(path.join(input.targetPath, "marker"), "utf8")).toBe("untouched"); }); it("refuses an oversized historical layer specification before any target effects", async () => { const input = await fixture(); await fs.mkdir(input.targetPath); await fs.writeFile(path.join(input.targetPath, "marker"), "untouched"); const reuseRoots = Array.from({ length: 2000 }, (_, index) => `/qualified-image-${index}/` + "a".repeat(64)); await expect(upload(input, { reuseRoots })).rejects.toThrow("runner_remote_provider_archive_transport_unavailable"); expect(input.runner.execute).not.toHaveBeenCalled(); expect(input.runner.syncIn).not.toHaveBeenCalled(); expect(await fs.readFile(path.join(input.targetPath, "marker"), "utf8")).toBe("untouched"); });