diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6048c277e41..150e6a9097d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -311,6 +311,29 @@ jobs: cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max labels: ${{ steps.meta.outputs.labels }} + # PID 1 must be an init that reaps adopted orphans. With node there, the + # orphans agent runs leave behind are never wait()ed and pin as zombies + # until the cgroup pid limit is exhausted and every fork() in the + # container fails. + # + # Probed here, on this leg's own architecture, against the exact digest + # just pushed. On master this runs while the exact-source gate is still + # pending, so the image pull (~95 s for amd64) is off the publication + # path, and arm64 is exercised too (the publisher runner is x86_64 only). + # Only a probed digest is exported, and merge-and-push signs an index + # only when its every Linux image is one of these probed digests. + - name: Verify PID 1 reaps orphaned processes + env: + # Through the environment, not interpolated into the script body. + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + IMAGE_NAME: ghcr.io/${{ github.repository }} + PLATFORM: ${{ matrix.platform }} + run: | + set -euo pipefail + test -n "$IMAGE_DIGEST" + echo "Verifying orphan reaping in $IMAGE_NAME@$IMAGE_DIGEST ($PLATFORM)" + docker run --rm -i --pull always --platform "$PLATFORM" "$IMAGE_NAME@$IMAGE_DIGEST" sh -s < scripts/assert-orphan-reaping.sh + # The digest is the only thing `merge` needs from this job. Carried as an # empty file named for it, which is the upstream pattern — the name is # the payload, so several legs can upload without colliding on content. @@ -368,8 +391,8 @@ jobs: id-token: write attestations: write steps: - # Checked out for `packages/db` (schema labels) and the orphan-reaping - # script the verification step pipes in. + # Checked out for `packages/db` (schema labels) and the source-proof and + # standard-image contract scripts. - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -444,37 +467,20 @@ jobs: $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *) - # PID 1 must be an init that reaps adopted orphans. With node there, the - # orphans agent runs leave behind are never wait()ed and pin as zombies - # until the cgroup pid limit is exhausted and every fork() in the - # container fails. Run against the pushed manifest rather than a local - # build: the legs push by digest, so nothing is loaded into this - # runner's daemon. - - name: Verify PID 1 reaps orphaned processes - env: - # Through the environment, not interpolated into the script body, so - # the tag text is data rather than shell. - IMAGE_TAGS: ${{ steps.meta.outputs.tags }} - run: | - set -euo pipefail - image="$(printf '%s\n' "$IMAGE_TAGS" | head -n 1)" - test -n "$image" - echo "Verifying orphan reaping in $image" - docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh + # The orphan-reaping probe ran in each build leg against the digest that + # leg pushed; /tmp/digests holds only probed digests. Pulling the image + # again here cost ~95 s after the source gate, on the publication path. # This generic contract is useful to any downstream image composer. Only # canonical master pushes own the full-SHA tag and its signed provenance. - # Capture exact registry bytes; both the smoke and attestation use their - # digest, so later tag movement cannot change what was verified. + # Capture exact registry bytes and sign that digest, so later tag movement + # cannot change what was verified. Registry content is addressed by + # digest: the resolve step fails unless every Linux image in the index is + # one the build legs probed. - name: Resolve standard image digest id: standard if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' - run: node scripts/standard-image-contract.mjs --resolve "$GITHUB_SHA" - - name: Verify the immutable standard image - if: steps.standard.outputs.digest != '' - env: - IMAGE_DIGEST: ${{ steps.standard.outputs.digest }} - run: docker run --rm -i --pull always "ghcr.io/paperclipai/paperclip@$IMAGE_DIGEST" sh -s < scripts/assert-orphan-reaping.sh + run: node scripts/standard-image-contract.mjs --resolve "$GITHUB_SHA" --verified-digests /tmp/digests - name: Attest the standard production image if: steps.standard.outputs.digest != '' uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 diff --git a/Dockerfile b/Dockerfile index e392be44845..dcbfeb9f6f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -164,6 +164,12 @@ RUN rm -rf packages/paperclip-runner/runner/target FROM build AS runner-provider-pack # Unstamped local builds remain usable, but cannot qualify a remote pack. # Never invent a source revision to make an unqualified pack look verified. +# Managed deployments can remap node's UID at startup, so the pack must stay +# readable by any UID. Grant that here, in the same layer that writes the +# files: COPY keeps these modes. A `chmod -R` after the COPY into production +# rewrote every changed file into a second copy of the pack (187 MB compressed +# and ~50 s of build time on amd64), and the production stage still checks +# readability as an unrelated UID below. RUN mkdir -p /provider-pack /provider-pack-transport \ && if [ -n "${PAPERCLIP_BUILD_COMMIT}" ]; then \ PAPERCLIP_RUNNER_SOURCE_REVISION="${PAPERCLIP_BUILD_COMMIT}" node packages/paperclip-runner/scripts/build-provider-pack.mjs /provider-pack \ @@ -171,7 +177,8 @@ RUN mkdir -p /provider-pack /provider-pack-transport \ && mv /provider-pack.archives /provider-pack.archive.json /provider-pack-transport/; \ else \ echo "Skipping remote provider pack: supply a full PAPERCLIP_BUILD_COMMIT to enable remote OpenCode/ACPX execution"; \ - fi + fi \ + && chmod -R a+rX /provider-pack FROM base AS production ARG USER_UID=1000 @@ -202,10 +209,10 @@ COPY --chown=0:0 --from=runner-provider-pack /provider-pack /opt/paperclip-runne COPY --from=runner-provider-pack /provider-pack-transport/ /opt/paperclip-runner/ # Managed deployments can remap node's UID at startup. This immutable pack # contains public code and integrity metadata, never credentials; it must remain -# readable afterward. +# readable afterward. The runner-provider-pack stage grants a+rX before this +# COPY, which preserves modes. # Keep it root-owned and verify access as an unrelated unprivileged UID. -RUN chmod -R a+rX /opt/paperclip-runner/provider-pack \ - && if [ -f /opt/paperclip-runner/provider-pack/provider-pack.json ]; then \ +RUN if [ -f /opt/paperclip-runner/provider-pack/provider-pack.json ]; then \ gosu 65534:65534 node -e 'const fs = require("node:fs"); const path = require("node:path"); const root = "/opt/paperclip-runner/provider-pack"; const manifest = JSON.parse(fs.readFileSync(path.join(root, "provider-pack.json"), "utf8")); for (const artifact of Object.values(manifest.payload.artifacts)) fs.readFileSync(path.join(root, artifact.path)); fs.accessSync(path.join(root, manifest.payload.artifacts.nodeCommand.path), fs.constants.X_OK);'; \ fi ENV PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH=/opt/paperclip-runner/provider-pack diff --git a/doc/standard-image-contract.md b/doc/standard-image-contract.md index 420acd4a4cc..f7441240eef 100644 --- a/doc/standard-image-contract.md +++ b/doc/standard-image-contract.md @@ -5,9 +5,11 @@ image at `ghcr.io/paperclipai/paperclip:sha-`. Existing short-SHA, version and channel tags continue to work. Other refs and manual dispatches do not write the canonical full-SHA tag. -The Docker workflow validates the index contains exactly one Linux amd64 and -one Linux arm64 image, verifies orphan reaping against its immutable digest, -and signs that digest using GitHub artifact attestations. The signer is +Each native build leg verifies orphan reaping by running the exact image digest +it pushed on its own architecture. The Docker workflow then validates the index +contains exactly one Linux amd64 and one Linux arm64 image, requires both to be +digests the build legs verified, and signs the index digest using GitHub +artifact attestations. The signer is `paperclipai/paperclip/.github/workflows/docker.yml@refs/heads/master`. Consumers must verify the signed source SHA, repository identity and ref, then compose from the verified digest, never from the mutable lookup tag. diff --git a/scripts/standard-image-contract.mjs b/scripts/standard-image-contract.mjs index 5e0ae3cc24c..c2e6f064da7 100644 --- a/scripts/standard-image-contract.mjs +++ b/scripts/standard-image-contract.mjs @@ -1,13 +1,16 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { appendFile, readFile } from "node:fs/promises"; +import { appendFile, readdir, readFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +const indexMediaTypes = ["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"]; +const sha256 = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + /** Validate the multi-platform index before it becomes a signed build contract. */ export function standardImageDigest(bytes) { const index = JSON.parse(bytes.toString("utf8")); assert.equal(index.schemaVersion, 2); - assert.ok(["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"].includes(index.mediaType)); + assert.ok(indexMediaTypes.includes(index.mediaType)); assert.ok(Array.isArray(index.manifests)); for (const architecture of ["amd64", "arm64"]) { const matches = index.manifests.filter(item => item.platform?.os === "linux" && item.platform?.architecture === architecture); @@ -15,13 +18,44 @@ export function standardImageDigest(bytes) { assert.match(matches[0].digest, /^sha256:[a-f0-9]{64}$/); assert.ok(Number.isSafeInteger(matches[0].size) && matches[0].size > 0); } - return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + return sha256(bytes); +} + +/** + * Platform image digests covered by the build legs' native PID 1 probe. Each + * leg probes the digest it pushed; that is a per-leg index (image plus + * provenance) or, without provenance, the image manifest itself. `legs` maps + * each probed digest to its registry bytes. + */ +export function verifiedImageDigests(legs) { + const verified = new Set(); + for (const [digest, bytes] of legs) { + assert.match(digest, /^sha256:[a-f0-9]{64}$/); + assert.equal(sha256(bytes), digest, "Verified image bytes do not match their digest"); + const manifest = JSON.parse(bytes.toString("utf8")); + if (!indexMediaTypes.includes(manifest.mediaType)) { verified.add(digest); continue; } + for (const item of manifest.manifests ?? []) { + if (item.platform?.os === "linux" && /^sha256:[a-f0-9]{64}$/.test(item.digest ?? "")) verified.add(item.digest); + } + } + return verified; +} + +/** Sign only an index whose every Linux image passed the build-leg probe. */ +export function assertIndexUsesVerifiedImages(indexBytes, verified) { + standardImageDigest(indexBytes); + const index = JSON.parse(indexBytes.toString("utf8")); + for (const item of index.manifests.filter(entry => entry.platform?.os === "linux")) { + assert.ok(verified.has(item.digest), + `linux/${item.platform.architecture} image ${item.digest} did not pass the build-leg orphan-reaping probe`); + } } /** Read registry bytes directly; CLI stdout formatting is not artifact content. */ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), missing = "error", + verifiedDigests, } = {}) { assert.match(sha ?? "", /^[a-f0-9]{40}$/); const request = async (url, headers, consume, allowNotFound = false) => { @@ -55,10 +89,10 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { }; const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json()); assert.ok(typeof token === "string" && token, "Missing public pull token"); - const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { - authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", + const readManifest = (reference, accept, allowNotFound) => request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/${reference}`, { + authorization: `Bearer ${token}`, accept, }, async response => { - assert.ok(response.body, "Standard image index has no body"); + assert.ok(response.body, "Registry manifest has no body"); const reader = response.body.getReader(); const chunks = []; let length = 0; @@ -67,21 +101,47 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { const next = await reader.read(); if (next.done) break; length += next.value.length; - if (length > 1024 * 1024) throw new RangeError("Standard index exceeds its size limit"); + if (length > 1024 * 1024) throw new RangeError("Registry manifest exceeds its size limit"); chunks.push(next.value); } } finally { await reader.cancel(); } return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") }; - }, true); + }, allowNotFound); + const manifest = await readManifest(`sha-${sha}`, indexMediaTypes.join(", "), true); if (manifest === null) return null; const { raw, expected } = manifest; const digest = standardImageDigest(raw); assert.equal(digest, expected, "Registry digest does not match the index bytes"); + if (verifiedDigests) { + // The build legs ran the PID 1 probe on these digests. Registry content is + // addressed by digest, so membership proves the signed index names exactly + // the probed images without pulling them again here. + assert.ok(verifiedDigests.length > 0, "No build leg reported a probed image"); + const legs = []; + for (const legDigest of verifiedDigests) { + assert.match(legDigest, /^sha256:[a-f0-9]{64}$/); + const leg = await readManifest(legDigest, [...indexMediaTypes, "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json"].join(", "), false); + legs.push([legDigest, leg.raw]); + } + assertIndexUsesVerifiedImages(raw, verifiedImageDigests(legs)); + } return digest; } +/** Digests the build legs probed, carried as empty files named for each digest. */ +async function readVerifiedDigests(directory) { + const names = await readdir(directory); + return names.map(name => { + assert.match(name, /^[a-f0-9]{64}$/, `Unexpected verified digest entry: ${name}`); + return `sha256:${name}`; + }); +} + if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - const digest = process.argv[2] === "--resolve" ? await resolveStandardImageDigest(process.argv[3]) : standardImageDigest(await readFile(process.argv[2])); + const verifiedFlag = process.argv.indexOf("--verified-digests"); + const verifiedDigests = verifiedFlag > 0 ? await readVerifiedDigests(process.argv[verifiedFlag + 1]) : undefined; + const digest = process.argv[2] === "--resolve" ? await resolveStandardImageDigest(process.argv[3], fetch, { verifiedDigests }) : standardImageDigest(await readFile(process.argv[2])); if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `digest=${digest}\n`); console.log(digest); } diff --git a/scripts/standard-image-contract.test.mjs b/scripts/standard-image-contract.test.mjs index d61f3fb862d..67be12169cd 100644 --- a/scripts/standard-image-contract.test.mjs +++ b/scripts/standard-image-contract.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import test from "node:test"; -import { standardImageDigest, resolveStandardImageDigest } from "./standard-image-contract.mjs"; +import { standardImageDigest, resolveStandardImageDigest, verifiedImageDigests, assertIndexUsesVerifiedImages } from "./standard-image-contract.mjs"; import { isNode24ImageTag } from "./node-image-policy.mjs"; test("Node image policy accepts Node 24 major tags and exact security pins", () => { @@ -91,3 +91,50 @@ test("optional lookup falls back only for a missing manifest", async () => { assert.equal(calls, status === 503 ? 5 : 1); } }); + +const digestOf = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +const image = name => Buffer.from(JSON.stringify({ schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", name })); +// The build legs push by digest with provenance, so each leg digest is an index +// of one platform image plus its attestation manifest. The merged standard +// index names those same child digests. +function legFixture() { + const amd64 = digestOf(image("amd64")), arm64 = digestOf(image("arm64")); + const attestation = arch => ({ platform: { os: "unknown", architecture: "unknown" }, digest: digestOf(image(`att-${arch}`)), size: 50, + annotations: { "vnd.docker.reference.type": "attestation-manifest" } }); + const leg = (arch, digest) => index([{ platform: { os: "linux", architecture: arch }, digest, size: 100 }, attestation(arch)]); + const legs = new Map([[digestOf(leg("amd64", amd64)), leg("amd64", amd64)], [digestOf(leg("arm64", arm64)), leg("arm64", arm64)]]); + const standard = index([{ platform: { os: "linux", architecture: "arm64" }, digest: arm64, size: 100 }, attestation("arm64"), + { platform: { os: "linux", architecture: "amd64" }, digest: amd64, size: 100 }, attestation("amd64")]); + return { amd64, legs, standard }; +} + +test("signing requires every Linux image in the index to have passed a build-leg probe", () => { + const { amd64, legs, standard } = legFixture(); + assert.doesNotThrow(() => assertIndexUsesVerifiedImages(standard, verifiedImageDigests(legs))); + const onlyArm = new Map([...legs].filter(([, bytes]) => !bytes.toString().includes(amd64))); + assert.throws(() => assertIndexUsesVerifiedImages(standard, verifiedImageDigests(onlyArm)), /linux\/amd64 image .* orphan-reaping probe/); + // A leg that pushed a bare image manifest verifies exactly that digest. + const bare = image("amd64"); + assert.deepEqual([...verifiedImageDigests(new Map([[digestOf(bare), bare]]))], [digestOf(bare)]); + // Bytes must hash to the digest the leg reported. + assert.throws(() => verifiedImageDigests(new Map([[digestOf(bare), image("other")]])), /do not match their digest/); +}); + +test("lookup with verified digests reads each leg by digest and rejects an unprobed index", async () => { + const { legs, standard } = legFixture(); + const expected = standardImageDigest(standard); + const requested = []; + const fetchImpl = async (url, options) => { + if (url.includes("/token?")) return Response.json({ token: "public-fixture" }); + const reference = url.split("/manifests/")[1]; + requested.push(reference); + if (reference.startsWith("sha-")) return new Response(standard, { headers: { "docker-content-digest": expected } }); + assert.match(options.headers.accept, /image\.manifest\.v1\+json/); + return new Response(legs.get(reference), { headers: { "docker-content-digest": reference } }); + }; + const legDigests = [...legs.keys()]; + assert.equal(await resolveStandardImageDigest("a".repeat(40), fetchImpl, { verifiedDigests: legDigests }), expected); + assert.deepEqual(requested, [`sha-${"a".repeat(40)}`, ...legDigests]); + await assert.rejects(resolveStandardImageDigest("a".repeat(40), fetchImpl, { verifiedDigests: legDigests.slice(0, 1) }), /orphan-reaping probe/); + await assert.rejects(resolveStandardImageDigest("a".repeat(40), fetchImpl, { verifiedDigests: [] }), /No build leg/); +}); diff --git a/server/src/__tests__/container-init-reaping.test.ts b/server/src/__tests__/container-init-reaping.test.ts index 364b1755f8a..7c2adc07791 100644 --- a/server/src/__tests__/container-init-reaping.test.ts +++ b/server/src/__tests__/container-init-reaping.test.ts @@ -135,6 +135,21 @@ describe("orphan-reaping probe", () => { expect(source).toContain("scripts/assert-orphan-reaping.sh"); }); + it("probes each pushed platform digest before it can be published or signed", () => { + // The probe runs in every native build leg, before that leg exports the + // digest merge-and-push assembles. Publication then signs only an index + // whose Linux images are those probed digests, instead of pulling the + // image again after the exact-source gate. + const leg = dockerWorkflow.split("\n build-and-push:\n")[1].split("\n publication_verification:\n")[0]; + const probe = leg.indexOf("- name: Verify PID 1 reaps orphaned processes"); + expect(probe).toBeGreaterThan(leg.indexOf("- name: Build and push by digest")); + expect(probe).toBeLessThan(leg.indexOf("- name: Export digest")); + expect(leg).toContain('--platform "$PLATFORM" "$IMAGE_NAME@$IMAGE_DIGEST" sh -s < scripts/assert-orphan-reaping.sh'); + const publisher = dockerWorkflow.split("\n merge-and-push:\n")[1].split("\n promote_canary_channel:\n")[0]; + expect(publisher).toContain('standard-image-contract.mjs --resolve "$GITHUB_SHA" --verified-digests /tmp/digests'); + expect(publisher).not.toContain("docker run"); + }); + it("fails closed when the orphan is never adopted by PID 1", () => { // A probe whose grandchild is not reparented onto PID 1 proves nothing, so // it must error rather than report a pass it did not observe. diff --git a/server/src/__tests__/docker-build-stamp.test.ts b/server/src/__tests__/docker-build-stamp.test.ts index 3e52e6e4652..3abff18739b 100644 --- a/server/src/__tests__/docker-build-stamp.test.ts +++ b/server/src/__tests__/docker-build-stamp.test.ts @@ -118,7 +118,10 @@ describe("Standard image remote provider pack", () => { expect(pack).toContain("Skipping remote provider pack"); expect(production).toContain("--from=runner-provider-pack /provider-pack /opt/paperclip-runner/provider-pack"); expect(production).not.toContain("--chown=node:node --from=runner-provider-pack"); - expect(production).toContain("chmod -R a+rX /opt/paperclip-runner/provider-pack"); + // Modes are granted in the layer that writes the pack. A recursive chmod + // after the COPY duplicates every changed file into another image layer. + expect(pack).toMatch(/build-provider-pack\.mjs[\s\S]*&& chmod -R a\+rX \/provider-pack$/m); + expect(production).not.toMatch(/chmod -R [^\n]*\/opt\/paperclip-runner\/provider-pack/); expect(production).toContain("gosu 65534:65534 node"); expect(production).toContain("Object.values(manifest.payload.artifacts)"); expect(production).toContain("PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH=/opt/paperclip-runner/provider-pack");