diff --git a/Dockerfile b/Dockerfile index dcbfeb9f6f8..b7e150b4d1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -179,6 +179,16 @@ RUN mkdir -p /provider-pack /provider-pack-transport \ echo "Skipping remote provider pack: supply a full PAPERCLIP_BUILD_COMMIT to enable remote OpenCode/ACPX execution"; \ fi \ && chmod -R a+rX /provider-pack +# The pack is ~2.5 GB of provider binaries that change only when a provider +# version does, next to small files that change on every commit (the manifest +# carries the source revision). Move the large files to their own tree, after +# the transport archive has read the whole pack, so production can copy them +# as a layer that stays identical between commits. Normalized mtimes keep that +# layer's bytes identical when this stage reruns. +RUN mkdir -p /provider-pack-large \ + && cd /provider-pack \ + && find . -type f -size +1048575c -exec sh -c 'mkdir -p "/provider-pack-large/$(dirname "$1")" && mv "$1" "/provider-pack-large/$1"' _ {} \; \ + && find /provider-pack-large -exec touch -h -d @0 {} + FROM base AS production ARG USER_UID=1000 @@ -203,8 +213,16 @@ RUN echo "cli-tools-epoch: ${CLI_TOOLS_CACHE_EPOCH}" \ COPY scripts/docker-entrypoint.sh /usr/local/bin/ RUN chmod +x /usr/local/bin/docker-entrypoint.sh -COPY --chown=node:node --from=build /app /app +# Installed dependencies are ~2.9 GB of /app and change only with the lockfile. +# Copied from the deps stage as their own layer, a commit that leaves the +# lockfile alone does not push or pull them again. The build stage writes +# nothing under the root node_modules, so the rest of /app is copied without it. +COPY --chown=node:node --from=deps /app/node_modules /app/node_modules +COPY --chown=node:node --exclude=node_modules --from=build /app /app +# Large provider binaries first, linked so the layer does not depend on the +# per-commit /app layer above; then the small per-commit remainder of the pack. +COPY --link --chown=0:0 --from=runner-provider-pack /provider-pack-large/ /opt/paperclip-runner/provider-pack/ COPY --chown=0:0 --from=runner-provider-pack /provider-pack /opt/paperclip-runner/provider-pack COPY --from=runner-provider-pack /provider-pack-transport/ /opt/paperclip-runner/ # Managed deployments can remap node's UID at startup. This immutable pack diff --git a/server/src/__tests__/docker-image-layer-reuse.test.ts b/server/src/__tests__/docker-image-layer-reuse.test.ts new file mode 100644 index 00000000000..898919295b2 --- /dev/null +++ b/server/src/__tests__/docker-image-layer-reuse.test.ts @@ -0,0 +1,56 @@ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; + +/** + * Drift guard for the layers of the production image that must stay identical + * between commits. + * + * Installed dependencies (~2.9 GB) and the large provider-pack binaries + * (~2.5 GB) change only when the lockfile or a provider version changes. When + * they shared a layer with files that change on every commit, each master + * build pushed about 3.5 GB of unchanged bytes and each deploy pulled them. + */ + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +const dockerfile = readFileSync(path.join(repoRoot, "Dockerfile"), "utf8"); + +function stageBody(stageName: string): string { + const froms = [...dockerfile.matchAll(/^FROM .*$/gm)]; + const startIdx = froms.findIndex((m) => new RegExp(`\\bAS ${stageName}\\s*$`).test(m[0])); + expect(startIdx, `Dockerfile must declare a '${stageName}' stage`).toBeGreaterThanOrEqual(0); + return dockerfile.slice(froms[startIdx].index ?? 0, froms[startIdx + 1]?.index ?? dockerfile.length); +} + +it("copies installed dependencies as their own layer, before the per-commit app files", () => { + const production = stageBody("production"); + const dependencies = production.search(/^COPY --chown=node:node --from=deps \/app\/node_modules \/app\/node_modules$/m); + const app = production.search(/^COPY --chown=node:node --exclude=node_modules --from=build \/app \/app$/m); + expect(dependencies, "dependencies must come from the deps stage").toBeGreaterThanOrEqual(0); + expect(app, "the app copy must leave out the root node_modules").toBeGreaterThan(dependencies); + // A copy of all of /app from the build stage would put the dependencies + // back into the per-commit layer. + expect(production).not.toMatch(/^COPY (?!.*--exclude=node_modules).*--from=build \/app \/app$/m); +}); + +it("copies the large provider-pack files as a linked layer with fixed timestamps", () => { + const pack = stageBody("runner-provider-pack"); + const archive = pack.indexOf("create-provider-pack-archive.mjs"); + const split = pack.indexOf("mv \"$1\" \"/provider-pack-large/$1\""); + const timestamps = pack.indexOf("find /provider-pack-large -exec touch -h -d @0 {} +"); + // The transport archive reads the whole pack, so the split must follow it. + expect(archive).toBeGreaterThanOrEqual(0); + expect(split).toBeGreaterThan(archive); + expect(timestamps).toBeGreaterThan(split); + + const production = stageBody("production"); + const large = production.search( + /^COPY --link --chown=0:0 --from=runner-provider-pack \/provider-pack-large\/ \/opt\/paperclip-runner\/provider-pack\/$/m, + ); + const rest = production.search( + /^COPY --chown=0:0 --from=runner-provider-pack \/provider-pack \/opt\/paperclip-runner\/provider-pack$/m, + ); + expect(large, "large files need --link so the layer does not depend on the app layer").toBeGreaterThanOrEqual(0); + expect(rest).toBeGreaterThan(large); +});