diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 68230fcb677..d88794e7232 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -137,6 +137,13 @@ discarded. The native daemon build cache holds dependency crates only. Its key covers the Cargo manifest, Cargo configuration, host and target, but not the Runner's own source, so a Runner change does not discard the dependencies. +During a release (`PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR` set), the Runner +package build does not compile `paperclip-runnerd`. It verifies the +exact-source release daemon for the publisher's own target (source, digest, +architecture and build metadata) and stages it at `dist/bin`, so the build +stamp and the server's vendored Runner stay current. Every other build still +runs `build:binary`. + After publication, the onboarding smoke checks out the same source SHA and installs its test dependencies from the publisher's resolved lockfile with `--frozen-lockfile`. The same-run artifact is captured before tracked files are diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index c89d6b11040..b57658f5b54 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -67,7 +67,7 @@ }, "sideEffects": false, "scripts": { - "build": "node scripts/record-build-inputs.mjs --begin && pnpm run check:protocol-manifest && pnpm run build:typescript && pnpm run check:capability-contract && pnpm run check:capability-inventory && pnpm run check:protocol-coverage && node scripts/generate-semantic-contracts.mjs --check && node scripts/check-runner-workflow-traceability.mjs && pnpm run build:binary && node scripts/generate-replay-goldens.mjs --check && node scripts/generate-semantic-action-catalog.mjs --check && node scripts/record-build-inputs.mjs --finish", + "build": "node scripts/record-build-inputs.mjs --begin && pnpm run check:protocol-manifest && pnpm run build:typescript && pnpm run check:capability-contract && pnpm run check:capability-inventory && pnpm run check:protocol-coverage && node scripts/generate-semantic-contracts.mjs --check && node scripts/check-runner-workflow-traceability.mjs && node scripts/build-runner-binary.mjs && node scripts/generate-replay-goldens.mjs --check && node scripts/generate-semantic-action-catalog.mjs --check && node scripts/record-build-inputs.mjs --finish", "build:typescript": "pnpm run check:acpx-profiles && pnpm run ensure:eval-build-deps && pnpm run check:protocol-types && node ./node_modules/typescript/bin/tsc --version && node ./node_modules/typescript/bin/tsc -p tsconfig.json && node ./node_modules/typescript/bin/tsc -p tsconfig.surfaces.json && node scripts/build-verified-provider-entrypoints.mjs && node scripts/install-copilot-assets.mjs", "build:rust": "cargo build --manifest-path runner/Cargo.toml --locked --workspace --bins", "build:binary": "cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd && node scripts/stage-runner-binary.mjs", diff --git a/packages/paperclip-runner/scripts/build-runner-binary.mjs b/packages/paperclip-runner/scripts/build-runner-binary.mjs new file mode 100644 index 00000000000..dd9b25be093 --- /dev/null +++ b/packages/paperclip-runner/scripts/build-runner-binary.mjs @@ -0,0 +1,52 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { stageRunnerBinary } from "./stage-runner-binary.mjs"; + +export const RELEASE_ARTIFACTS_VARIABLE = "PAPERCLIP_RELEASE_RUNNER_ARTIFACTS_DIR"; + +export function hostRunnerTarget(platform = process.platform, arch = process.arch) { + return `${platform}-${arch}`; +} + +export async function stageReleaseHostBinary({ artifactDirectory, sourceRevision, packageRoot, verifyMetadata, target = hostRunnerTarget(), stage = stageRunnerBinary }) { + const { releaseDaemonManifest } = await import("./stage-release-runner-binaries.mjs"); + const manifest = releaseDaemonManifest(artifactDirectory, sourceRevision, verifyMetadata); + const artifact = manifest.platforms[target]; + if (!artifact) throw new Error(`No exact-source release daemon for ${target}`); + const executable = target.startsWith("win32-") ? "paperclip-runnerd.exe" : "paperclip-runnerd"; + await stage(artifact.path, path.join(packageRoot, "dist", "bin", executable)); + return { target, sha256: artifact.sha256 }; +} + +function pnpmInvocation(env) { + const npmExecPath = env.npm_execpath?.trim(); + if (npmExecPath && npmExecPath.toLowerCase().includes("pnpm")) { + return /\.(c|m)?js$/i.test(npmExecPath) ? { command: process.execPath, args: [npmExecPath] } : { command: npmExecPath, args: [] }; + } + return { command: "pnpm", args: [] }; +} + +export async function main({ env = process.env, packageRoot, log = console.log } = {}) { + const root = packageRoot ?? path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + const artifactDirectory = env[RELEASE_ARTIFACTS_VARIABLE]?.trim(); + if (artifactDirectory) { + const { metadataContract } = await import("./stage-release-runner-binaries.mjs"); + const sourceRevision = execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); + const staged = await stageReleaseHostBinary({ artifactDirectory, sourceRevision, packageRoot: root, verifyMetadata: await metadataContract() }); + log(`Staged the verified exact-source ${staged.target} release daemon (${staged.sha256}) instead of compiling it.`); + return { reused: true, ...staged }; + } + const { command, args } = pnpmInvocation(env); + const result = spawnSync(command, [...args, "run", "build:binary"], { cwd: root, env, stdio: "inherit", shell: process.platform === "win32" && command === "pnpm" }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`build:binary exited with ${result.status ?? result.signal}`); + return { reused: false }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { + console.error(error.message); + process.exitCode = 1; + }); +} diff --git a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs index ab6153b7ce7..0b5b1224572 100644 --- a/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs +++ b/packages/paperclip-runner/scripts/stage-release-runner-binaries.mjs @@ -31,7 +31,7 @@ function regularFile(path, maximumSize) { return readFileSync(path); } -async function metadataContract() { +export async function metadataContract() { const { PAPERCLIP_RUNNER_BUILD_METADATA } = await import("../dist/evals/build-metadata.js"); const { parsePaperclipRunnerdBuildMetadata } = await import("../dist/evals/runnerd-artifact.js"); return metadata => verifyReleaseRunnerMetadata(parsePaperclipRunnerdBuildMetadata(metadata), PAPERCLIP_RUNNER_BUILD_METADATA); diff --git a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs index ddffc53e4c8..e262d64a213 100644 --- a/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs +++ b/packages/paperclip-runner/scripts/stage-runner-binary.test.mjs @@ -7,6 +7,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; import { stageBuiltRunnerBinary, stageRunnerBinary } from "./stage-runner-binary.mjs"; +import { main as buildRunnerBinary, RELEASE_ARTIFACTS_VARIABLE, stageReleaseHostBinary } from "./build-runner-binary.mjs"; import { RELEASE_RUNNER_TARGETS, releaseRunnerTargets, releaseDaemonManifest, verifyReleaseRunnerMetadata } from "./stage-release-runner-binaries.mjs"; import { paperclipRunnerBinaryNeedsBuild, resolveNativeRunnerDevBinary } from "../../../scripts/dev-runner-native-binary.mjs"; import { resolvePackagedRunnerBinary } from "../src/live/runner-binary.ts"; @@ -231,3 +232,60 @@ test("failed preparation preserves the published bytes and mode", async () => { await rm(root, { recursive: true, force: true }); } }); + +async function writeReleaseDaemons(root, source) { + for (const target of RELEASE_RUNNER_TARGETS) { + await mkdir(path.join(root, target), { recursive: true }); + const bytes = Buffer.alloc(64); + if (target.startsWith("linux-")) { + bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(target === "linux-x64" ? 62 : 183, 18); + } else { bytes.writeUInt32LE(0xfeedfacf); bytes.writeUInt32LE(target === "darwin-x64" ? 0x01000007 : 0x0100000c, 4); } + await writeFile(path.join(root, target, "paperclip-runnerd"), bytes); + await writeFile(path.join(root, target, "receipt.json"), JSON.stringify({ schema: "paperclip.runner.release-artifact.v1", sourceRevision: source, + target, sha256: "sha256:" + createHash("sha256").update(bytes).digest("hex"), buildMetadata: metadata })); + } +} + +test("a release build stages the verified exact-source host daemon instead of compiling it", async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "runner-release-host-"))), source = "a".repeat(40); + const artifacts = path.join(root, "artifacts"), packageRoot = path.join(root, "package"); + const verifyMetadata = value => verifyReleaseRunnerMetadata(value, contract); + const staged = []; + const stage = async (from, to) => { staged.push([from, to]); await mkdir(path.dirname(to), { recursive: true }); await copyFile(from, to); }; + try { + await writeReleaseDaemons(artifacts, source); + for (const target of RELEASE_RUNNER_TARGETS) { + const result = await stageReleaseHostBinary({ artifactDirectory: artifacts, sourceRevision: source, packageRoot, verifyMetadata, target, stage }); + assert.equal(result.target, target); + const bytes = await readFile(path.join(packageRoot, "dist", "bin", "paperclip-runnerd")); + assert.deepEqual(bytes, await readFile(path.join(artifacts, target, "paperclip-runnerd"))); + assert.deepEqual(staged.at(-1), [path.join(artifacts, target, "paperclip-runnerd"), path.join(packageRoot, "dist", "bin", "paperclip-runnerd")]); + } + await assert.rejects(stageReleaseHostBinary({ artifactDirectory: artifacts, sourceRevision: "b".repeat(40), packageRoot, verifyMetadata, target: "linux-x64", stage }), + /source, digest or architecture/); + await assert.rejects(stageReleaseHostBinary({ artifactDirectory: artifacts, sourceRevision: source, packageRoot, verifyMetadata, target: "win32-x64", stage }), + /No exact-source release daemon for win32-x64/); + await writeFile(path.join(artifacts, "linux-x64", "paperclip-runnerd"), Buffer.alloc(64, 1)); + await assert.rejects(stageReleaseHostBinary({ artifactDirectory: artifacts, sourceRevision: source, packageRoot, verifyMetadata, target: "linux-x64", stage }), + /digest or architecture/); + assert.equal(staged.length, RELEASE_RUNNER_TARGETS.length, "rejected artifacts are never staged"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("a build without release daemons still compiles the Runner binary", async () => { + const root = await realpath(await mkdtemp(path.join(tmpdir(), "runner-local-build-"))); + try { + const marker = path.join(root, "ran"); + await writeFile(path.join(root, "build.cjs"), `require("node:fs").writeFileSync(${JSON.stringify(marker)}, "yes");\n`); + await writeFile(path.join(root, "package.json"), JSON.stringify({ name: "fixture", private: true, scripts: { "build:binary": "node build.cjs" } })); + const env = { ...process.env }; + delete env[RELEASE_ARTIFACTS_VARIABLE]; + const result = await buildRunnerBinary({ env, packageRoot: root, log: () => {} }); + assert.deepEqual(result, { reused: false }); + assert.equal(await readFile(marker, "utf8"), "yes"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/server/src/__tests__/runner-image-identity.test.ts b/server/src/__tests__/runner-image-identity.test.ts index 43f497eac14..18e73885d62 100644 --- a/server/src/__tests__/runner-image-identity.test.ts +++ b/server/src/__tests__/runner-image-identity.test.ts @@ -348,7 +348,7 @@ describe("Runner image stages", () => { "pnpm run check:protocol-coverage", "node scripts/generate-semantic-contracts.mjs --check", "node scripts/check-runner-workflow-traceability.mjs", - "pnpm run build:binary", + "node scripts/build-runner-binary.mjs", "node scripts/generate-replay-goldens.mjs --check", "node scripts/generate-semantic-action-catalog.mjs --check", "node scripts/record-build-inputs.mjs --finish",