From 904199558d2e6c92e280107b4b8ffb41eecd02a5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 12 Aug 2026 02:12:22 +0000 Subject: [PATCH] fix(runner): qualify release eval artifacts Co-Authored-By: Paperclip --- .../docs/evals-integration.md | 21 ++- .../scripts/check-clean-consumers.mjs | 163 +++++++++++++++++- 2 files changed, 172 insertions(+), 12 deletions(-) diff --git a/packages/paperclip-runner/docs/evals-integration.md b/packages/paperclip-runner/docs/evals-integration.md index 4398c801b99..ecf0cb9588d 100644 --- a/packages/paperclip-runner/docs/evals-integration.md +++ b/packages/paperclip-runner/docs/evals-integration.md @@ -79,13 +79,26 @@ network or provider credentials: The input fixture is `protocol/fixtures/evals/harness-driver-conformance.json`. The packed -clean-consumer gate builds and copies runnerd into an isolated artifact -directory, packs the package, installs it offline, resolves only declared -exports, validates the native fixture, runs driver conformance, verifies binary -metadata/digest, and exercises both compatible and incompatible negotiation. +clean-consumer gate builds runnerd with Cargo's `release` profile, copies it +into an isolated artifact directory, packs the package, installs it offline, +resolves only declared exports, validates the native fixture, runs driver +conformance, verifies binary metadata/digest, and exercises both compatible +and incompatible negotiation. It never qualifies a debug binary. Run it with: ```sh pnpm --filter @paperclipai/paperclip-runner check:clean-consumers ``` + +Set `PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR` to retain the qualifying inputs and +machine-readable proof outside the temporary consumer. The output contains the +package tarball, platform-named release runnerd, `SHA256SUMS`, and +`paperclip-runner-evals-conformance.json`. That record is produced by the clean +consumer after it imports only the packed package, resolves the explicit binary +and digest, and completes deterministic conformance without provider calls. + +```sh +PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR=/absolute/release/directory \ + pnpm --filter @paperclipai/paperclip-runner check:clean-consumers +``` diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs index 7d1cd304a2d..337d6161195 100644 --- a/packages/paperclip-runner/scripts/check-clean-consumers.mjs +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -1,7 +1,7 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { chmod, copyFile, mkdir, mkdtemp, readFile, readdir, realpath, rm, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { tmpdir } from "node:os"; @@ -13,22 +13,48 @@ const scratchParent = process.env.PAPERCLIP_RUN_SCRATCH_DIR await mkdir(scratchParent, { recursive: true }); const scratchRoot = await mkdtemp(join(scratchParent, "paperclip-package-consumers-")); const artifactsRoot = resolve(scratchRoot, "artifacts"); +const publicationRoot = process.env.PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR === undefined + ? undefined + : resolve(process.env.PAPERCLIP_CLEAN_CONSUMER_OUTPUT_DIR); await mkdir(artifactsRoot, { recursive: true }); try { run("pnpm", ["run", "build:typescript"], runnerRoot); - run("pnpm", ["run", "build:runner-binaries"], runnerRoot); + run("cargo", [ + "build", + "--release", + "--manifest-path", + "runner/Cargo.toml", + "--locked", + "-p", + "paperclip-runner-core", + "--bin", + "paperclip-runnerd", + ], runnerRoot); run("pnpm", ["run", "build"], evalKernelRoot); const runnerTarball = await pack(runnerRoot, artifactsRoot); const evalKernelTarball = await pack(evalKernelRoot, artifactsRoot); const runtimeDependencyTarballs = await packRunnerRuntimeDependencies(artifactsRoot); const runnerdArtifact = await stageRunnerdArtifact(artifactsRoot); + const conformanceRecord = resolve( + artifactsRoot, + "paperclip-runner-evals-conformance.json", + ); + const sourceCommit = ( + process.env.PAPERCLIP_SOURCE_COMMIT + ?? capture("git", ["rev-parse", "HEAD"], runnerRoot) + ).trim(); + if (!/^[a-f0-9]{40}$/.test(sourceCommit)) { + throw new Error("PAPERCLIP_SOURCE_COMMIT must be a full lowercase Git commit SHA"); + } await verifyEvalsConsumer( resolve(scratchRoot, "evals-consumer"), runnerTarball, runtimeDependencyTarballs, runnerdArtifact, + conformanceRecord, + sourceCommit, ); await verifyAppDevConsumer( resolve(scratchRoot, "app-dev-consumer"), @@ -46,7 +72,16 @@ try { if ("@paperclipai/paperclip-eval-kernel" in runtimeDependencies) { throw new Error("runner tarball has a runtime dependency on Paperclip Evals"); } - process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App package/runnerd and App dev -> eval kernel.\n"); + if (publicationRoot !== undefined) { + await publishArtifacts({ + publicationRoot, + runnerTarball, + runnerdArtifact, + conformanceRecord, + }); + process.stdout.write(`Published clean-consumer artifacts at ${publicationRoot}\n`); + } + process.stdout.write("Clean-consumer pack/install checks passed for Evals -> App package/release runnerd and App dev -> eval kernel.\n"); } finally { if (process.env.PAPERCLIP_KEEP_PACKAGE_CONSUMERS !== "1") { await rm(scratchRoot, { recursive: true, force: true }); @@ -87,12 +122,16 @@ async function packRunnerRuntimeDependencies(destination) { async function stageRunnerdArtifact(destination) { const suffix = process.platform === "win32" ? ".exe" : ""; - const source = resolve(runnerRoot, `runner/target/debug/paperclip-runnerd${suffix}`); - const executablePath = resolve(destination, `paperclip-runnerd${suffix}`); + const source = resolve(runnerRoot, `runner/target/release/paperclip-runnerd${suffix}`); + const executablePath = resolve( + destination, + `paperclip-runnerd-${process.platform}-${process.arch}${suffix}`, + ); await copyFile(source, executablePath); if (process.platform !== "win32") await chmod(executablePath, 0o755); - const sha256 = `sha256:${createHash("sha256").update(await readFile(executablePath)).digest("hex")}`; - return { executablePath, sha256 }; + const bytes = await readFile(executablePath); + const sha256 = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + return { executablePath, sha256, byteSize: bytes.byteLength, buildProfile: "release" }; } function localOverrides(tarballs) { @@ -106,6 +145,8 @@ async function verifyEvalsConsumer( runnerTarball, runtimeDependencyTarballs, runnerdArtifact, + conformanceRecord, + sourceCommit, ) { await mkdir(consumerRoot, { recursive: true }); await writeFile(resolve(consumerRoot, "package.json"), `${JSON.stringify({ @@ -119,6 +160,10 @@ async function verifyEvalsConsumer( pnpm: { overrides: localOverrides(runtimeDependencyTarballs) }, }, null, 2)}\n`); await writeFile(resolve(consumerRoot, "verify.mjs"), ` +import { createHash } from "node:crypto"; +import { readFile, stat, writeFile } from "node:fs/promises"; +import { basename } from "node:path"; + import * as runtime from "@paperclipai/paperclip-runner"; import * as evals from "@paperclipai/paperclip-runner/evals"; import * as testing from "@paperclipai/paperclip-runner/testing"; @@ -218,20 +263,108 @@ const normalized = { effects: [], audit: [], }; -await testing.runSemanticConformanceKit({ +const semanticConformance = await testing.runSemanticConformanceKit({ vectors: [{ id: "finish", operationId: "finish_task", input: {} }], adapters: [ { id: "mock", execute: async () => normalized }, { id: "real", execute: async () => ({ audit: [], effects: [], state: { status: "done" }, authorization: { outcome: "allowed" } }) }, ], }); + +const packageArtifactPath = process.env.PAPERCLIP_RUNNER_PACKAGE_ARTIFACT; +const packageBytes = await readFile(packageArtifactPath); +const packageStat = await stat(packageArtifactPath); +const record = { + schema: "paperclip-runner/evals-clean-consumer-conformance/v1", + recordedAt: new Date().toISOString(), + sourceCommit: process.env.PAPERCLIP_SOURCE_COMMIT, + platform: { os: process.platform, arch: process.arch }, + artifactInputsOnly: true, + artifacts: { + package: { + filename: basename(packageArtifactPath), + packageName: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.name, + packageVersion: evals.PAPERCLIP_RUNNER_BUILD_METADATA.package.version, + sha256: "sha256:" + createHash("sha256").update(packageBytes).digest("hex"), + byteSize: packageStat.size, + }, + runnerd: { + filename: basename(runnerd.executablePath), + sha256: runnerd.sha256, + byteSize: runnerd.byteSize, + buildProfile: process.env.PAPERCLIP_RUNNERD_BUILD_PROFILE, + buildMetadata: runnerd.buildMetadata, + }, + }, + consumer: { + installMode: "offline-packed-artifact", + packageSpecifier: "file:" + basename(packageArtifactPath), + runnerdLocator: "explicit-path-and-sha256", + imports: [ + "@paperclipai/paperclip-runner", + "@paperclipai/paperclip-runner/evals", + "@paperclipai/paperclip-runner/testing", + ], + appSourceTreeImports: false, + providerCalls: 0, + }, + checks: { + packageExportsResolved: true, + mockControlPlaneConformance: report, + nativeExecutionFixture: { + schema: nativeBundle.schema, + rejectedToolEffectPreserved: true, + }, + harnessDriverConformance: driverConformance, + runnerdDigestAndMetadata: true, + compatibilityNegotiation: integration, + driverMismatchFailedClosed: driverMismatchFailed, + providerMismatchFailedClosed: failedExplicitly, + semanticConformance: { + schema: semanticConformance.schema, + rowCount: semanticConformance.rows.length, + adapterIds: semanticConformance.rows[0]?.adapterIds ?? [], + }, + transcriptComplete: driverConformance.checks.transcriptCompleteness, + }, +}; +await writeFile(process.env.PAPERCLIP_CONFORMANCE_RECORD, JSON.stringify(record, null, 2) + "\\n"); `); installAndRun(consumerRoot, { PAPERCLIP_RUNNERD_ARTIFACT: runnerdArtifact.executablePath, PAPERCLIP_RUNNERD_SHA256: runnerdArtifact.sha256, + PAPERCLIP_RUNNERD_BUILD_PROFILE: runnerdArtifact.buildProfile, + PAPERCLIP_RUNNER_PACKAGE_ARTIFACT: runnerTarball, + PAPERCLIP_CONFORMANCE_RECORD: conformanceRecord, + PAPERCLIP_SOURCE_COMMIT: sourceCommit, }); } +async function publishArtifacts({ + publicationRoot, + runnerTarball, + runnerdArtifact, + conformanceRecord, +}) { + await mkdir(publicationRoot, { recursive: true }); + const files = [runnerTarball, runnerdArtifact.executablePath, conformanceRecord]; + for (const file of files) { + await copyFile(file, resolve(publicationRoot, basename(file))); + } + if (process.platform !== "win32") { + await chmod( + resolve(publicationRoot, basename(runnerdArtifact.executablePath)), + 0o755, + ); + } + const checksums = []; + for (const file of files) { + const digest = createHash("sha256").update(await readFile(file)).digest("hex"); + checksums.push(`${digest} ${basename(file)}`); + } + await writeFile(resolve(publicationRoot, "SHA256SUMS"), `${checksums.join("\n")}\n`); +} + async function verifyAppDevConsumer( consumerRoot, runnerTarball, @@ -324,3 +457,17 @@ function run(command, args, cwd, { quiet = false, env = {} } = {}) { throw new Error(`${command} ${args.join(" ")} failed in ${cwd}`); } } + +function capture(command, args, cwd) { + const result = spawnSync(command, args, { + cwd, + encoding: "utf8", + env: { ...process.env, CI: "true" }, + stdio: ["ignore", "pipe", "pipe"], + }); + if (result.status !== 0) { + process.stderr.write(result.stderr ?? ""); + throw new Error(`${command} ${args.join(" ")} failed in ${cwd}`); + } + return result.stdout; +}