From 8c9d2a0e4ac12dbce8f04e9c39c50af386d9e0bd Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 24 Oct 2026 15:27:35 -0700 Subject: [PATCH] feat(evals): package a portable Runner controller runtime Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 45 +++++++++ packages/paperclip-runner/package.json | 2 +- .../scripts/controlled-evals/archive.mjs | 4 +- .../controlled-evals/build-runtime.mjs | 93 +++++++++++++++++++ .../controlled-evals/controller.test.mjs | 13 ++- .../scripts/controlled-evals/manifest.mjs | 6 +- .../controlled-evals/portable.test.mjs | 88 ++++++++++++++++++ .../scripts/controlled-evals/report.mjs | 6 +- .../scripts/controlled-evals/run.mjs | 2 +- .../scripts/controlled-evals/runtime.mjs | 4 + .../controlled-evals/verify-runtime.mjs | 68 ++++++++++++++ 11 files changed, 315 insertions(+), 16 deletions(-) create mode 100644 packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/runtime.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 4a87b8a6aa2..4192fcdee6d 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -329,6 +329,51 @@ GitHub workers. Hosted integration must bind unique cells to the prepared allocation, reject workflow/job reruns, and preserve the controller evidence before enabling paid work. Hosted tracing, destination readback, vendor metering bounds, and actual paid qualification remain separate gates. + +### Portable controller runtime + +The controller can run from a copied artifact without a repository checkout, +`tsx`, or a worker-side dependency install. Build it once from the trusted +orchestration checkout after installing that checkout's pinned dependencies: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs \ + /private/new-controller-runtime-directory "$(git rev-parse HEAD)" +``` + +The output directory must not exist. The build copies the controller entrypoints, +bundles the shared cost and tracing code, and copies the installed protobuf +runtime dependency closure. It does not install dependencies or run package +lifecycle scripts. `runtime.json` records the caller-supplied source revision, +source input hashes, installed package manifest hashes, and every output file's +size and SHA-256. The caller must build from the intended trusted checkout; the +revision label alone is not source attestation. The artifact is limited to +32 MiB and 4,096 files. It contains no campaign credentials or allowance. + +Save the printed `receiptSha256` outside the artifact, such as a trusted build +job output. Before a worker loads any copied code, invoke the verifier from its +trusted orchestration checkout with that digest and the expected revision: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs \ + /private/copied-controller-runtime-directory \ + "$TRUSTED_CONTROLLER_REVISION" "$TRUSTED_CONTROLLER_RECEIPT_SHA256" +``` + +The verifier rejects altered, missing, unexpected, or nonregular files and a +mismatched receipt. Do not obtain the expected digest from the downloaded +artifact itself or execute its verifier before checking it with trusted code. +Verification assumes a worker-owned directory that no other process can mutate +between verification and execution. + +Use the copied `controlled-evals/run.mjs` as `--runner-cli` and its sibling +`archive.mjs` as `PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI`. The existing reviewed campaign +manifest is still required. Its executable paths and digests bind the actual +Runner build, not this controller wrapper. The artifact does not authorize paid +work, coordinate worker allocations, or prevent a workflow rerun by itself. +The hosted matrix still needs the shared manifest and per-worker admission +checks described above. + ## Controller tracing allocations and retained proof The opt-in controller uses the existing native launch generator and Product E2E diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index f42661d79c4..b49d71285e3 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index f17ef9b882c..eb843d5ef41 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -1,5 +1,5 @@ import { constants } from 'node:fs'; -import { lstat, open, readdir, writeFile } from 'node:fs/promises'; +import { lstat, open, readdir, realpath, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; @@ -179,7 +179,7 @@ export async function summarizeControlledEvidence(bundles, options) { incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns }; } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { const [command, ...args] = process.argv.slice(2); if (command === 'verify' && args.length === 2) { const { manifest } = validateManifest(await readJson(args[0])); diff --git a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs new file mode 100644 index 00000000000..b7b690fc931 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs @@ -0,0 +1,93 @@ +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { cp, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { RUNTIME_MAX_BYTES, RUNTIME_MAX_FILES } from './verify-runtime.mjs'; + +const source = dirname(fileURLToPath(import.meta.url)); +const repository = resolve(source, '../../../..'); +const require = createRequire(join(repository, 'server/package.json')); +const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); +const scripts = ['archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; + +export async function buildControllerRuntime({ outputDirectory, sourceRevision }) { + if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error('Controller runtime requires an exact source revision'); + const output = resolve(outputDirectory); + await mkdir(output, { mode: 0o700 }); + const controller = join(output, 'controlled-evals'); + await mkdir(controller, { mode: 0o700 }); + for (const file of scripts) await cp(join(source, file), join(controller, file), { errorOnExist: true, force: false }); + const runtime = join(output, 'tests/runner-e2e/shared-runtime.mjs'); + await mkdir(dirname(runtime), { recursive: true, mode: 0o700 }); + const { build } = await import(require.resolve('esbuild')); + const result = await build({ entryPoints: [join(source, 'shared-cost-runtime.ts')], outfile: runtime, bundle: true, + platform: 'node', format: 'esm', target: 'node24', metafile: true, logLevel: 'silent', + banner: { js: "import { createRequire as __paperclipCreateRequire } from 'node:module'; const require = __paperclipCreateRequire(import.meta.url);" } }); + if (result.warnings.length) throw new Error('Controller runtime bundle emitted unresolved build warnings'); + const { isBuiltin } = await import('node:module'); + if (Object.values(result.metafile.outputs).some(item => item.imports.some(item => !isBuiltin(item.path)))) throw new Error('Controller runtime has unbundled imports'); + async function packageManifest(name, parentRequire) { + try { return await realpath(parentRequire.resolve(`${name}/package.json`)); } + catch (error) { if (error.code !== 'ERR_PACKAGE_PATH_NOT_EXPORTED') throw error; } + let directory = dirname(await realpath(parentRequire.resolve(name))); + while (true) { + const candidate = join(directory, 'package.json'); + try { + if (JSON.parse(await readFile(candidate, 'utf8')).name === name) return candidate; + } catch (error) { if (error.code !== 'ENOENT') throw error; } + const parent = dirname(directory); + if (parent === directory) throw new Error('Controller dependency package metadata is missing'); + directory = parent; + } + } + let packages = 0; + const packageInputs = {}; + async function copyPackage(name, parentRequire, destination, ancestors = new Set()) { + if (++packages > 128 || ancestors.size > 16) throw new Error('Controller dependency closure exceeds its limit'); + const manifest = await packageManifest(name, parentRequire); + if (ancestors.has(manifest)) throw new Error('Controller dependency closure contains an unsupported cycle'); + const bytes = await readFile(manifest), metadata = JSON.parse(bytes.toString('utf8')); + packageInputs[`${metadata.name}@${metadata.version}`] = sha256(bytes); + const target = join(destination, 'node_modules', name); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await cp(dirname(manifest), target, { recursive: true, dereference: true, errorOnExist: true, force: false, + filter: path => basename(path) !== 'node_modules' }); + for (const dependency of Object.keys(metadata.dependencies ?? {}).sort()) { + await copyPackage(dependency, createRequire(manifest), target, new Set([...ancestors, manifest])); + } + } + const server = join(output, 'server'); await mkdir(server, { mode: 0o700 }); + await writeFile(join(server, 'package.json'), '{"private":true}\n', { flag: 'wx', mode: 0o600 }); + await copyPackage('protobufjs', require, server); + await writeFile(join(controller, 'runtime.mjs'), "export * as costRuntime from '../tests/runner-e2e/shared-runtime.mjs';\n", { flag: 'wx', mode: 0o600 }); + const inputs = {}; + for (const path of Object.keys(result.metafile.inputs)) { + const absolute = resolve(path); + inputs[relative(repository, absolute).split('\\').join('/')] = sha256(await readFile(absolute)); + } + for (const file of [...scripts, 'build-runtime.mjs', 'runtime.mjs']) inputs[relative(repository, join(source, file))] = sha256(await readFile(join(source, file))); + const files = {}; let totalBytes = 0; + async function inventory(directory) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await inventory(path); + else if (entry.isFile()) { + const bytes = await readFile(path); totalBytes += bytes.length; + if (totalBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES) throw new Error('Controller runtime exceeds its artifact allowance'); + files[relative(output, path).split('\\').join('/')] = { sha256: sha256(bytes), sizeBytes: bytes.length }; + } else throw new Error('Controller runtime contains a nonregular file'); + } + } + await inventory(output); + const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, totalBytes, files }; + const receiptBytes = JSON.stringify(receipt, null, 2) + '\n'; + await writeFile(join(output, 'runtime.json'), receiptBytes, { flag: 'wx', mode: 0o600 }); + return { ...receipt, receiptSha256: sha256(receiptBytes) }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { + if (process.argv.length !== 4) throw new Error('Usage: build-runtime.mjs '); + const receipt = await buildControllerRuntime({ outputDirectory: process.argv[2], sourceRevision: process.argv[3] }); + process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length }) + '\n'); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 7162d41663e..00463e55d11 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -3,11 +3,14 @@ import { test } from 'node:test'; import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; -import { attemptDirectory, runControlledEval } from './run.mjs'; -import { summarizeControlledCampaign } from './report.mjs'; -import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME + ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; +const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } = await import(new URL('./manifest.mjs', runtime)); +const { attemptDirectory, runControlledEval } = await import(new URL('./run.mjs', runtime)); +const { summarizeControlledCampaign } = await import(new URL('./report.mjs', runtime)); +const { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } = await import(new URL('./archive.mjs', runtime)); import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const childScript = options => ` diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index 333e13cb81d..90d309b2545 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -1,11 +1,9 @@ import { createHash } from 'node:crypto'; -import { createRequire } from 'node:module'; import { readFile } from 'node:fs/promises'; import { isAbsolute } from 'node:path'; -const require = createRequire(new URL('../../../../server/package.json', import.meta.url)); -const { tsImport } = await import(require.resolve('tsx/esm/api')); -export const costRuntime = await tsImport('./shared-cost-runtime.ts', import.meta.url); +import { costRuntime } from './runtime.mjs'; +export { costRuntime }; export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); export const canonical = value => JSON.stringify(value, (_key, item) => item && typeof item === 'object' && !Array.isArray(item) ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item); diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs new file mode 100644 index 00000000000..e81f7b615d7 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -0,0 +1,88 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { test } from 'node:test'; +import { buildControllerRuntime } from './build-runtime.mjs'; +import { verifyControllerRuntime } from './verify-runtime.mjs'; + +const exec = promisify(execFile); +const source = dirname(fileURLToPath(import.meta.url)); +const isolatedProbe = ` +import assert from 'node:assert/strict'; +import { registerHooks, isBuiltin, createRequire } from 'node:module'; +import { realpath } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { join } from 'node:path'; +const root = await realpath(process.argv[2]), prefix = pathToFileURL(root + '/').href; +registerHooks({ resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (context.parentURL?.startsWith(prefix) && !isBuiltin(result.url) && !result.url.startsWith(prefix)) throw new Error('Runtime depends on a file outside its artifact'); + return result; +} }); +const { costRuntime } = await import(prefix + 'controlled-evals/manifest.mjs'); +const protobuf = createRequire(join(root, 'server/package.json'))('protobufjs'); +const type = protobuf.parse('syntax = "proto3"; message Value { int64 int_value = 3; } message Log { fixed64 time_unix_nano = 1; Value body = 5; } message Scope { repeated Log log_records = 2; } message Resource { repeated Scope scope_logs = 2; } message Data { repeated Resource resource_logs = 1; }').root.lookupType('Data'); +const received = []; +const receiptPath = join(process.cwd(), 'protobuf-receipt.jsonl'); +const relay = await costRuntime.startEvalOtlpRelay({ projectId: '11111111-1111-4111-8111-111111111111', apiKey: 'synthetic-only', + campaignId: 'portable', attemptId: 'portable-protobuf', receiptPath, deadlineAt: Date.now() + 10000, + transport: async (_url, init) => { received.push(type.decode(Buffer.from(init.body)).toJSON()); return Response.json({}); } }); +const varint = number => { let value = BigInt(number), bytes = []; do { const byte = Number(value & 127n); value >>= 7n; bytes.push(byte | (value ? 128 : 0)); } while (value); return Buffer.from(bytes); }; +const message = (field, ...parts) => { const bytes = Buffer.concat(parts); return Buffer.concat([varint(field * 8 + 2), varint(bytes.length), bytes]); }; +const nanos = '1792879903123456789', integer = '9223372036854775806'; +const timestamp = Buffer.alloc(9); timestamp[0] = 9; timestamp.writeBigUInt64LE(BigInt(nanos), 1); +const body = message(1, message(2, message(2, timestamp, message(5, varint(24), varint(integer))))); +try { + const result = await fetch(relay.baseUrl + '/v1/logs', { method: 'POST', headers: { authorization: 'Bearer ' + relay.token, 'content-type': 'application/x-protobuf' }, body }); + assert.equal(result.status, 200); await result.arrayBuffer(); + assert.equal(received.length, 1); + const record = received[0].resourceLogs[0].scopeLogs[0].logRecords[0]; + assert.equal(record.timeUnixNano, nanos); assert.equal(record.body.intValue, integer); +} finally { await relay.close(); } +console.log('portable protobuf preserves 64-bit values'); +`; + +test('copied controller runs without source dependencies, retains native evidence, and detects artifact changes', { timeout: 120_000 }, async t => { + const root = await mkdtemp(join(tmpdir(), 'runner-portable-controller-')); + t.after(() => rm(root, { recursive: true, force: true })); + const built = join(root, 'build'), directory = join(root, 'relocated'); + const sourceRevision = 'a'.repeat(40); + const receipt = await buildControllerRuntime({ outputDirectory: built, sourceRevision }); + await assert.rejects(buildControllerRuntime({ outputDirectory: built, sourceRevision }), { code: 'EEXIST' }); + await rename(built, directory); + const input = { directory, sourceRevision, receiptSha256: receipt.receiptSha256 }; + assert.equal((await verifyControllerRuntime(input)).totalBytes, receipt.totalBytes); + const environment = { PATH: process.env.PATH, ...(process.env.LANG ? { LANG: process.env.LANG } : {}) }; + const probe = join(root, 'probe.mjs'); await writeFile(probe, isolatedProbe); + const isolated = await exec(process.execPath, [probe, directory], { cwd: root, env: environment, timeout: 20_000 }); + assert.match(isolated.stdout, /preserves 64-bit values/); + const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export'; + const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { + cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, + }); + assert.match(behavioral.stdout, /# pass 5\b/); + const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { + cwd: root, env: environment, timeout: 10_000, + }).then(value => value, error => error); + assert.equal(cli.code, 1); assert.match(cli.stderr, /ENOENT|no such file/); + const entryAlias = join(root, 'run-alias.mjs'); await symlink(join(directory, 'controlled-evals/run.mjs'), entryAlias); + const runCli = await exec(process.execPath, [entryAlias], { cwd: root, env: environment, timeout: 10_000 }) + .then(value => value, error => error); + assert.equal(runCli.code, 2); assert.match(runCli.stderr, /admission or retention failed/); + await assert.rejects(verifyControllerRuntime({ ...input, sourceRevision: 'b'.repeat(40) }), /Invalid runtime receipt/); + await assert.rejects(verifyControllerRuntime({ ...input, receiptSha256: '0'.repeat(64) }), /digest mismatch/); + const file = join(directory, 'controlled-evals/run.mjs'), bytes = await readFile(file); + await writeFile(file, Buffer.concat([bytes, Buffer.from('\n')])); + await assert.rejects(verifyControllerRuntime(input), /oversized|content mismatch/); + await rm(file); await assert.rejects(verifyControllerRuntime(input), /inventory mismatch/); + await symlink(probe, file); await assert.rejects(verifyControllerRuntime(input), /nonregular entry/); + await rm(file); await writeFile(file, bytes); + const extra = join(directory, 'unexpected.mjs'); await writeFile(extra, 'throw new Error()'); + await assert.rejects(verifyControllerRuntime(input), /Unexpected runtime artifact/); + await rm(extra); + assert.equal((await verifyControllerRuntime(input)).receiptSha256, receipt.receiptSha256); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index 5a30f01f298..17d8cf4ee56 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -1,5 +1,5 @@ -import { readFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { readFile, realpath } from 'node:fs/promises'; +import { join } from 'node:path'; import { pathToFileURL } from 'node:url'; import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { inspectControlledObservability } from './observability.mjs'; @@ -94,7 +94,7 @@ export async function summarizeControlledCampaign(value, { filesByAttempt } = {} committedUsd: usd(committed), overBudget: committed > units(manifest.maxUsd), incompleteAttempts: attempts.filter(attempt => attempt.status === 'incomplete').length, costStatus: attempts.some(attempt => attempt.status === 'incomplete') || committed > units(manifest.maxUsd) ? 'incomplete' : 'provisional', attempts }; } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { if (process.argv.length !== 3) throw new Error('Usage: report.mjs '); const result = await summarizeControlledCampaign(await readJson(process.argv[2])); process.stdout.write(JSON.stringify(result, null, 2) + '\n'); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 005afa43922..c97864cb5ea 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -202,7 +202,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { } finally { await observability?.close(); } } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { const controller = new AbortController(); const cancel = () => controller.abort(); process.once('SIGINT', cancel); process.once('SIGTERM', cancel); diff --git a/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs new file mode 100644 index 00000000000..e78a8d0ba2d --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs @@ -0,0 +1,4 @@ +import { createRequire } from 'node:module'; +const require = createRequire(new URL('../../../../server/package.json', import.meta.url)); +const { tsImport } = await import(require.resolve('tsx/esm/api')); +export const costRuntime = await tsImport('./shared-cost-runtime.ts', import.meta.url); diff --git a/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs new file mode 100644 index 00000000000..5d5164d191c --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs @@ -0,0 +1,68 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, open, readdir, realpath } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +export const RUNTIME_MAX_BYTES = 32 * 1_048_576; +export const RUNTIME_MAX_FILES = 4096; +const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); +async function readRegular(path, limit) { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > limit) throw new Error('Runtime artifact contains an oversized or nonregular file'); + const buffer = Buffer.alloc(stat.size + 1); + let size = 0; + while (size < buffer.length) { + const { bytesRead } = await handle.read(buffer, size, buffer.length - size, null); + if (!bytesRead) break; + size += bytesRead; + } + if (size !== stat.size) throw new Error('Runtime artifact changed during verification'); + return buffer.subarray(0, size); + } finally { await handle.close(); } +} + +export async function verifyControllerRuntime({ directory, sourceRevision, receiptSha256 }) { + if (!/^[a-f0-9]{40}$/.test(sourceRevision ?? '') || !/^[a-f0-9]{64}$/.test(receiptSha256 ?? '')) throw new Error('Runtime verification requires trusted revision and receipt digest'); + const root = resolve(directory); + if (!(await lstat(root)).isDirectory()) throw new Error('Runtime root must be a real directory'); + const receiptBytes = await readRegular(join(root, 'runtime.json'), 2 * 1_048_576); + if (sha256(receiptBytes) !== receiptSha256) throw new Error('Runtime receipt digest mismatch'); + const receipt = JSON.parse(receiptBytes.toString('utf8')); + if (receipt.schema !== 'paperclip.runner-eval.controller-runtime/v1' || receipt.sourceRevision !== sourceRevision + || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.totalBytes) || receipt.totalBytes < 1 || receipt.totalBytes > RUNTIME_MAX_BYTES + || !receipt.files || typeof receipt.files !== 'object' || Array.isArray(receipt.files) + || !Object.keys(receipt.files).length || Object.keys(receipt.files).length > RUNTIME_MAX_FILES) throw new Error('Invalid runtime receipt'); + const pending = new Map(Object.entries(receipt.files)); + let bytes = 0, entries = 0; + async function inspect(path, prefix = '') { + for (const item of await readdir(path, { withFileTypes: true })) { + if (++entries > RUNTIME_MAX_FILES * 2) throw new Error('Too many runtime artifact entries'); + const name = prefix + item.name, fullPath = join(path, item.name); + if (item.isDirectory()) await inspect(fullPath, name + '/'); + else if (item.isFile()) { + if (name === 'runtime.json') continue; + const expected = pending.get(name); + if (!expected || !Number.isSafeInteger(expected.sizeBytes) || expected.sizeBytes < 0 || expected.sizeBytes > RUNTIME_MAX_BYTES + || !/^[a-f0-9]{64}$/.test(expected.sha256 ?? '')) throw new Error('Unexpected runtime artifact file'); + const data = await readRegular(fullPath, Math.min(expected.sizeBytes, RUNTIME_MAX_BYTES - bytes)); + bytes += data.length; + if (data.length !== expected.sizeBytes || sha256(data) !== expected.sha256) throw new Error('Runtime artifact content mismatch'); + pending.delete(name); + } else throw new Error('Runtime artifact contains a nonregular entry'); + } + } + await inspect(root); + if (pending.size || bytes !== receipt.totalBytes) throw new Error('Runtime artifact inventory mismatch'); + for (const name of ['run.mjs', 'archive.mjs', 'runtime.mjs']) { + if (!Object.hasOwn(receipt.files, `controlled-evals/${name}`)) throw new Error('Runtime entrypoint is missing'); + } + return { sourceRevision, receiptSha256, totalBytes: bytes, files: Object.keys(receipt.files).length }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { + if (process.argv.length !== 5) throw new Error('Usage: verify-runtime.mjs '); + process.stdout.write(JSON.stringify(await verifyControllerRuntime({ directory: process.argv[2], sourceRevision: process.argv[3], receiptSha256: process.argv[4] })) + '\n'); +}