From 8c702d59dbc45ea5d491379b11b4883c17caebab Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 26 Oct 2026 14:32:46 -0500 Subject: [PATCH] feat(evals): package a portable Runner controller runtime (#17506) Package a verified portable Runner eval controller with complete bounded file receipts, copied dependency closure, private evidence replay and symlink-safe CLI entrypoints. Preserve fail-closed cost holds, native trace receipts and all parent security repairs. Verified head: 1c892ff4640370f2a658beab3339080fd4a86de4. Validation: 72 synthetic source/portable tests pass after current-master sync. Full current-head CI passed (40386180906); Greptile reviewed this exact head at5/5, with zero unresolved findings. Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 46 ++++++ packages/paperclip-runner/package.json | 2 +- .../scripts/controlled-evals/archive.mjs | 4 +- .../controlled-evals/build-runtime.mjs | 95 +++++++++++++ .../controlled-evals/controller.test.mjs | 17 ++- .../scripts/controlled-evals/entrypoint.mjs | 8 ++ .../scripts/controlled-evals/manifest.mjs | 6 +- .../controlled-evals/portable.test.mjs | 131 ++++++++++++++++++ .../scripts/controlled-evals/report.mjs | 6 +- .../scripts/controlled-evals/run.mjs | 3 +- .../scripts/controlled-evals/runtime.mjs | 4 + .../controlled-evals/verify-runtime.mjs | 74 ++++++++++ .../issue-list-assignee-filter-routes.test.ts | 2 + tests/e2e/paused-composer.spec.ts | 6 + 14 files changed, 386 insertions(+), 18 deletions(-) create mode 100644 packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/runtime.mjs create mode 100644 packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 03a7d7788b3..14640e2baa5 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -413,6 +413,52 @@ GitHub workers. Hosted integration must bind unique cells to the prepared allocation, reject workflow/job reruns, and preserve the controller evidence before enabling paid work. Hosted tracing, destination readback, vendor metering bounds, and actual paid qualification remain separate gates. + +### Portable controller runtime + +The controller can run from a copied artifact without a repository checkout, +`tsx`, or a worker-side dependency install. Build it once from the trusted +orchestration checkout after installing that checkout's pinned dependencies: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs \ + /private/new-controller-runtime-directory "$(git rev-parse HEAD)" +``` + +The output directory must not exist. The build copies the controller entrypoints, +bundles the shared cost and tracing code, and copies the installed protobuf +runtime dependency closure. It does not install dependencies or run package +lifecycle scripts. `runtime.json` records the caller-supplied source revision, +source input hashes, installed package manifest hashes, and every output file's +size and SHA-256. The caller must build from the intended trusted checkout; the +revision label alone is not source attestation. The artifact is limited to +32 MiB and 4,096 files, including a receipt limited to 2 MiB by both the builder +and verifier. It contains no campaign credentials or allowance. + +Save the printed `receiptSha256` outside the artifact, such as a trusted build +job output. Before a worker loads any copied code, invoke the verifier from its +trusted orchestration checkout with that digest and the expected revision: + +```sh +node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs \ + /private/copied-controller-runtime-directory \ + "$TRUSTED_CONTROLLER_REVISION" "$TRUSTED_CONTROLLER_RECEIPT_SHA256" +``` + +The verifier rejects altered, missing, unexpected, or nonregular files and a +mismatched receipt. Do not obtain the expected digest from the downloaded +artifact itself or execute its verifier before checking it with trusted code. +Verification assumes a worker-owned directory that no other process can mutate +between verification and execution. + +Use the copied `controlled-evals/run.mjs` as `--runner-cli` and its sibling +`archive.mjs` as `PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI`. The existing reviewed campaign +manifest is still required. Its executable paths and digests bind the actual +Runner build, not this controller wrapper. The artifact does not authorize paid +work, coordinate worker allocations, or prevent a workflow rerun by itself. +The hosted matrix still needs the shared manifest and per-worker admission +checks described above. + ## Controller tracing allocations and retained proof The opt-in controller uses the existing native launch generator and Product E2E diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index f42661d79c4..b49d71285e3 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index 88f77864d44..ec246123490 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -1,6 +1,6 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { lstat, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; @@ -170,7 +170,7 @@ export async function summarizeControlledEvidence(bundles, options) { incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns }; } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { const [command, ...args] = process.argv.slice(2); if (command === 'verify' && args.length === 2) { const { manifest } = validateManifest(await readJson(args[0])); diff --git a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs new file mode 100644 index 00000000000..4b4a2650205 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs @@ -0,0 +1,95 @@ +import { isEntrypoint } from './entrypoint.mjs'; +import { createHash } from 'node:crypto'; +import { createRequire } from 'node:module'; +import { cp, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import { basename, dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { encodeRuntimeReceipt, RUNTIME_MAX_BYTES, RUNTIME_MAX_FILES } from './verify-runtime.mjs'; + +const source = dirname(fileURLToPath(import.meta.url)); +const repository = resolve(source, '../../../..'); +const require = createRequire(join(repository, 'server/package.json')); +const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); +const scripts = ['entrypoint.mjs', 'evidence-files.mjs', 'archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; + +export async function buildControllerRuntime({ outputDirectory, sourceRevision }) { + if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error('Controller runtime requires an exact source revision'); + const output = resolve(outputDirectory); + await mkdir(output, { mode: 0o700 }); + const controller = join(output, 'controlled-evals'); + await mkdir(controller, { mode: 0o700 }); + for (const file of scripts) await cp(join(source, file), join(controller, file), { errorOnExist: true, force: false }); + const runtime = join(output, 'tests/runner-e2e/shared-runtime.mjs'); + await mkdir(dirname(runtime), { recursive: true, mode: 0o700 }); + const { build } = await import(require.resolve('esbuild')); + const result = await build({ entryPoints: [join(source, 'shared-cost-runtime.ts')], outfile: runtime, bundle: true, + platform: 'node', format: 'esm', target: 'node24', metafile: true, logLevel: 'silent', + banner: { js: "import { createRequire as __paperclipCreateRequire } from 'node:module'; const require = __paperclipCreateRequire(import.meta.url);" } }); + if (result.warnings.length) throw new Error('Controller runtime bundle emitted unresolved build warnings'); + const { isBuiltin } = await import('node:module'); + if (Object.values(result.metafile.outputs).some(item => item.imports.some(item => !isBuiltin(item.path)))) throw new Error('Controller runtime has unbundled imports'); + async function packageManifest(name, parentRequire) { + try { return await realpath(parentRequire.resolve(`${name}/package.json`)); } + catch (error) { if (error.code !== 'ERR_PACKAGE_PATH_NOT_EXPORTED') throw error; } + let directory = dirname(await realpath(parentRequire.resolve(name))); + while (true) { + const candidate = join(directory, 'package.json'); + try { + if (JSON.parse(await readFile(candidate, 'utf8')).name === name) return candidate; + } catch (error) { if (error.code !== 'ENOENT') throw error; } + const parent = dirname(directory); + if (parent === directory) throw new Error('Controller dependency package metadata is missing'); + directory = parent; + } + } + let packages = 0; + const packageInputs = {}; + async function copyPackage(name, parentRequire, destination, ancestors = new Set()) { + if (++packages > 128 || ancestors.size > 16) throw new Error('Controller dependency closure exceeds its limit'); + const manifest = await packageManifest(name, parentRequire); + if (ancestors.has(manifest)) throw new Error('Controller dependency closure contains an unsupported cycle'); + const bytes = await readFile(manifest), metadata = JSON.parse(bytes.toString('utf8')); + packageInputs[`${metadata.name}@${metadata.version}`] = sha256(bytes); + const target = join(destination, 'node_modules', name); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + await cp(dirname(manifest), target, { recursive: true, dereference: true, errorOnExist: true, force: false, + filter: path => basename(path) !== 'node_modules' }); + for (const dependency of Object.keys(metadata.dependencies ?? {}).sort()) { + await copyPackage(dependency, createRequire(manifest), target, new Set([...ancestors, manifest])); + } + } + const server = join(output, 'server'); await mkdir(server, { mode: 0o700 }); + await writeFile(join(server, 'package.json'), '{"private":true}\n', { flag: 'wx', mode: 0o600 }); + await copyPackage('protobufjs', require, server); + await writeFile(join(controller, 'runtime.mjs'), "export * as costRuntime from '../tests/runner-e2e/shared-runtime.mjs';\n", { flag: 'wx', mode: 0o600 }); + const inputs = {}; + for (const path of Object.keys(result.metafile.inputs)) { + const absolute = resolve(path); + inputs[relative(repository, absolute).split('\\').join('/')] = sha256(await readFile(absolute)); + } + for (const file of [...scripts, 'build-runtime.mjs', 'runtime.mjs']) inputs[relative(repository, join(source, file))] = sha256(await readFile(join(source, file))); + const files = {}; let payloadBytes = 0; + async function inventory(directory) { + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) await inventory(path); + else if (entry.isFile()) { + const bytes = await readFile(path); payloadBytes += bytes.length; + if (payloadBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES - 1) throw new Error('Controller runtime exceeds its artifact allowance'); + files[relative(output, path).split('\\').join('/')] = { sha256: sha256(bytes), sizeBytes: bytes.length }; + } else throw new Error('Controller runtime contains a nonregular file'); + } + } + await inventory(output); + const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, payloadBytes, files }; + const receiptBytes = encodeRuntimeReceipt(receipt); + if (payloadBytes + Buffer.byteLength(receiptBytes) > RUNTIME_MAX_BYTES) throw new Error('Controller runtime receipt exceeds its artifact allowance'); + await writeFile(join(output, 'runtime.json'), receiptBytes, { flag: 'wx', mode: 0o600 }); + return { ...receipt, totalBytes: payloadBytes + Buffer.byteLength(receiptBytes), receiptSha256: sha256(receiptBytes) }; +} + +if (await isEntrypoint(import.meta.url)) { + if (process.argv.length !== 4) throw new Error('Usage: build-runtime.mjs '); + const receipt = await buildControllerRuntime({ outputDirectory: process.argv[2], sourceRevision: process.argv[3] }); + process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length + 1 }) + '\n'); +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 2de69815d93..b3362a8031b 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -4,14 +4,17 @@ import { execFileSync } from 'node:child_process'; import { mkdtemp, mkdir, open, readFile, readdir, rm, stat, symlink, truncate, writeFile } from 'node:fs/promises'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { join, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { createRequire } from 'node:module'; -import { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } from './manifest.mjs'; -import { attemptDirectory, runControlledEval } from './run.mjs'; -import { summarizeControlledCampaign } from './report.mjs'; -import { readRegularFile, replaceEvidenceFile } from './evidence-files.mjs'; -import { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } from './archive.mjs'; +const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME + ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; +const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } = await import(new URL('./manifest.mjs', runtime)); +const { attemptDirectory, runControlledEval } = await import(new URL('./run.mjs', runtime)); +const { summarizeControlledCampaign } = await import(new URL('./report.mjs', runtime)); +const { decodeControlledEvidence, packControlledAttempt, readControlledEvidence, readAttemptControlledEvidence, summarizeControlledEvidence } = await import(new URL('./archive.mjs', runtime)); +const { readRegularFile, replaceEvidenceFile } = await import(new URL('./evidence-files.mjs', runtime)); + import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runner-protocol-eval-campaign.mjs'; const { build } = createRequire(new URL('../../../../package.json', import.meta.url))('esbuild'); diff --git a/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs b/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs new file mode 100644 index 00000000000..32730bdd180 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs @@ -0,0 +1,8 @@ +import { realpath } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +export async function isEntrypoint(url, argument = process.argv[1]) { + if (!argument || argument === '-') return false; + try { return url === pathToFileURL(await realpath(argument)).href; } + catch { return false; } +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index 1c437cd0f1e..60f184af35d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -1,11 +1,9 @@ import { createHash } from 'node:crypto'; -import { createRequire } from 'node:module'; import { readFile } from 'node:fs/promises'; import { isAbsolute } from 'node:path'; -const require = createRequire(new URL('../../../../server/package.json', import.meta.url)); -const { tsImport } = await import(require.resolve('tsx/esm/api')); -export const costRuntime = await tsImport('./shared-cost-runtime.ts', import.meta.url); +import { costRuntime } from './runtime.mjs'; +export { costRuntime }; export const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); export const canonical = value => JSON.stringify(value, (_key, item) => item && typeof item === 'object' && !Array.isArray(item) ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item); diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs new file mode 100644 index 00000000000..c2b970a36f0 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -0,0 +1,131 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { promisify } from 'node:util'; +import { test } from 'node:test'; +import { buildControllerRuntime } from './build-runtime.mjs'; +import { encodeRuntimeReceipt, RUNTIME_MAX_RECEIPT_BYTES, verifyControllerRuntime } from './verify-runtime.mjs'; + +const exec = promisify(execFile); +const source = dirname(fileURLToPath(import.meta.url)); +const isolatedProbe = ` +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { registerHooks, isBuiltin, createRequire } from 'node:module'; +import { realpath } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; +import { join } from 'node:path'; +const root = await realpath(process.argv[2]), prefix = pathToFileURL(root + '/').href; +registerHooks({ resolve(specifier, context, nextResolve) { + const result = nextResolve(specifier, context); + if (context.parentURL?.startsWith(prefix) && !isBuiltin(result.url) && !result.url.startsWith(prefix)) throw new Error('Runtime depends on a file outside its artifact'); + return result; +} }); +const { costRuntime } = await import(prefix + 'controlled-evals/manifest.mjs'); +const protobuf = createRequire(join(root, 'server/package.json'))('protobufjs'); +const type = protobuf.parse('syntax = "proto3"; message Value { int64 int_value = 3; } message Log { fixed64 time_unix_nano = 1; Value body = 5; } message Scope { repeated Log log_records = 2; } message Resource { repeated Scope scope_logs = 2; } message Data { repeated Resource resource_logs = 1; }').root.lookupType('Data'); +const received = []; +const receiptPath = join(process.cwd(), 'protobuf-receipt.jsonl'); +const relay = await costRuntime.startEvalOtlpRelay({ projectId: '11111111-1111-4111-8111-111111111111', apiKey: 'synthetic-only', + campaignId: 'portable', attemptId: 'portable-protobuf', receiptPath, deadlineAt: Date.now() + 10000, + transport: async (_url, init) => { received.push(type.decode(Buffer.from(init.body)).toJSON()); return Response.json({}); } }); +const varint = number => { let value = BigInt(number), bytes = []; do { const byte = Number(value & 127n); value >>= 7n; bytes.push(byte | (value ? 128 : 0)); } while (value); return Buffer.from(bytes); }; +const message = (field, ...parts) => { const bytes = Buffer.concat(parts); return Buffer.concat([varint(field * 8 + 2), varint(bytes.length), bytes]); }; +const nanos = '1792879903123456789', integer = '9223372036854775806'; +const timestamp = Buffer.alloc(9); timestamp[0] = 9; timestamp.writeBigUInt64LE(BigInt(nanos), 1); +const body = message(1, message(2, message(2, timestamp, message(5, varint(24), varint(integer))))); +try { + const result = await fetch(relay.baseUrl + '/v1/logs', { method: 'POST', headers: { authorization: 'Bearer ' + relay.token, 'content-type': 'application/x-protobuf' }, body }); + assert.equal(result.status, 200); await result.arrayBuffer(); + assert.equal(received.length, 1); + const record = received[0].resourceLogs[0].scopeLogs[0].logRecords[0]; + assert.equal(record.timeUnixNano, nanos); assert.equal(record.body.intValue, integer); +} finally { await relay.close(); } +console.log('portable protobuf preserves 64-bit values'); +`; + +test('copied controller runs without source dependencies, retains native evidence, and detects artifact changes', { timeout: 120_000 }, async t => { + const root = await mkdtemp(join(tmpdir(), 'runner-portable-controller-')); + t.after(() => rm(root, { recursive: true, force: true })); + const built = join(root, 'build'), directory = join(root, 'relocated'); + const sourceRevision = 'a'.repeat(40); + const receipt = await buildControllerRuntime({ outputDirectory: built, sourceRevision }); + await assert.rejects(buildControllerRuntime({ outputDirectory: built, sourceRevision }), { code: 'EEXIST' }); + await rename(built, directory); + const input = { directory, sourceRevision, receiptSha256: receipt.receiptSha256 }; + assert.equal((await verifyControllerRuntime(input)).totalBytes, receipt.totalBytes); + const rawReceipt = JSON.parse(await readFile(join(directory, 'runtime.json'), 'utf8')); + assert(encodeRuntimeReceipt(rawReceipt).length < RUNTIME_MAX_RECEIPT_BYTES); + const manyInputs = Object.fromEntries(Array.from({ length: 25_000 }, (_, index) => [`src/part-${index}.ts`, 'a'.repeat(64)])); + assert.throws(() => encodeRuntimeReceipt({ ...rawReceipt, inputs: manyInputs }), /receipt exceeds its byte allowance/); + const environment = { PATH: process.env.PATH, ...(process.env.LANG ? { LANG: process.env.LANG } : {}) }; + const probe = join(root, 'probe.mjs'); await writeFile(probe, isolatedProbe); + const isolated = await exec(process.execPath, [probe, directory], { cwd: root, env: environment, timeout: 20_000 }); + assert.match(isolated.stdout, /preserves 64-bit values/); + const stdin = exec(process.execPath, ['--input-type=module', '-'], { cwd: root, env: environment, timeout: 10_000 }); + stdin.child.stdin.end(` + for (const name of ['run.mjs', 'archive.mjs', 'report.mjs', 'verify-runtime.mjs']) await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals') + '/').href)} + name); + await import(${JSON.stringify(pathToFileURL(join(source, 'build-runtime.mjs')).href)}); + const { isEntrypoint } = await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals/entrypoint.mjs')).href)}); + if (await isEntrypoint(import.meta.url, '/missing/entrypoint.mjs')) throw new Error('Missing entrypoint was accepted'); + console.log('stdin imports are side-effect free'); + `); + assert.match((await stdin).stdout, /stdin imports are side-effect free/); + const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|native post-child otlp-native.jsonl rejects fifo|credential redaction replaces|missing provider artifacts|failed final OTLP receipt'; + const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { + cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, + }); + assert.match(behavioral.stdout, /# pass 9\b/); + const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { + cwd: root, env: environment, timeout: 10_000, + }).then(value => value, error => error); + assert.equal(cli.code, 1); assert.match(cli.stderr, /ENOENT|no such file/); + const entryAlias = join(root, 'run-alias.mjs'); await symlink(join(directory, 'controlled-evals/run.mjs'), entryAlias); + const runCli = await exec(process.execPath, [entryAlias], { cwd: root, env: environment, timeout: 10_000 }) + .then(value => value, error => error); + assert.equal(runCli.code, 2); assert.match(runCli.stderr, /admission or retention failed/); + await assert.rejects(verifyControllerRuntime({ ...input, sourceRevision: 'b'.repeat(40) }), /Invalid runtime receipt/); + await assert.rejects(verifyControllerRuntime({ ...input, receiptSha256: '0'.repeat(64) }), /digest mismatch/); + const file = join(directory, 'controlled-evals/run.mjs'), bytes = await readFile(file); + await writeFile(file, Buffer.concat([bytes, Buffer.from('\n')])); + await assert.rejects(verifyControllerRuntime(input), /oversized|content mismatch/); + await rm(file); await assert.rejects(verifyControllerRuntime(input), /inventory mismatch/); + await symlink(probe, file); await assert.rejects(verifyControllerRuntime(input), /nonregular entry/); + await rm(file); await writeFile(file, bytes); + const extra = join(directory, 'unexpected.mjs'); await writeFile(extra, 'throw new Error()'); + await assert.rejects(verifyControllerRuntime(input), /Unexpected runtime artifact/); + await rm(extra); + assert.equal((await verifyControllerRuntime(input)).receiptSha256, receipt.receiptSha256); +}); + + +test('only the root runtime receipt is excluded from the verified file inventory', async t => { + const root = await mkdtemp(join(tmpdir(), 'runner-runtime-nested-receipt-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sourceRevision = 'a'.repeat(40), files = {}; + const digest = bytes => createHash('sha256').update(bytes).digest('hex'); + let payloadBytes = 0; + for (const name of ['controlled-evals/run.mjs', 'controlled-evals/archive.mjs', 'controlled-evals/runtime.mjs', 'dependency/runtime.json']) { + const bytes = Buffer.from(name.endsWith('.json') ? '{"dependency":true}' : 'export {};'); + await mkdir(dirname(join(root, name)), { recursive: true }); + await writeFile(join(root, name), bytes); + files[name] = { sizeBytes: bytes.length, sha256: digest(bytes) }; payloadBytes += bytes.length; + } + const receipt = encodeRuntimeReceipt({ schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, payloadBytes, files }); + await writeFile(join(root, 'runtime.json'), receipt); + const input = { directory: root, sourceRevision, receiptSha256: digest(receipt) }; + assert.equal((await verifyControllerRuntime(input)).files, 5); + const nested = join(root, 'dependency/runtime.json'), original = await readFile(nested); + await writeFile(nested, Buffer.from('{"dependency":fake}')); + await assert.rejects(verifyControllerRuntime(input), /content mismatch|oversized/); + await writeFile(nested, original); + await mkdir(join(root, 'dependency/deeper')); + await writeFile(join(root, 'dependency/deeper/runtime.json'), '{}'); + await assert.rejects(verifyControllerRuntime(input), /Unexpected runtime artifact file/); + await rm(join(root, 'dependency/deeper'), { recursive: true }); + assert.equal((await verifyControllerRuntime(input)).files, 5); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index b3614f1b72a..cef7a45e583 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -1,5 +1,5 @@ -import { join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { isEntrypoint } from './entrypoint.mjs'; +import { join } from 'node:path'; import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { inspectControlledObservability } from './observability.mjs'; import { attemptDirectory } from './run.mjs'; @@ -96,7 +96,7 @@ export async function summarizeControlledCampaign(value, { filesByAttempt } = {} committedUsd: usd(committed), overBudget: committed > units(manifest.maxUsd), incompleteAttempts: attempts.filter(attempt => attempt.status === 'incomplete').length, costStatus: attempts.some(attempt => attempt.status === 'incomplete') || committed > units(manifest.maxUsd) ? 'incomplete' : 'provisional', attempts }; } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { if (process.argv.length !== 3) throw new Error('Usage: report.mjs '); const result = await summarizeControlledCampaign(await readJson(process.argv[2])); process.stdout.write(JSON.stringify(result, null, 2) + '\n'); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 5389b2dba23..5f303e093ee 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -1,3 +1,4 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'; @@ -213,7 +214,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { } -if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { const controller = new AbortController(); const cancel = () => controller.abort(); process.once('SIGINT', cancel); process.once('SIGTERM', cancel); diff --git a/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs new file mode 100644 index 00000000000..e78a8d0ba2d --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/runtime.mjs @@ -0,0 +1,4 @@ +import { createRequire } from 'node:module'; +const require = createRequire(new URL('../../../../server/package.json', import.meta.url)); +const { tsImport } = await import(require.resolve('tsx/esm/api')); +export const costRuntime = await tsImport('./shared-cost-runtime.ts', import.meta.url); diff --git a/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs new file mode 100644 index 00000000000..15d08a0a912 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs @@ -0,0 +1,74 @@ +import { isEntrypoint } from './entrypoint.mjs'; +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { lstat, open, readdir } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; + +export const RUNTIME_MAX_BYTES = 32 * 1_048_576; +export const RUNTIME_MAX_FILES = 4096; +export const RUNTIME_MAX_RECEIPT_BYTES = 2 * 1_048_576; +export function encodeRuntimeReceipt(receipt) { + const bytes = Buffer.from(JSON.stringify(receipt, null, 2) + '\n'); + if (bytes.length > RUNTIME_MAX_RECEIPT_BYTES) throw new Error('Controller runtime receipt exceeds its byte allowance'); + return bytes; +} +const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); +async function readRegular(path, limit) { + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > limit) throw new Error('Runtime artifact contains an oversized or nonregular file'); + const buffer = Buffer.alloc(stat.size + 1); + let size = 0; + while (size < buffer.length) { + const { bytesRead } = await handle.read(buffer, size, buffer.length - size, null); + if (!bytesRead) break; + size += bytesRead; + } + if (size !== stat.size) throw new Error('Runtime artifact changed during verification'); + return buffer.subarray(0, size); + } finally { await handle.close(); } +} + +export async function verifyControllerRuntime({ directory, sourceRevision, receiptSha256 }) { + if (!/^[a-f0-9]{40}$/.test(sourceRevision ?? '') || !/^[a-f0-9]{64}$/.test(receiptSha256 ?? '')) throw new Error('Runtime verification requires trusted revision and receipt digest'); + const root = resolve(directory); + if (!(await lstat(root)).isDirectory()) throw new Error('Runtime root must be a real directory'); + const receiptBytes = await readRegular(join(root, 'runtime.json'), RUNTIME_MAX_RECEIPT_BYTES); + if (sha256(receiptBytes) !== receiptSha256) throw new Error('Runtime receipt digest mismatch'); + const receipt = JSON.parse(receiptBytes.toString('utf8')); + if (receipt.schema !== 'paperclip.runner-eval.controller-runtime/v1' || receipt.sourceRevision !== sourceRevision + || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.payloadBytes) || receipt.payloadBytes < 1 || receipt.payloadBytes + receiptBytes.length > RUNTIME_MAX_BYTES + || !receipt.files || typeof receipt.files !== 'object' || Array.isArray(receipt.files) + || !Object.keys(receipt.files).length || Object.keys(receipt.files).length >= RUNTIME_MAX_FILES) throw new Error('Invalid runtime receipt'); + const pending = new Map(Object.entries(receipt.files)); + let bytes = 0, entries = 0; + async function inspect(path, prefix = '') { + for (const item of await readdir(path, { withFileTypes: true })) { + if (++entries > RUNTIME_MAX_FILES * 2) throw new Error('Too many runtime artifact entries'); + const name = prefix + item.name, fullPath = join(path, item.name); + if (item.isDirectory()) await inspect(fullPath, name + '/'); + else if (item.isFile()) { + if (name === 'runtime.json') continue; + const expected = pending.get(name); + if (!expected || !Number.isSafeInteger(expected.sizeBytes) || expected.sizeBytes < 0 || expected.sizeBytes > RUNTIME_MAX_BYTES + || !/^[a-f0-9]{64}$/.test(expected.sha256 ?? '')) throw new Error('Unexpected runtime artifact file'); + const data = await readRegular(fullPath, Math.min(expected.sizeBytes, RUNTIME_MAX_BYTES - bytes)); + bytes += data.length; + if (data.length !== expected.sizeBytes || sha256(data) !== expected.sha256) throw new Error('Runtime artifact content mismatch'); + pending.delete(name); + } else throw new Error('Runtime artifact contains a nonregular entry'); + } + } + await inspect(root); + if (pending.size || bytes !== receipt.payloadBytes) throw new Error('Runtime artifact inventory mismatch'); + for (const name of ['run.mjs', 'archive.mjs', 'runtime.mjs']) { + if (!Object.hasOwn(receipt.files, `controlled-evals/${name}`)) throw new Error('Runtime entrypoint is missing'); + } + return { sourceRevision, receiptSha256, totalBytes: bytes + receiptBytes.length, files: Object.keys(receipt.files).length + 1 }; +} + +if (await isEntrypoint(import.meta.url)) { + if (process.argv.length !== 5) throw new Error('Usage: verify-runtime.mjs '); + process.stdout.write(JSON.stringify(await verifyControllerRuntime({ directory: process.argv[2], sourceRevision: process.argv[3], receiptSha256: process.argv[4] })) + '\n'); +} diff --git a/server/src/__tests__/issue-list-assignee-filter-routes.test.ts b/server/src/__tests__/issue-list-assignee-filter-routes.test.ts index e2ab63d853a..1960ccd28b6 100644 --- a/server/src/__tests__/issue-list-assignee-filter-routes.test.ts +++ b/server/src/__tests__/issue-list-assignee-filter-routes.test.ts @@ -883,6 +883,7 @@ describeEmbeddedPostgres("issue list routes assigneeAgentId filter", () => { "descendantOf", "labelId", "participantAgentId", + "createdFromIssueId", ])("returns 422 instead of a DB error for malformed %s filters", async (param) => { const companyId = randomUUID(); await db.insert(companies).values({ @@ -911,6 +912,7 @@ describeEmbeddedPostgres("issue list routes assigneeAgentId filter", () => { "descendantOf", "labelId", "participantAgentId", + "createdFromIssueId", ])("returns 422 from issues/count for malformed %s filters", async (param) => { const companyId = randomUUID(); await db.insert(companies).values({ diff --git a/tests/e2e/paused-composer.spec.ts b/tests/e2e/paused-composer.spec.ts index f3c41525648..92890e48037 100644 --- a/tests/e2e/paused-composer.spec.ts +++ b/tests/e2e/paused-composer.spec.ts @@ -111,7 +111,13 @@ for (const classic of [false, true]) { const subtreeWake = page.getByRole("dialog").getByRole("checkbox"); if (await subtreeWake.isEnabled()) await subtreeWake.uncheck(); else await expect(subtreeWake).not.toBeChecked(); + const released = page.waitForResponse((response) => + response.request().method() === "POST" + && new URL(response.url()).pathname.startsWith(`/api/issues/${task.identifier}/tree-holds/`) + && new URL(response.url()).pathname.endsWith("/release"), + ); await page.getByRole("dialog").getByRole("button", { name: "Resume subtree", exact: true }).click(); + expect((await released).ok()).toBe(true); await expect(page.getByRole("dialog")).toHaveCount(0); await page.goto(`/${company.issuePrefix}/issues/${child.identifier}`); await expect(takeover).toHaveCount(0);