diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index eb843d5ef41..c19be136b54 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -1,7 +1,7 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { constants } from 'node:fs'; -import { lstat, open, readdir, realpath, writeFile } from 'node:fs/promises'; +import { lstat, open, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; @@ -179,7 +179,7 @@ export async function summarizeControlledEvidence(bundles, options) { incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns }; } -if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { const [command, ...args] = process.argv.slice(2); if (command === 'verify' && args.length === 2) { const { manifest } = validateManifest(await readJson(args[0])); diff --git a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs index 4fdffefca17..18619a59b4b 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs @@ -1,15 +1,16 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { createHash } from 'node:crypto'; import { createRequire } from 'node:module'; import { cp, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; import { basename, dirname, join, relative, resolve } from 'node:path'; -import { fileURLToPath, pathToFileURL } from 'node:url'; +import { fileURLToPath } from 'node:url'; import { RUNTIME_MAX_BYTES, RUNTIME_MAX_FILES } from './verify-runtime.mjs'; const source = dirname(fileURLToPath(import.meta.url)); const repository = resolve(source, '../../../..'); const require = createRequire(join(repository, 'server/package.json')); const sha256 = bytes => createHash('sha256').update(bytes).digest('hex'); -const scripts = ['archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; +const scripts = ['entrypoint.mjs', 'archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs']; export async function buildControllerRuntime({ outputDirectory, sourceRevision }) { if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error('Controller runtime requires an exact source revision'); @@ -67,28 +68,28 @@ export async function buildControllerRuntime({ outputDirectory, sourceRevision } inputs[relative(repository, absolute).split('\\').join('/')] = sha256(await readFile(absolute)); } for (const file of [...scripts, 'build-runtime.mjs', 'runtime.mjs']) inputs[relative(repository, join(source, file))] = sha256(await readFile(join(source, file))); - const files = {}; let totalBytes = 0; + const files = {}; let payloadBytes = 0; async function inventory(directory) { for (const entry of await readdir(directory, { withFileTypes: true })) { const path = join(directory, entry.name); if (entry.isDirectory()) await inventory(path); else if (entry.isFile()) { - const bytes = await readFile(path); totalBytes += bytes.length; - if (totalBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES - 1) throw new Error('Controller runtime exceeds its artifact allowance'); + const bytes = await readFile(path); payloadBytes += bytes.length; + if (payloadBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES - 1) throw new Error('Controller runtime exceeds its artifact allowance'); files[relative(output, path).split('\\').join('/')] = { sha256: sha256(bytes), sizeBytes: bytes.length }; } else throw new Error('Controller runtime contains a nonregular file'); } } await inventory(output); - const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, totalBytes, files }; + const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, payloadBytes, files }; const receiptBytes = JSON.stringify(receipt, null, 2) + '\n'; - if (totalBytes + Buffer.byteLength(receiptBytes) > RUNTIME_MAX_BYTES) throw new Error('Controller runtime receipt exceeds its artifact allowance'); + if (payloadBytes + Buffer.byteLength(receiptBytes) > RUNTIME_MAX_BYTES) throw new Error('Controller runtime receipt exceeds its artifact allowance'); await writeFile(join(output, 'runtime.json'), receiptBytes, { flag: 'wx', mode: 0o600 }); - return { ...receipt, receiptSha256: sha256(receiptBytes) }; + return { ...receipt, totalBytes: payloadBytes + Buffer.byteLength(receiptBytes), receiptSha256: sha256(receiptBytes) }; } -if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { if (process.argv.length !== 4) throw new Error('Usage: build-runtime.mjs '); const receipt = await buildControllerRuntime({ outputDirectory: process.argv[2], sourceRevision: process.argv[3] }); - process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length }) + '\n'); + process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length + 1 }) + '\n'); } diff --git a/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs b/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs new file mode 100644 index 00000000000..32730bdd180 --- /dev/null +++ b/packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs @@ -0,0 +1,8 @@ +import { realpath } from 'node:fs/promises'; +import { pathToFileURL } from 'node:url'; + +export async function isEntrypoint(url, argument = process.argv[1]) { + if (!argument || argument === '-') return false; + try { return url === pathToFileURL(await realpath(argument)).href; } + catch { return false; } +} diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index e81f7b615d7..321afc88eb0 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -3,7 +3,7 @@ import { execFile } from 'node:child_process'; import { mkdtemp, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; import { promisify } from 'node:util'; import { test } from 'node:test'; import { buildControllerRuntime } from './build-runtime.mjs'; @@ -60,6 +60,15 @@ test('copied controller runs without source dependencies, retains native evidenc const probe = join(root, 'probe.mjs'); await writeFile(probe, isolatedProbe); const isolated = await exec(process.execPath, [probe, directory], { cwd: root, env: environment, timeout: 20_000 }); assert.match(isolated.stdout, /preserves 64-bit values/); + const stdin = exec(process.execPath, ['--input-type=module', '-'], { cwd: root, env: environment, timeout: 10_000 }); + stdin.child.stdin.end(` + for (const name of ['run.mjs', 'archive.mjs', 'report.mjs', 'verify-runtime.mjs']) await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals') + '/').href)} + name); + await import(${JSON.stringify(pathToFileURL(join(source, 'build-runtime.mjs')).href)}); + const { isEntrypoint } = await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals/entrypoint.mjs')).href)}); + if (await isEntrypoint(import.meta.url, '/missing/entrypoint.mjs')) throw new Error('Missing entrypoint was accepted'); + console.log('stdin imports are side-effect free'); + `); + assert.match((await stdin).stdout, /stdin imports are side-effect free/); const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export'; const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, diff --git a/packages/paperclip-runner/scripts/controlled-evals/report.mjs b/packages/paperclip-runner/scripts/controlled-evals/report.mjs index 17d8cf4ee56..801514efdd8 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/report.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/report.mjs @@ -1,6 +1,6 @@ -import { readFile, realpath } from 'node:fs/promises'; +import { isEntrypoint } from './entrypoint.mjs'; +import { readFile } from 'node:fs/promises'; import { join } from 'node:path'; -import { pathToFileURL } from 'node:url'; import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { inspectControlledObservability } from './observability.mjs'; import { attemptDirectory } from './run.mjs'; @@ -94,7 +94,7 @@ export async function summarizeControlledCampaign(value, { filesByAttempt } = {} committedUsd: usd(committed), overBudget: committed > units(manifest.maxUsd), incompleteAttempts: attempts.filter(attempt => attempt.status === 'incomplete').length, costStatus: attempts.some(attempt => attempt.status === 'incomplete') || committed > units(manifest.maxUsd) ? 'incomplete' : 'provisional', attempts }; } -if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { if (process.argv.length !== 3) throw new Error('Usage: report.mjs '); const result = await summarizeControlledCampaign(await readJson(process.argv[2])); process.stdout.write(JSON.stringify(result, null, 2) + '\n'); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index c97864cb5ea..2ac16dc9ae1 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -1,3 +1,4 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises'; @@ -202,7 +203,7 @@ registerHooks({ resolve(specifier, context, nextResolve) { } finally { await observability?.close(); } } -if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { const controller = new AbortController(); const cancel = () => controller.abort(); process.once('SIGINT', cancel); process.once('SIGTERM', cancel); diff --git a/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs index c88ad62abcf..1bf3821acda 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs @@ -1,8 +1,8 @@ +import { isEntrypoint } from './entrypoint.mjs'; import { createHash } from 'node:crypto'; import { constants } from 'node:fs'; -import { lstat, open, readdir, realpath } from 'node:fs/promises'; +import { lstat, open, readdir } from 'node:fs/promises'; import { join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; export const RUNTIME_MAX_BYTES = 32 * 1_048_576; export const RUNTIME_MAX_FILES = 4096; @@ -32,7 +32,7 @@ export async function verifyControllerRuntime({ directory, sourceRevision, recei if (sha256(receiptBytes) !== receiptSha256) throw new Error('Runtime receipt digest mismatch'); const receipt = JSON.parse(receiptBytes.toString('utf8')); if (receipt.schema !== 'paperclip.runner-eval.controller-runtime/v1' || receipt.sourceRevision !== sourceRevision - || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.totalBytes) || receipt.totalBytes < 1 || receipt.totalBytes + receiptBytes.length > RUNTIME_MAX_BYTES + || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.payloadBytes) || receipt.payloadBytes < 1 || receipt.payloadBytes + receiptBytes.length > RUNTIME_MAX_BYTES || !receipt.files || typeof receipt.files !== 'object' || Array.isArray(receipt.files) || !Object.keys(receipt.files).length || Object.keys(receipt.files).length >= RUNTIME_MAX_FILES) throw new Error('Invalid runtime receipt'); const pending = new Map(Object.entries(receipt.files)); @@ -55,14 +55,14 @@ export async function verifyControllerRuntime({ directory, sourceRevision, recei } } await inspect(root); - if (pending.size || bytes !== receipt.totalBytes) throw new Error('Runtime artifact inventory mismatch'); + if (pending.size || bytes !== receipt.payloadBytes) throw new Error('Runtime artifact inventory mismatch'); for (const name of ['run.mjs', 'archive.mjs', 'runtime.mjs']) { if (!Object.hasOwn(receipt.files, `controlled-evals/${name}`)) throw new Error('Runtime entrypoint is missing'); } - return { sourceRevision, receiptSha256, totalBytes: bytes, files: Object.keys(receipt.files).length }; + return { sourceRevision, receiptSha256, totalBytes: bytes + receiptBytes.length, files: Object.keys(receipt.files).length + 1 }; } -if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) { +if (await isEntrypoint(import.meta.url)) { if (process.argv.length !== 5) throw new Error('Usage: verify-runtime.mjs '); process.stdout.write(JSON.stringify(await verifyControllerRuntime({ directory: process.argv[2], sourceRevision: process.argv[3], receiptSha256: process.argv[4] })) + '\n'); }