fix(evals): keep stdin imports free of CLI side effects

Also include receipt bytes and the receipt file in reported artifact totals.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/scripts/controlled-evals/archive.mjs

@@ -1,7 +1,7 @@
+import { isEntrypoint } from './entrypoint.mjs';
 import { constants } from 'node:fs';
-import { lstat, open, readdir, realpath, writeFile } from 'node:fs/promises';
+import { lstat, open, readdir, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
 import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { attemptDirectory, controlledResultArtifact } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
@@ -179,7 +179,7 @@ export async function summarizeControlledEvidence(bundles, options) {
     incompleteAttempts: campaigns.reduce((total, c) => total + c.incompleteAttempts, 0), overBudget: campaigns.some(c => c.overBudget), campaigns };
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) {
+if (await isEntrypoint(import.meta.url)) {
   const [command, ...args] = process.argv.slice(2);
   if (command === 'verify' && args.length === 2) {
     const { manifest } = validateManifest(await readJson(args[0]));

packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs

@@ -1,15 +1,16 @@
+import { isEntrypoint } from './entrypoint.mjs';
 import { createHash } from 'node:crypto';
 import { createRequire } from 'node:module';
 import { cp, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises';
 import { basename, dirname, join, relative, resolve } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
+import { fileURLToPath } from 'node:url';
 import { RUNTIME_MAX_BYTES, RUNTIME_MAX_FILES } from './verify-runtime.mjs';
 
 const source = dirname(fileURLToPath(import.meta.url));
 const repository = resolve(source, '../../../..');
 const require = createRequire(join(repository, 'server/package.json'));
 const sha256 = bytes => createHash('sha256').update(bytes).digest('hex');
-const scripts = ['archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs'];
+const scripts = ['entrypoint.mjs', 'archive.mjs', 'manifest.mjs', 'observability.mjs', 'report.mjs', 'run.mjs', 'verify-runtime.mjs'];
 
 export async function buildControllerRuntime({ outputDirectory, sourceRevision }) {
   if (!/^[a-f0-9]{40}$/.test(sourceRevision)) throw new Error('Controller runtime requires an exact source revision');
@@ -67,28 +68,28 @@ export async function buildControllerRuntime({ outputDirectory, sourceRevision }
     inputs[relative(repository, absolute).split('\\').join('/')] = sha256(await readFile(absolute));
   }
   for (const file of [...scripts, 'build-runtime.mjs', 'runtime.mjs']) inputs[relative(repository, join(source, file))] = sha256(await readFile(join(source, file)));
-  const files = {}; let totalBytes = 0;
+  const files = {}; let payloadBytes = 0;
   async function inventory(directory) {
     for (const entry of await readdir(directory, { withFileTypes: true })) {
       const path = join(directory, entry.name);
       if (entry.isDirectory()) await inventory(path);
       else if (entry.isFile()) {
-        const bytes = await readFile(path); totalBytes += bytes.length;
-        if (totalBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES - 1) throw new Error('Controller runtime exceeds its artifact allowance');
+        const bytes = await readFile(path); payloadBytes += bytes.length;
+        if (payloadBytes > RUNTIME_MAX_BYTES || Object.keys(files).length >= RUNTIME_MAX_FILES - 1) throw new Error('Controller runtime exceeds its artifact allowance');
         files[relative(output, path).split('\\').join('/')] = { sha256: sha256(bytes), sizeBytes: bytes.length };
       } else throw new Error('Controller runtime contains a nonregular file');
     }
   }
   await inventory(output);
-  const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, totalBytes, files };
+  const receipt = { schema: 'paperclip.runner-eval.controller-runtime/v1', sourceRevision, nodeMajor: 24, inputs, packageInputs, payloadBytes, files };
   const receiptBytes = JSON.stringify(receipt, null, 2) + '\n';
-  if (totalBytes + Buffer.byteLength(receiptBytes) > RUNTIME_MAX_BYTES) throw new Error('Controller runtime receipt exceeds its artifact allowance');
+  if (payloadBytes + Buffer.byteLength(receiptBytes) > RUNTIME_MAX_BYTES) throw new Error('Controller runtime receipt exceeds its artifact allowance');
   await writeFile(join(output, 'runtime.json'), receiptBytes, { flag: 'wx', mode: 0o600 });
-  return { ...receipt, receiptSha256: sha256(receiptBytes) };
+  return { ...receipt, totalBytes: payloadBytes + Buffer.byteLength(receiptBytes), receiptSha256: sha256(receiptBytes) };
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) {
+if (await isEntrypoint(import.meta.url)) {
   if (process.argv.length !== 4) throw new Error('Usage: build-runtime.mjs <new-output-directory> <source-revision>');
   const receipt = await buildControllerRuntime({ outputDirectory: process.argv[2], sourceRevision: process.argv[3] });
-  process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length }) + '\n');
+  process.stdout.write(JSON.stringify({ sourceRevision: receipt.sourceRevision, totalBytes: receipt.totalBytes, receiptSha256: receipt.receiptSha256, files: Object.keys(receipt.files).length + 1 }) + '\n');
 }

packages/paperclip-runner/scripts/controlled-evals/entrypoint.mjs

@@ -0,0 +1,8 @@
+import { realpath } from 'node:fs/promises';
+import { pathToFileURL } from 'node:url';
+
+export async function isEntrypoint(url, argument = process.argv[1]) {
+  if (!argument || argument === '-') return false;
+  try { return url === pathToFileURL(await realpath(argument)).href; }
+  catch { return false; }
+}

packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs

@@ -3,7 +3,7 @@ import { execFile } from 'node:child_process';
 import { mkdtemp, readFile, rename, rm, symlink, writeFile } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 import { promisify } from 'node:util';
 import { test } from 'node:test';
 import { buildControllerRuntime } from './build-runtime.mjs';
@@ -60,6 +60,15 @@ test('copied controller runs without source dependencies, retains native evidenc
   const probe = join(root, 'probe.mjs'); await writeFile(probe, isolatedProbe);
   const isolated = await exec(process.execPath, [probe, directory], { cwd: root, env: environment, timeout: 20_000 });
   assert.match(isolated.stdout, /preserves 64-bit values/);
+  const stdin = exec(process.execPath, ['--input-type=module', '-'], { cwd: root, env: environment, timeout: 10_000 });
+  stdin.child.stdin.end(`
+    for (const name of ['run.mjs', 'archive.mjs', 'report.mjs', 'verify-runtime.mjs']) await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals') + '/').href)} + name);
+    await import(${JSON.stringify(pathToFileURL(join(source, 'build-runtime.mjs')).href)});
+    const { isEntrypoint } = await import(${JSON.stringify(pathToFileURL(join(directory, 'controlled-evals/entrypoint.mjs')).href)});
+    if (await isEntrypoint(import.meta.url, '/missing/entrypoint.mjs')) throw new Error('Missing entrypoint was accepted');
+    console.log('stdin imports are side-effect free');
+  `);
+  assert.match((await stdin).stdout, /stdin imports are side-effect free/);
   const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export';
   const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], {
     cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576,

packages/paperclip-runner/scripts/controlled-evals/report.mjs

@@ -1,6 +1,6 @@
-import { readFile, realpath } from 'node:fs/promises';
+import { isEntrypoint } from './entrypoint.mjs';
+import { readFile } from 'node:fs/promises';
 import { join } from 'node:path';
-import { pathToFileURL } from 'node:url';
 import { canonical, costRuntime, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { inspectControlledObservability } from './observability.mjs';
 import { attemptDirectory } from './run.mjs';
@@ -94,7 +94,7 @@ export async function summarizeControlledCampaign(value, { filesByAttempt } = {}
     committedUsd: usd(committed), overBudget: committed > units(manifest.maxUsd), incompleteAttempts: attempts.filter(attempt => attempt.status === 'incomplete').length,
     costStatus: attempts.some(attempt => attempt.status === 'incomplete') || committed > units(manifest.maxUsd) ? 'incomplete' : 'provisional', attempts };
 }
-if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) {
+if (await isEntrypoint(import.meta.url)) {
   if (process.argv.length !== 3) throw new Error('Usage: report.mjs <controller-manifest.json>');
   const result = await summarizeControlledCampaign(await readJson(process.argv[2]));
   process.stdout.write(JSON.stringify(result, null, 2) + '\n');

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -1,3 +1,4 @@
+import { isEntrypoint } from './entrypoint.mjs';
 import { spawn } from 'node:child_process';
 import { once } from 'node:events';
 import { lstat, mkdir, open, readFile, realpath, rename, rm, writeFile } from 'node:fs/promises';
@@ -202,7 +203,7 @@ registerHooks({ resolve(specifier, context, nextResolve) {
   } finally { await observability?.close(); }
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) {
+if (await isEntrypoint(import.meta.url)) {
   const controller = new AbortController();
   const cancel = () => controller.abort();
   process.once('SIGINT', cancel); process.once('SIGTERM', cancel);

packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs

@@ -1,8 +1,8 @@
+import { isEntrypoint } from './entrypoint.mjs';
 import { createHash } from 'node:crypto';
 import { constants } from 'node:fs';
-import { lstat, open, readdir, realpath } from 'node:fs/promises';
+import { lstat, open, readdir } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
 
 export const RUNTIME_MAX_BYTES = 32 * 1_048_576;
 export const RUNTIME_MAX_FILES = 4096;
@@ -32,7 +32,7 @@ export async function verifyControllerRuntime({ directory, sourceRevision, recei
   if (sha256(receiptBytes) !== receiptSha256) throw new Error('Runtime receipt digest mismatch');
   const receipt = JSON.parse(receiptBytes.toString('utf8'));
   if (receipt.schema !== 'paperclip.runner-eval.controller-runtime/v1' || receipt.sourceRevision !== sourceRevision
-      || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.totalBytes) || receipt.totalBytes < 1 || receipt.totalBytes + receiptBytes.length > RUNTIME_MAX_BYTES
+      || receipt.nodeMajor !== 24 || !Number.isSafeInteger(receipt.payloadBytes) || receipt.payloadBytes < 1 || receipt.payloadBytes + receiptBytes.length > RUNTIME_MAX_BYTES
       || !receipt.files || typeof receipt.files !== 'object' || Array.isArray(receipt.files)
       || !Object.keys(receipt.files).length || Object.keys(receipt.files).length >= RUNTIME_MAX_FILES) throw new Error('Invalid runtime receipt');
   const pending = new Map(Object.entries(receipt.files));
@@ -55,14 +55,14 @@ export async function verifyControllerRuntime({ directory, sourceRevision, recei
     }
   }
   await inspect(root);
-  if (pending.size || bytes !== receipt.totalBytes) throw new Error('Runtime artifact inventory mismatch');
+  if (pending.size || bytes !== receipt.payloadBytes) throw new Error('Runtime artifact inventory mismatch');
   for (const name of ['run.mjs', 'archive.mjs', 'runtime.mjs']) {
     if (!Object.hasOwn(receipt.files, `controlled-evals/${name}`)) throw new Error('Runtime entrypoint is missing');
   }
-  return { sourceRevision, receiptSha256, totalBytes: bytes, files: Object.keys(receipt.files).length };
+  return { sourceRevision, receiptSha256, totalBytes: bytes + receiptBytes.length, files: Object.keys(receipt.files).length + 1 };
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(await realpath(process.argv[1])).href) {
+if (await isEntrypoint(import.meta.url)) {
   if (process.argv.length !== 5) throw new Error('Usage: verify-runtime.mjs <directory> <source-revision> <trusted-receipt-sha256>');
   process.stdout.write(JSON.stringify(await verifyControllerRuntime({ directory: process.argv[2], sourceRevision: process.argv[3], receiptSha256: process.argv[4] })) + '\n');
 }