PaperClipAI/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs

import { readFile, mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { isEntrypoint } from './entrypoint.mjs';
import { assertHostedAdmission, canonical, fields, requestBindingSha256, sha256, units, validateManifest } from './manifest.mjs';
import { readRegularFile } from './evidence-files.mjs';

export function hostedPaths(runId) {
  if (typeof runId !== 'string' || !/^[1-9][0-9]{0,19}$/.test(runId)) throw new Error('Invalid hosted run ID');
  const root = `/tmp/paperclip-runner-controlled-${runId}`;
  return { root, runner: join(root, 'runner'), controller: join(root, 'runtime'),
    manifest: join(root, 'controller-manifest.json'), evidence: join(root, 'evidence') };
}

export const HOSTED_DEFAULTS_SCHEMA = 'paperclip.runner-eval.hosted-defaults/v1';
export const HOSTED_POLICY_SCHEMA = 'paperclip.runner-eval.hosted-policy/v2';
const MAX_HOSTED_CELLS = 512;
// The committed price table has no review expiry. Each campaign receives a
// validity window that covers only its own deadline plus a short settlement margin.
const SETTLEMENT_MARGIN_MS = 3_600_000;

// Derive the campaign policy from committed defaults. Cell count comes from the
// selected catalog, prices from the committed table and the dependency lock from
// the committed frozen lockfile. Only the campaign spend cap may be overridden.
export function hostedPolicy(defaults, { cellCount, now = Date.now(), maxUsd, projectId, controllerLockSha256 }) {
  fields(defaults, ['schema', 'maxUsd', 'durationMs', 'route', 'models', 'allocations', 'overhead', 'observability']);
  if (defaults.schema !== HOSTED_DEFAULTS_SCHEMA || !defaults.observability || !Array.isArray(defaults.models) || !defaults.models.length
      || !Number.isSafeInteger(defaults.durationMs) || defaults.durationMs < 1 || defaults.durationMs > 4 * 3_600_000
      || !Array.isArray(defaults.allocations) || !defaults.allocations.length) throw new Error('A bounded hosted policy with tracing is required');
  if (!Number.isSafeInteger(cellCount) || cellCount < 1 || cellCount > MAX_HOSTED_CELLS) throw new Error('Invalid controlled campaign selection');
  if (!/^[a-f0-9]{64}$/.test(controllerLockSha256 ?? '')) throw new Error('Invalid hosted controller lock binding');
  fields(defaults.overhead, ['version', 'accountingReadiness', 'reference', 'runner', 'components']);
  fields(defaults.observability, ['capture', 'nativeLimits']);
  if (defaults.overhead.accountingReadiness !== 'model_requests') throw new Error('Controller requires current full-lifecycle operating bounds');
  const cap = maxUsd === undefined || maxUsd === '' ? defaults.maxUsd : maxUsd;
  units(cap);
  const verifiedAt = new Date(now).toISOString();
  const expiresAt = new Date(now + defaults.durationMs + SETTLEMENT_MARGIN_MS).toISOString();
  return { schema: HOSTED_POLICY_SCHEMA, maxUsd: cap, durationMs: defaults.durationMs, maxCells: cellCount,
    pricingVerifiedAt: verifiedAt, pricingExpiresAt: expiresAt, route: defaults.route,
    ...(defaults.route === 'braintrust' && projectId ? { projectId } : {}),
    models: structuredClone(defaults.models), allocations: structuredClone(defaults.allocations),
    overhead: { ...structuredClone(defaults.overhead), verifiedAt, expiresAt },
    observability: { ...(projectId ? { projectId } : {}), ...structuredClone(defaults.observability) }, controllerLockSha256 };
}

export function prepareHostedManifest({ catalog, catalogSha256, prepared, defaults, maxUsd, projectId, controllerLockSha256, artifacts, environment, now = Date.now() }) {
  const policy = hostedPolicy(defaults, { cellCount: Array.isArray(prepared?.cells) ? prepared.cells.length : 0,
    now, maxUsd, projectId, controllerLockSha256 });
  fields(policy.observability, ['projectId', 'capture', 'nativeLimits']);
  const runId = environment.GITHUB_RUN_ID, paths = hostedPaths(runId);
  if (prepared.schema !== 'paperclip.runner-eval.prepared-requests/v1' || prepared.githubRunId !== runId || prepared.githubRunAttempt !== 1
      || prepared.campaignId !== `gha-${runId}-1` || prepared.catalogSha256 !== catalogSha256
      || catalog.schema !== 'paperclip.runner-protocol-eval.catalog/v1' || catalog.campaignId !== prepared.campaignId
      || prepared.sourceRevision !== environment.GITHUB_SHA || prepared.sourceRevision !== environment.PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA || prepared.sourceRevision !== catalog.source?.paperclipSha
      || prepared.definitionsRevision !== environment.PAPERCLIP_PROTOCOL_EVALS_SHA || prepared.definitionsRevision !== catalog.source?.evalsSha
      || !Array.isArray(prepared.cells) || !prepared.cells.length || prepared.cells.length > policy.maxCells
      || !Array.isArray(catalog.cells) || catalog.cells.length !== prepared.cells.length
      || !Array.isArray(catalog.matrices) || catalog.matrices.length !== 2) throw new Error('Prepared selection does not match the authorized catalog');
  const indexed = new Map(catalog.cells.map(cell => [cell.cellId, cell]));
  const scheduled = catalog.matrices.flatMap(matrix => Array.isArray(matrix.include) ? matrix.include : []);
  if (indexed.size !== catalog.cells.length || scheduled.length !== catalog.cells.length
      || new Set(scheduled.map(cell => cell.cellId)).size !== indexed.size
      || scheduled.some(cell => canonical(cell) !== canonical(indexed.get(cell.cellId)))) throw new Error('Every selected cell must occur exactly once');
  const key = item => canonical([item.provider, item.acpxAgent ?? null, item.model]);
  const allocations = new Map();
  for (const allocation of policy.allocations) {
    fields(allocation, ['provider', 'acpxAgent', 'model', 'maxUsd', 'maxRequests', 'maxConcurrent', 'timeoutMs']);
    if (allocations.has(key(allocation))) throw new Error('Duplicate hosted allocation policy');
    allocations.set(key(allocation), allocation);
  }
  const seen = new Set();
  const attempts = prepared.cells.map(cell => {
    const request = cell.request, expected = indexed.get(cell.cellId), allocation = allocations.get(key(request));
    if (!expected || seen.has(cell.cellId) || !allocation || cell.runId !== `gha-${runId}-1-${cell.cellId}`
        || request.schema !== 'paperclip-runner/eval-session-request/v1'
        || !(request.provider === 'codex' && !request.acpxAgent || request.provider === 'acpx' && request.acpxAgent === 'claude')
        || request.driver !== (request.provider === 'codex' ? 'codex_app_server' : 'acpx_runtime') || request.driver !== expected.driver
        || request.provider !== expected.provider || (request.acpxAgent ?? null) !== (expected.acpxAgent ?? null) || request.model !== expected.model
        || request.runnerd?.path !== join(paths.runner, 'paperclip-runnerd') || request.runnerd.sha256?.replace(/^sha256:/, '') !== artifacts.runnerd.sha256
        || prepared.runnerd?.path !== request.runnerd.path || prepared.runnerd.sha256 !== request.runnerd.sha256) throw new Error('Unsupported or mismatched controlled cell');
    seen.add(cell.cellId);
    return { ...allocation, executionId: cell.cellId, attemptId: request.attemptId, requestSha256: requestBindingSha256(request) };
  });
  const manifest = { schema: 'paperclip.runner-eval.controller/v1', campaignId: prepared.campaignId,
    maxUsd: policy.maxUsd, deadlineAt: new Date(now + policy.durationMs).toISOString(), pricingVerifiedAt: policy.pricingVerifiedAt,
    pricingExpiresAt: policy.pricingExpiresAt, route: policy.route, ...(policy.projectId ? { projectId: policy.projectId } : {}),
    models: policy.models, overhead: policy.overhead, observability: { ...policy.observability },
    sourceRevision: prepared.sourceRevision, definitionsRevision: prepared.definitionsRevision, ...artifacts,
    outputDirectory: paths.evidence, hosted: { repository: environment.GITHUB_REPOSITORY, runId, runAttempt: 1, workflowRevision: environment.GITHUB_SHA, controllerLockSha256: policy.controllerLockSha256, policySha256: sha256(canonical(policy)) }, attempts };
  const validated = validateManifest(manifest, now);
  assertHostedAdmission(manifest, attempts[0], { ...environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: attempts[0].executionId });
  return { ...validated, policy };
}

export async function verifyHostedCell({ manifestPath, manifestSha256, cellId, environment }) {
  const bytes = await readRegularFile(manifestPath, 16 * 1_048_576);
  if (sha256(bytes) !== manifestSha256) throw new Error('Hosted manifest bytes changed');
  const { manifest } = validateManifest(JSON.parse(bytes.toString('utf8')));
  const attempt = manifest.attempts.find(item => item.executionId === cellId);
  if (!attempt) throw new Error('Cell is outside the controlled selection');
  assertHostedAdmission(manifest, attempt, environment);
  return { manifest, attempt };
}

const safeAdmissionReasons = new Set([
  'Invalid hosted run ID', 'Invalid hosted controller lock binding', 'Invalid controller fields', 'A bounded hosted policy with tracing is required',
  'Invalid reviewed cost bound evidence', 'Reviewed cost bound configuration changed',
  'Prepared selection does not match the authorized catalog', 'Every selected cell must occur exactly once',
  'Duplicate hosted allocation policy', 'Unsupported or mismatched controlled cell',
  'Invalid controller campaign identity or validity window', 'Controller requires current full-lifecycle operating bounds',
  'Invalid controlled campaign selection', 'Invalid controller executable provenance', 'Invalid hosted controller identity',
  'Invalid controller model pricing', 'Invalid controlled attempt allocation', 'Controlled allocations exceed the campaign budget',
  'Incomplete operating overhead allowance', 'Operating overhead leaves no model allowance',
  'Invalid operating overhead component', 'Invalid operating overhead fields', 'Invalid controller observability allocation',
  'Observability capacity requires its operating bound evidence', 'Hosted manifest bytes changed',
  'Cell is outside the controlled selection', 'Hosted controller admission does not match the current first-attempt worker',
  'Hosted controller requires GitHub-hosted operating origin',
]);
export function hostedAdmissionDiagnostic(error) {
  if (error instanceof Error && safeAdmissionReasons.has(error.message)) return error.message;
  if (error?.code === 'ENOENT') return 'A required hosted input file is missing';
  if (error instanceof SyntaxError) return 'A hosted input is not valid JSON';
  return 'Invalid or unreadable hosted input; check the catalog, committed defaults and runtime bindings';
}

if (await isEntrypoint(import.meta.url)) {
  try {
    const [command, ...args] = process.argv.slice(2);
    if (command === 'prepare' && args.length === 4) {
      // Defaults are read beside this verified controller code, never from the workspace,
      // which the private evals program can modify during preparation.
      const [catalogPath, preparedPath, runner, output] = args;
      const defaultsPath = fileURLToPath(new URL('./hosted-defaults.json', import.meta.url));
      const json = async path => JSON.parse((await readRegularFile(path, 16 * 1_048_576)).toString('utf8'));
      const catalogBytes = await readRegularFile(catalogPath, 16 * 1_048_576);
      const prepared = await json(preparedPath), paths = hostedPaths(process.env.GITHUB_RUN_ID), artifacts = {};
      for (const [name, relative] of [['runnerCli', 'portable/dist/cli/eval-session.js'], ['runnerPackage', 'paperclip-runner.tgz'], ['runnerd', 'paperclip-runnerd']]) {
        artifacts[name] = { path: join(paths.runner, relative), sha256: sha256(await readFile(join(runner, relative))) };
      }
      const result = prepareHostedManifest({ catalog: JSON.parse(catalogBytes), catalogSha256: sha256(catalogBytes), prepared,
        defaults: await json(defaultsPath), maxUsd: process.env.RUNNER_PROTOCOL_EVAL_MAX_USD || undefined,
        projectId: process.env.PAPERCLIP_RUNNER_EVAL_BRAINTRUST_PROJECT_ID || undefined,
        controllerLockSha256: process.env.PAPERCLIP_RUNNER_EVAL_CONTROLLER_LOCK_SHA256, artifacts, environment: process.env });
      await mkdir(output, { mode: 0o700 });
      await writeFile(join(output, 'controller-manifest.json'), canonical(result.manifest), { flag: 'wx', mode: 0o600 });
      await writeFile(join(output, 'hosted-policy.json'), `${JSON.stringify(result.policy, null, 2)}\n`, { flag: 'wx', mode: 0o600 });
      console.log(JSON.stringify({ manifestSha256: result.manifestSha256, cells: result.manifest.attempts.length,
        maxUsd: result.manifest.maxUsd, allocatedUsd: result.allocatedUsd, providerCalls: 0 }));
    } else if (command === 'verify' && args.length === 3) {
      await verifyHostedCell({ manifestPath: args[0], manifestSha256: args[1], cellId: args[2], environment: process.env });
    } else throw new Error('Usage: hosted.mjs <prepare catalog prepared runner output | verify manifest digest cell>');
  } catch (error) {
    console.error(`Hosted campaign admission failed: ${hostedAdmissionDiagnostic(error)}. No provider work was started by this command.`); process.exitCode = 2;
  }
}