fix(evals): bind hosted admission to the current first attempt

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -238,6 +238,27 @@ The manifest schema is `paperclip.runner-eval.controller/v1`. It contains:
   `nativeLimits`, and a reviewed full-lifecycle `provisionalBound`. When absent,
   the controller starts no exporter or OTLP relay.
 
+For GitHub workers, the manifest must also contain `hosted` with `repository`,
+`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the trusted
+orchestration commit). Its `campaignId` must be `gha-<runId>-1`. Before creating
+an admission record, reading provider credentials, or starting a proxy, both
+`archive.mjs verify` and `run.mjs` compare this identity with the current
+`GITHUB_ACTIONS`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_RUN_ATTEMPT` and
+`GITHUB_SHA`. Set `PAPERCLIP_RUNNER_EVAL_EXECUTION_ID` from the worker's selected
+cell. It must equal the bound allocation's `executionId`. A reused preparation
+job cannot authorize a later GitHub run attempt. Local manifests remain usable
+outside a GitHub context. A hosted manifest cannot execute with missing context.
+
+This checks the trusted workflow's current environment; it is not a distributed
+lock or signed GitHub attestation. The workflow must prepare one complete manifest,
+distribute identical bytes and fan out each selected cell exactly once. The
+private `prepare_controlled_campaign.py` command prepares requests with the
+existing case compiler and rejects duplicate or missing matrix cells. Independent
+budgets, a second workflow run, or duplicate first-attempt jobs are not authorized
+by this guard. Pack and report commands remain usable offline after the run, so
+expired or failed attempts can still be retained and reconciled. The current
+live workflow has not yet been connected to this contract.
+
 Use `requestBindingSha256` from `controlled-evals/manifest.mjs` to bind the
 complete compiled request. It excludes only the temporary working directory
 and native executable path; the executable digest remains bound and its path

packages/paperclip-runner/scripts/controlled-evals/archive.mjs

@@ -2,7 +2,7 @@ import { isEntrypoint } from './entrypoint.mjs';
 import { constants } from 'node:fs';
 import { lstat, open, readdir, writeFile } from 'node:fs/promises';
 import { basename, join, resolve } from 'node:path';
-import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
+import { assertHostedAdmission, bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs';
 import { attemptDirectory, controlledResultArtifact } from './run.mjs';
 import { summarizeControlledCampaign } from './report.mjs';
 
@@ -183,7 +183,8 @@ if (await isEntrypoint(import.meta.url)) {
   const [command, ...args] = process.argv.slice(2);
   if (command === 'verify' && args.length === 2) {
     const { manifest } = validateManifest(await readJson(args[0]));
-    bindRequest(manifest, await readJson(args[1], 8 * MiB));
+    const attempt = bindRequest(manifest, await readJson(args[1], 8 * MiB));
+    assertHostedAdmission(manifest, attempt, process.env);
   } else if (command === 'pack' && args.length === 3) {
     const bundle = await packControlledAttempt({ manifest: await readJson(args[0]), attemptId: args[1] });
     await writeFile(args[2], JSON.stringify(bundle) + '\n', { flag: 'wx', mode: 0o600 });

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -1,10 +1,12 @@
 import assert from 'node:assert/strict';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 import { test } from 'node:test';
 import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises';
 import { writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME
   ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url;
 const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } = await import(new URL('./manifest.mjs', runtime));
@@ -15,6 +17,8 @@ import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runn
 
 const childScript = options => `
 import assert from 'node:assert/strict';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
 import {createHash} from 'node:crypto';
 import {spawn} from 'node:child_process';
 import {once} from 'node:events';
@@ -622,3 +626,54 @@ test('cancellation during root export stops forwarding and cannot close the atte
   const summary = await summarizeControlledCampaign(f.manifest);
   assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000');
 });
+
+
+test('hosted workers reject reruns and identity drift before requests, while retained evidence replays offline', async t => {
+  const f = await fixture(t);
+  f.manifest.campaignId = 'gha-12345-1';
+  f.manifest.hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) };
+  await f.save();
+  const context = { GITHUB_ACTIONS: 'true', GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_RUN_ID: '12345',
+    GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'c'.repeat(40), PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' };
+  let calls = 0;
+  const transport = async () => { calls++; return success(); };
+  for (const [key, value] of Object.entries(context)) {
+    for (const wrong of [undefined, key === 'GITHUB_RUN_ATTEMPT' ? '2' : 'wrong']) {
+      Object.assign(f.environment, context, { [key]: wrong });
+      await assert.rejects(f.run(0, transport), /current first-attempt worker/);
+      assert.equal(calls, 0);
+      await assert.rejects(readdir(f.manifest.outputDirectory), { code: 'ENOENT' });
+    }
+    f.environment[key] = value;
+  }
+  Object.assign(f.environment, context);
+  const command = [fileURLToPath(new URL('./archive.mjs', runtime)), 'verify', f.manifestPath, f.requests[0].requestPath];
+  const verify = promisify(execFile);
+  await verify(process.execPath, command, { env: { PATH: process.env.PATH, ...context }, timeout: 15000 });
+  await assert.rejects(verify(process.execPath, command, {
+    env: { PATH: process.env.PATH, ...context, GITHUB_RUN_ATTEMPT: '2' }, timeout: 15000,
+  }), /current first-attempt worker/);
+  assert.equal(calls, 0);
+  assert.equal((await f.run(0, transport)).code, 0);
+  assert.equal(calls, 2);
+  const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment });
+  const replay = await summarizeControlledEvidence([bundle], { environment: {} });
+  assert.equal(replay.incompleteAttempts, 0);
+  assert.equal(replay.campaigns[0].campaignId, 'gha-12345-1');
+});
+
+test('hosted context requires a manifest identity and rejects invalid or later-attempt manifests', async t => {
+  const f = await fixture(t);
+  f.environment.GITHUB_ACTIONS = 'true';
+  await assert.rejects(f.run(), /requires a hosted controller identity/);
+  const hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) };
+  f.manifest.campaignId = 'gha-12345-1';
+  for (const change of [{ repository: '' }, { repository: 'paperclip' }, { runId: 12345 }, { runId: '012345' },
+    { runAttempt: 2 }, { workflowRevision: 'master' }, { workflowRevision: ['c'.repeat(40)] }, { unexpected: true }]) {
+    f.manifest.hosted = { ...hosted, ...change };
+    assert.throws(() => validateManifest(f.manifest), /hosted controller identity|controller fields/);
+  }
+  f.manifest.hosted = hosted;
+  f.manifest.campaignId = 'gha-12345-2';
+  assert.throws(() => validateManifest(f.manifest), /hosted controller identity/);
+});

packages/paperclip-runner/scripts/controlled-evals/manifest.mjs

@@ -32,7 +32,7 @@ export async function readJson(file, maximum = 1_048_576) {
 }
 export function validateManifest(value, now = Date.now(), { admission = true } = {}) {
   fields(value, ['schema', 'campaignId', 'maxUsd', 'deadlineAt', 'pricingVerifiedAt', 'pricingExpiresAt', 'overhead',
-    'sourceRevision', 'definitionsRevision', 'runnerCli', 'runnerPackage', 'runnerd', 'outputDirectory', 'route', 'projectId', 'models', 'attempts', 'observability']);
+    'sourceRevision', 'definitionsRevision', 'runnerCli', 'runnerPackage', 'runnerd', 'outputDirectory', 'route', 'projectId', 'models', 'attempts', 'observability', 'hosted']);
   const deadline = Date.parse(value.deadlineAt);
   const verified = Date.parse(value.pricingVerifiedAt), expires = Date.parse(value.pricingExpiresAt);
   if (value.schema !== 'paperclip.runner-eval.controller/v1' || !id(value.campaignId)
@@ -46,6 +46,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } =
     fields(artifact, ['path', 'sha256']);
     if (!isAbsolute(artifact.path ?? '') || !hash(artifact.sha256)) throw new Error('Invalid controller executable provenance');
   }
+  validateHostedIdentity(value);
   const overhead = costRuntime.validateOperatingOverhead(value.overhead, value.maxUsd);
   if (admission && Date.parse(value.overhead.verifiedAt) > now || Date.parse(value.overhead.expiresAt) <= deadline
       || value.overhead.components.some(component => !component.provisionalBound)) throw new Error('Controller requires current full-lifecycle operating bounds');
@@ -82,6 +83,33 @@ export function validateManifest(value, now = Date.now(), { admission = true } =
   validateObservability(value);
   return { manifest: structuredClone(value), manifestSha256: sha256(canonical(value)), allocatedUsd: usd(allocated), overheadUsd: usd(overhead) };
 }
+function validateHostedIdentity(manifest) {
+  const hosted = manifest.hosted;
+  if (hosted === undefined) return;
+  fields(hosted, ['repository', 'runId', 'runAttempt', 'workflowRevision']);
+  if (typeof hosted.repository !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}\/[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$/.test(hosted.repository)
+      || typeof hosted.runId !== 'string' || !/^[1-9][0-9]{0,19}$/.test(hosted.runId)
+      || hosted.runAttempt !== 1 || typeof hosted.workflowRevision !== 'string' || !/^[a-f0-9]{40}$/.test(hosted.workflowRevision)
+      || manifest.campaignId !== `gha-${hosted.runId}-1`) throw new Error('Invalid hosted controller identity');
+}
+
+export function assertHostedAdmission(manifest, attempt, environment) {
+  validateHostedIdentity(manifest);
+  const hosted = manifest.hosted;
+  if (!hosted) {
+    if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'].some(key => environment[key] !== undefined)) {
+      throw new Error('GitHub execution requires a hosted controller identity');
+    }
+    return;
+  }
+  if (environment.GITHUB_ACTIONS !== 'true' || environment.GITHUB_REPOSITORY !== hosted.repository
+      || environment.GITHUB_RUN_ID !== hosted.runId || environment.GITHUB_RUN_ATTEMPT !== '1'
+      || environment.GITHUB_SHA !== hosted.workflowRevision
+      || environment.PAPERCLIP_RUNNER_EVAL_EXECUTION_ID !== attempt.executionId) {
+    throw new Error('Hosted controller admission does not match the current first-attempt worker');
+  }
+}
+
 export function bindRequest(manifest, request) {
   const attempt = manifest.attempts.find(item => item.attemptId === request.attemptId);
   if (!attempt || requestBindingSha256(request) !== attempt.requestSha256 || request.provider !== attempt.provider

packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs

@@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc
     console.log('stdin imports are side-effect free');
   `);
   assert.match((await stdin).stdout, /stdin imports are side-effect free/);
-  const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export';
+  const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns';
   const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], {
     cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576,
   });
-  assert.match(behavioral.stdout, /# pass 5\b/);
+  assert.match(behavioral.stdout, /# pass 6\b/);
   const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], {
     cwd: root, env: environment, timeout: 10_000,
   }).then(value => value, error => error);

packages/paperclip-runner/scripts/controlled-evals/run.mjs

@@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { parseArgs } from 'node:util';
 import { startControlledObservability } from './observability.mjs';
-import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
+import { assertHostedAdmission, bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs';
 
 async function atomicJson(file, value) {
   const temporary = `${file}.tmp`;
@@ -68,6 +68,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath,
   signal?.throwIfAborted();
   const { manifest, manifestSha256 } = validateManifest(await readJson(manifestPath));
   const request = await readJson(requestPath, 8_388_608), attempt = bindRequest(manifest, request);
+  assertHostedAdmission(manifest, attempt, environment);
   try { await lstat(outputPath); throw new Error('Controlled output already exists'); }
   catch (error) { if (error.code !== 'ENOENT') throw error; }
   if (request.session.workingDirectory) {