fix(evals): reject incomplete hosted context before admission

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -239,8 +239,8 @@ The manifest schema is `paperclip.runner-eval.controller/v1`. It contains:
   the controller starts no exporter or OTLP relay.
 
 For GitHub workers, the manifest must also contain `hosted` with `repository`,
-`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the trusted
-orchestration commit). Its `campaignId` must be `gha-<runId>-1`. Before creating
+`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the run's
+`GITHUB_SHA`, including the merge commit for a pull-request run). Its `campaignId` must be `gha-<runId>-1`. Before creating
 an admission record, reading provider credentials, or starting a proxy, both
 `archive.mjs verify` and `run.mjs` compare this identity with the current
 `GITHUB_ACTIONS`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_RUN_ATTEMPT` and

packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs

@@ -17,8 +17,6 @@ import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runn
 
 const childScript = options => `
 import assert from 'node:assert/strict';
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
 import {createHash} from 'node:crypto';
 import {spawn} from 'node:child_process';
 import {once} from 'node:events';
@@ -664,8 +662,13 @@ test('hosted workers reject reruns and identity drift before requests, while ret
 
 test('hosted context requires a manifest identity and rejects invalid or later-attempt manifests', async t => {
   const f = await fixture(t);
-  f.environment.GITHUB_ACTIONS = 'true';
-  await assert.rejects(f.run(), /requires a hosted controller identity/);
+  let calls = 0;
+  const transport = async () => { calls++; return success(); };
+  for (const key of ['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT', 'GITHUB_SHA']) {
+    f.environment[key] = key === 'GITHUB_SHA' ? 'c'.repeat(40) : 'true';
+    await assert.rejects(f.run(0, transport), /requires a hosted controller identity/);
+    delete f.environment[key];
+  }
   const hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) };
   f.manifest.campaignId = 'gha-12345-1';
   for (const change of [{ repository: '' }, { repository: 'paperclip' }, { runId: 12345 }, { runId: '012345' },
@@ -674,6 +677,10 @@ test('hosted context requires a manifest identity and rejects invalid or later-a
     assert.throws(() => validateManifest(f.manifest), /hosted controller identity|controller fields/);
   }
   f.manifest.hosted = hosted;
+  await f.save();
+  await assert.rejects(f.run(0, transport), /current first-attempt worker/);
+  assert.equal(calls, 0);
+  await assert.rejects(readdir(f.manifest.outputDirectory), { code: 'ENOENT' });
   f.manifest.campaignId = 'gha-12345-2';
   assert.throws(() => validateManifest(f.manifest), /hosted controller identity/);
 });

packages/paperclip-runner/scripts/controlled-evals/manifest.mjs

@@ -97,7 +97,7 @@ export function assertHostedAdmission(manifest, attempt, environment) {
   validateHostedIdentity(manifest);
   const hosted = manifest.hosted;
   if (!hosted) {
-    if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'].some(key => environment[key] !== undefined)) {
+    if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT', 'GITHUB_SHA'].some(key => environment[key] !== undefined)) {
       throw new Error('GitHub execution requires a hosted controller identity');
     }
     return;

packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs

@@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc
     console.log('stdin imports are side-effect free');
   `);
   assert.match((await stdin).stdout, /stdin imports are side-effect free/);
-  const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns';
+  const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns|hosted context requires a manifest';
   const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], {
     cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576,
   });
-  assert.match(behavioral.stdout, /# pass 6\b/);
+  assert.match(behavioral.stdout, /# pass 7\b/);
   const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], {
     cwd: root, env: environment, timeout: 10_000,
   }).then(value => value, error => error);