From bdd157b2735eae03558f1570d829fe913e8f3b87 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 07:58:14 -0500 Subject: [PATCH] fix(evals): reject incomplete hosted context before admission Co-Authored-By: Paperclip --- .../docs/runner-protocol-live-evals.md | 4 ++-- .../scripts/controlled-evals/controller.test.mjs | 15 +++++++++++---- .../scripts/controlled-evals/manifest.mjs | 2 +- .../scripts/controlled-evals/portable.test.mjs | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index c9db66c0649..c24b49e3b1c 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -239,8 +239,8 @@ The manifest schema is `paperclip.runner-eval.controller/v1`. It contains: the controller starts no exporter or OTLP relay. For GitHub workers, the manifest must also contain `hosted` with `repository`, -`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the trusted -orchestration commit). Its `campaignId` must be `gha--1`. Before creating +`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the run's +`GITHUB_SHA`, including the merge commit for a pull-request run). Its `campaignId` must be `gha--1`. Before creating an admission record, reading provider credentials, or starting a proxy, both `archive.mjs verify` and `run.mjs` compare this identity with the current `GITHUB_ACTIONS`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_RUN_ATTEMPT` and diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 45242316a1d..c3524ce155d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -17,8 +17,6 @@ import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runn const childScript = options => ` import assert from 'node:assert/strict'; -import { execFile } from 'node:child_process'; -import { promisify } from 'node:util'; import {createHash} from 'node:crypto'; import {spawn} from 'node:child_process'; import {once} from 'node:events'; @@ -664,8 +662,13 @@ test('hosted workers reject reruns and identity drift before requests, while ret test('hosted context requires a manifest identity and rejects invalid or later-attempt manifests', async t => { const f = await fixture(t); - f.environment.GITHUB_ACTIONS = 'true'; - await assert.rejects(f.run(), /requires a hosted controller identity/); + let calls = 0; + const transport = async () => { calls++; return success(); }; + for (const key of ['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT', 'GITHUB_SHA']) { + f.environment[key] = key === 'GITHUB_SHA' ? 'c'.repeat(40) : 'true'; + await assert.rejects(f.run(0, transport), /requires a hosted controller identity/); + delete f.environment[key]; + } const hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) }; f.manifest.campaignId = 'gha-12345-1'; for (const change of [{ repository: '' }, { repository: 'paperclip' }, { runId: 12345 }, { runId: '012345' }, @@ -674,6 +677,10 @@ test('hosted context requires a manifest identity and rejects invalid or later-a assert.throws(() => validateManifest(f.manifest), /hosted controller identity|controller fields/); } f.manifest.hosted = hosted; + await f.save(); + await assert.rejects(f.run(0, transport), /current first-attempt worker/); + assert.equal(calls, 0); + await assert.rejects(readdir(f.manifest.outputDirectory), { code: 'ENOENT' }); f.manifest.campaignId = 'gha-12345-2'; assert.throws(() => validateManifest(f.manifest), /hosted controller identity/); }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index ea362e4ce44..700d7176446 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -97,7 +97,7 @@ export function assertHostedAdmission(manifest, attempt, environment) { validateHostedIdentity(manifest); const hosted = manifest.hosted; if (!hosted) { - if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'].some(key => environment[key] !== undefined)) { + if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT', 'GITHUB_SHA'].some(key => environment[key] !== undefined)) { throw new Error('GitHub execution requires a hosted controller identity'); } return; diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index 1308c5388e2..7a91f3c6146 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc console.log('stdin imports are side-effect free'); `); assert.match((await stdin).stdout, /stdin imports are side-effect free/); - const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns'; + const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns|hosted context requires a manifest'; const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, }); - assert.match(behavioral.stdout, /# pass 6\b/); + assert.match(behavioral.stdout, /# pass 7\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { cwd: root, env: environment, timeout: 10_000, }).then(value => value, error => error);