diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index c6e3334eaeb..c9db66c0649 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -238,6 +238,27 @@ The manifest schema is `paperclip.runner-eval.controller/v1`. It contains: `nativeLimits`, and a reviewed full-lifecycle `provisionalBound`. When absent, the controller starts no exporter or OTLP relay. +For GitHub workers, the manifest must also contain `hosted` with `repository`, +`runId` (a decimal string), `runAttempt: 1`, and `workflowRevision` (the trusted +orchestration commit). Its `campaignId` must be `gha--1`. Before creating +an admission record, reading provider credentials, or starting a proxy, both +`archive.mjs verify` and `run.mjs` compare this identity with the current +`GITHUB_ACTIONS`, `GITHUB_REPOSITORY`, `GITHUB_RUN_ID`, `GITHUB_RUN_ATTEMPT` and +`GITHUB_SHA`. Set `PAPERCLIP_RUNNER_EVAL_EXECUTION_ID` from the worker's selected +cell. It must equal the bound allocation's `executionId`. A reused preparation +job cannot authorize a later GitHub run attempt. Local manifests remain usable +outside a GitHub context. A hosted manifest cannot execute with missing context. + +This checks the trusted workflow's current environment; it is not a distributed +lock or signed GitHub attestation. The workflow must prepare one complete manifest, +distribute identical bytes and fan out each selected cell exactly once. The +private `prepare_controlled_campaign.py` command prepares requests with the +existing case compiler and rejects duplicate or missing matrix cells. Independent +budgets, a second workflow run, or duplicate first-attempt jobs are not authorized +by this guard. Pack and report commands remain usable offline after the run, so +expired or failed attempts can still be retained and reconciled. The current +live workflow has not yet been connected to this contract. + Use `requestBindingSha256` from `controlled-evals/manifest.mjs` to bind the complete compiled request. It excludes only the temporary working directory and native executable path; the executable digest remains bound and its path diff --git a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs index c19be136b54..f186cdf6241 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/archive.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/archive.mjs @@ -2,7 +2,7 @@ import { isEntrypoint } from './entrypoint.mjs'; import { constants } from 'node:fs'; import { lstat, open, readdir, writeFile } from 'node:fs/promises'; import { basename, join, resolve } from 'node:path'; -import { bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; +import { assertHostedAdmission, bindRequest, canonical, fields, readJson, requestBindingSha256, sha256, units, usd, validateManifest } from './manifest.mjs'; import { attemptDirectory, controlledResultArtifact } from './run.mjs'; import { summarizeControlledCampaign } from './report.mjs'; @@ -183,7 +183,8 @@ if (await isEntrypoint(import.meta.url)) { const [command, ...args] = process.argv.slice(2); if (command === 'verify' && args.length === 2) { const { manifest } = validateManifest(await readJson(args[0])); - bindRequest(manifest, await readJson(args[1], 8 * MiB)); + const attempt = bindRequest(manifest, await readJson(args[1], 8 * MiB)); + assertHostedAdmission(manifest, attempt, process.env); } else if (command === 'pack' && args.length === 3) { const bundle = await packControlledAttempt({ manifest: await readJson(args[0]), attemptId: args[1] }); await writeFile(args[2], JSON.stringify(bundle) + '\n', { flag: 'wx', mode: 0o600 }); diff --git a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs index 00463e55d11..45242316a1d 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/controller.test.mjs @@ -1,10 +1,12 @@ import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; import { test } from 'node:test'; import { mkdtemp, mkdir, readFile, readdir, rm, symlink, writeFile } from 'node:fs/promises'; import { writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; -import { pathToFileURL } from 'node:url'; +import { fileURLToPath, pathToFileURL } from 'node:url'; const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; const { canonical, costRuntime, observabilityBoundConfiguration, requestBindingSha256, sha256, validateManifest } = await import(new URL('./manifest.mjs', runtime)); @@ -15,6 +17,8 @@ import { aggregateProtocolEvalCampaign, sanitizeProtocolEvalRuns } from '../runn const childScript = options => ` import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; import {createHash} from 'node:crypto'; import {spawn} from 'node:child_process'; import {once} from 'node:events'; @@ -622,3 +626,54 @@ test('cancellation during root export stops forwarding and cannot close the atte const summary = await summarizeControlledCampaign(f.manifest); assert.equal(summary.committedUsd, '0.014000000'); assert.equal(summary.knownUsd, '0.000068000'); }); + + +test('hosted workers reject reruns and identity drift before requests, while retained evidence replays offline', async t => { + const f = await fixture(t); + f.manifest.campaignId = 'gha-12345-1'; + f.manifest.hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) }; + await f.save(); + const context = { GITHUB_ACTIONS: 'true', GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_RUN_ID: '12345', + GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'c'.repeat(40), PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' }; + let calls = 0; + const transport = async () => { calls++; return success(); }; + for (const [key, value] of Object.entries(context)) { + for (const wrong of [undefined, key === 'GITHUB_RUN_ATTEMPT' ? '2' : 'wrong']) { + Object.assign(f.environment, context, { [key]: wrong }); + await assert.rejects(f.run(0, transport), /current first-attempt worker/); + assert.equal(calls, 0); + await assert.rejects(readdir(f.manifest.outputDirectory), { code: 'ENOENT' }); + } + f.environment[key] = value; + } + Object.assign(f.environment, context); + const command = [fileURLToPath(new URL('./archive.mjs', runtime)), 'verify', f.manifestPath, f.requests[0].requestPath]; + const verify = promisify(execFile); + await verify(process.execPath, command, { env: { PATH: process.env.PATH, ...context }, timeout: 15000 }); + await assert.rejects(verify(process.execPath, command, { + env: { PATH: process.env.PATH, ...context, GITHUB_RUN_ATTEMPT: '2' }, timeout: 15000, + }), /current first-attempt worker/); + assert.equal(calls, 0); + assert.equal((await f.run(0, transport)).code, 0); + assert.equal(calls, 2); + const bundle = await packControlledAttempt({ manifest: f.manifest, attemptId: 'attempt-0', environment: f.environment }); + const replay = await summarizeControlledEvidence([bundle], { environment: {} }); + assert.equal(replay.incompleteAttempts, 0); + assert.equal(replay.campaigns[0].campaignId, 'gha-12345-1'); +}); + +test('hosted context requires a manifest identity and rejects invalid or later-attempt manifests', async t => { + const f = await fixture(t); + f.environment.GITHUB_ACTIONS = 'true'; + await assert.rejects(f.run(), /requires a hosted controller identity/); + const hosted = { repository: 'paperclipai/paperclip', runId: '12345', runAttempt: 1, workflowRevision: 'c'.repeat(40) }; + f.manifest.campaignId = 'gha-12345-1'; + for (const change of [{ repository: '' }, { repository: 'paperclip' }, { runId: 12345 }, { runId: '012345' }, + { runAttempt: 2 }, { workflowRevision: 'master' }, { workflowRevision: ['c'.repeat(40)] }, { unexpected: true }]) { + f.manifest.hosted = { ...hosted, ...change }; + assert.throws(() => validateManifest(f.manifest), /hosted controller identity|controller fields/); + } + f.manifest.hosted = hosted; + f.manifest.campaignId = 'gha-12345-2'; + assert.throws(() => validateManifest(f.manifest), /hosted controller identity/); +}); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index 90d309b2545..ea362e4ce44 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -32,7 +32,7 @@ export async function readJson(file, maximum = 1_048_576) { } export function validateManifest(value, now = Date.now(), { admission = true } = {}) { fields(value, ['schema', 'campaignId', 'maxUsd', 'deadlineAt', 'pricingVerifiedAt', 'pricingExpiresAt', 'overhead', - 'sourceRevision', 'definitionsRevision', 'runnerCli', 'runnerPackage', 'runnerd', 'outputDirectory', 'route', 'projectId', 'models', 'attempts', 'observability']); + 'sourceRevision', 'definitionsRevision', 'runnerCli', 'runnerPackage', 'runnerd', 'outputDirectory', 'route', 'projectId', 'models', 'attempts', 'observability', 'hosted']); const deadline = Date.parse(value.deadlineAt); const verified = Date.parse(value.pricingVerifiedAt), expires = Date.parse(value.pricingExpiresAt); if (value.schema !== 'paperclip.runner-eval.controller/v1' || !id(value.campaignId) @@ -46,6 +46,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } = fields(artifact, ['path', 'sha256']); if (!isAbsolute(artifact.path ?? '') || !hash(artifact.sha256)) throw new Error('Invalid controller executable provenance'); } + validateHostedIdentity(value); const overhead = costRuntime.validateOperatingOverhead(value.overhead, value.maxUsd); if (admission && Date.parse(value.overhead.verifiedAt) > now || Date.parse(value.overhead.expiresAt) <= deadline || value.overhead.components.some(component => !component.provisionalBound)) throw new Error('Controller requires current full-lifecycle operating bounds'); @@ -82,6 +83,33 @@ export function validateManifest(value, now = Date.now(), { admission = true } = validateObservability(value); return { manifest: structuredClone(value), manifestSha256: sha256(canonical(value)), allocatedUsd: usd(allocated), overheadUsd: usd(overhead) }; } +function validateHostedIdentity(manifest) { + const hosted = manifest.hosted; + if (hosted === undefined) return; + fields(hosted, ['repository', 'runId', 'runAttempt', 'workflowRevision']); + if (typeof hosted.repository !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}\/[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$/.test(hosted.repository) + || typeof hosted.runId !== 'string' || !/^[1-9][0-9]{0,19}$/.test(hosted.runId) + || hosted.runAttempt !== 1 || typeof hosted.workflowRevision !== 'string' || !/^[a-f0-9]{40}$/.test(hosted.workflowRevision) + || manifest.campaignId !== `gha-${hosted.runId}-1`) throw new Error('Invalid hosted controller identity'); +} + +export function assertHostedAdmission(manifest, attempt, environment) { + validateHostedIdentity(manifest); + const hosted = manifest.hosted; + if (!hosted) { + if (['GITHUB_ACTIONS', 'GITHUB_REPOSITORY', 'GITHUB_RUN_ID', 'GITHUB_RUN_ATTEMPT'].some(key => environment[key] !== undefined)) { + throw new Error('GitHub execution requires a hosted controller identity'); + } + return; + } + if (environment.GITHUB_ACTIONS !== 'true' || environment.GITHUB_REPOSITORY !== hosted.repository + || environment.GITHUB_RUN_ID !== hosted.runId || environment.GITHUB_RUN_ATTEMPT !== '1' + || environment.GITHUB_SHA !== hosted.workflowRevision + || environment.PAPERCLIP_RUNNER_EVAL_EXECUTION_ID !== attempt.executionId) { + throw new Error('Hosted controller admission does not match the current first-attempt worker'); + } +} + export function bindRequest(manifest, request) { const attempt = manifest.attempts.find(item => item.attemptId === request.attemptId); if (!attempt || requestBindingSha256(request) !== attempt.requestSha256 || request.provider !== attempt.provider diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index 6191860bb64..1308c5388e2 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -73,11 +73,11 @@ test('copied controller runs without source dependencies, retains native evidenc console.log('stdin imports are side-effect free'); `); assert.match((await stdin).stdout, /stdin imports are side-effect free/); - const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export'; + const cases = 'real child requests|disjoint concurrent|controller owns native|cancellation during root export|hosted workers reject reruns'; const behavioral = await exec(process.execPath, ['--test', '--test-reporter=tap', '--test-name-pattern', cases, join(source, 'controller.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 80_000, maxBuffer: 1_048_576, }); - assert.match(behavioral.stdout, /# pass 5\b/); + assert.match(behavioral.stdout, /# pass 6\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { cwd: root, env: environment, timeout: 10_000, }).then(value => value, error => error); diff --git a/packages/paperclip-runner/scripts/controlled-evals/run.mjs b/packages/paperclip-runner/scripts/controlled-evals/run.mjs index 2ac16dc9ae1..631ae67854b 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/run.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/run.mjs @@ -6,7 +6,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import { parseArgs } from 'node:util'; import { startControlledObservability } from './observability.mjs'; -import { bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs'; +import { assertHostedAdmission, bindRequest, canonical, costRuntime, readJson, sha256, validateManifest } from './manifest.mjs'; async function atomicJson(file, value) { const temporary = `${file}.tmp`; @@ -68,6 +68,7 @@ export async function runControlledEval({ manifestPath, requestPath, outputPath, signal?.throwIfAborted(); const { manifest, manifestSha256 } = validateManifest(await readJson(manifestPath)); const request = await readJson(requestPath, 8_388_608), attempt = bindRequest(manifest, request); + assertHostedAdmission(manifest, attempt, environment); try { await lstat(outputPath); throw new Error('Controlled output already exists'); } catch (error) { if (error.code !== 'ENOENT') throw error; } if (request.session.workingDirectory) {