diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs index 4a9f42cb95a..7a6f15dcc55 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs @@ -95,6 +95,7 @@ const safeAdmissionReasons = new Set([ 'Invalid operating overhead component', 'Invalid operating overhead fields', 'Invalid controller observability allocation', 'Observability capacity requires its operating bound evidence', 'Hosted manifest bytes changed', 'Cell is outside the controlled selection', 'Hosted controller admission does not match the current first-attempt worker', + 'Hosted controller requires GitHub-hosted operating origin', ]); export function hostedAdmissionDiagnostic(error) { if (error instanceof Error && safeAdmissionReasons.has(error.message)) return error.message; diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs index ba64159479d..89b14b2bb39 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs @@ -6,7 +6,7 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { fixture, success } from './controller-fixture.mjs'; -import { canonical, sha256 } from './manifest.mjs'; +import { canonical, sha256, validateManifest } from './manifest.mjs'; const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; const { hostedPaths, hostedPolicyConfiguration, prepareHostedManifest, verifyHostedCell, hostedAdmissionDiagnostic } = await import(new URL('./hosted.mjs', runtime)); const { packControlledAttempt, summarizeControlledManifest } = await import(new URL('./archive.mjs', runtime)); @@ -96,6 +96,25 @@ test('hosted preparation rejects selection drift, unsupported cells and unreview assert.throws(() => prepareHostedManifest(input), /budget/); }); +test('hosted admission rejects local and fleet origins while preserving retained manifests', async t => { + const c = await campaign(t); + c.input.policy.overhead.accountingReadiness = 'model_requests'; + for (const component of c.input.policy.overhead.components) delete component.provisionalBound; + delete c.input.policy.capacityBound; + const original = c.build().manifest; + for (const runner of ['local', 'aws-fleet']) { + const input = structuredClone(c.input); input.policy.overhead.runner = runner; + assert.throws(() => prepareHostedManifest(input), /GitHub-hosted operating origin/); + const retained = structuredClone(original); retained.overhead.runner = runner; + assert.equal(validateManifest(retained, Date.now(), { admission: false }).manifest.overhead.runner, runner); + await writeFile(c.paths.manifest, canonical(retained)); + await assert.rejects(verifyHostedCell({ manifestPath: c.paths.manifest, manifestSha256: sha256(canonical(retained)), + cellId: 'execution-0', environment: { ...c.environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' } }), /GitHub-hosted operating origin/); + } + const local = structuredClone(original); delete local.hosted; local.overhead.runner = 'local'; + assert.equal(validateManifest(local).manifest.overhead.runner, 'local'); +}); + test('worker preflight verifies identical manifest bytes and current cell/run before credentials', async t => { const c = await campaign(t), { manifest, manifestSha256 } = c.build(); await writeFile(c.paths.manifest, canonical(manifest)); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index 4149afe37da..c85cd3bd294 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -46,7 +46,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } = fields(artifact, ['path', 'sha256']); if (!isAbsolute(artifact.path ?? '') || !hash(artifact.sha256)) throw new Error('Invalid controller executable provenance'); } - validateHostedIdentity(value); + validateHostedIdentity(value, { admission }); const overhead = costRuntime.validateOperatingOverhead(value.overhead, value.maxUsd); if (admission && Date.parse(value.overhead.verifiedAt) > now || Date.parse(value.overhead.expiresAt) <= deadline || value.overhead.accountingReadiness !== 'model_requests' && value.overhead.components.some(component => !component.provisionalBound)) throw new Error('Controller requires current full-lifecycle operating bounds'); @@ -83,7 +83,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } = validateObservability(value); return { manifest: structuredClone(value), manifestSha256: sha256(canonical(value)), allocatedUsd: usd(allocated), overheadUsd: usd(overhead) }; } -function validateHostedIdentity(manifest) { +function validateHostedIdentity(manifest, { admission = true } = {}) { const hosted = manifest.hosted; if (hosted === undefined) return; fields(hosted, ['repository', 'runId', 'runAttempt', 'workflowRevision', 'policySha256', 'controllerLockSha256']); @@ -93,6 +93,7 @@ function validateHostedIdentity(manifest) { || hosted.controllerLockSha256 !== undefined && !hash(hosted.controllerLockSha256) || hosted.runAttempt !== 1 || typeof hosted.workflowRevision !== 'string' || !/^[a-f0-9]{40}$/.test(hosted.workflowRevision) || manifest.campaignId !== `gha-${hosted.runId}-1`) throw new Error('Invalid hosted controller identity'); + if (admission && manifest.overhead?.runner !== 'github-hosted') throw new Error('Hosted controller requires GitHub-hosted operating origin'); } export function assertHostedAdmission(manifest, attempt, environment) { diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index 334b6a0a7ff..9e1ef440986 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -92,7 +92,7 @@ test('copied controller runs without source dependencies, retains native evidenc const hosted = await exec(process.execPath, ['--test', '--test-reporter=tap', join(source, 'hosted.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 30000, maxBuffer: 1_048_576, }); - assert.match(hosted.stdout, /# pass 6\b/); + assert.match(hosted.stdout, /# pass 7\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 1a860869e37..b99c85ec149 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -512,6 +512,13 @@ existing Braintrust subscription with a reference to that policy and current plan evidence. Any newly incurred subscription or capacity charge needs its own incremental allocation; the existing fee must remain visible separately. +Unscheduled local controllers use `runner: "local"` to retain their actual +execution origin. The same component, cap, expiry and receipt rules apply. This +label supplies no default zero charge and releases no reservation. Scheduled +allowances still require `github-hosted`; a local allowance fails that gate. The +separate hosted Runner controller enforces the same origin when admitting work. +Historical manifest reads preserve their original label; they grant no new admission. + The catalog checks that this allowance matches the selected runner, covers at least the next four hours, and leaves room for a full conservative model-request reservation. This happens before the build and test runners start. The workflow passes diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts index 85bdeedf1a1..dd8b3a2a1cf 100644 --- a/tests/runner-e2e/operating-overhead.test.ts +++ b/tests/runner-e2e/operating-overhead.test.ts @@ -17,11 +17,11 @@ const currentAllowance = (time = Date.now()): EvalOperatingOverhead => ({ ...all const runner = "ubuntu-latest"; const modelIdentity = { executionId: "claude.local.fixture", attemptId: "attempt-1", phase: "agent" as const }; -it("keeps operating overhead inside the same cap through model completion and a journal restart", async () => { +it.each(["aws-fleet", "local"] as const)("keeps %s overhead inside the same cap through model completion and a journal restart", async runnerKind => { const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-")); const file = path.join(directory, "ledger.jsonl"); const limits: EvalSpendLimits = { campaignId: "fixture", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z", - maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() }; + maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: { ...currentAllowance(), runner: runnerKind } }; try { const ledger = await openEvalCostLedger(file, limits); try { @@ -89,6 +89,7 @@ it("requires fresh reviewed allowances tied to the actual CI runner before sched expect(() => scheduledOperatingOverhead(undefined, runner, "20", now)).toThrow("Missing reviewed"); expect(() => scheduledOperatingOverhead("not json", runner, "20", now)).toThrow("JSON"); expect(() => scheduledOperatingOverhead(JSON.stringify(allowance()), runner, "20", now)).toThrow("runner"); + expect(() => scheduledOperatingOverhead(JSON.stringify({ ...hosted, runner: "local" }), runner, "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, "unrecognized", "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-10-19T23:59:59Z"))).toThrow("needs review"); diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts index 40c57283fc5..90b4915a223 100644 --- a/tests/runner-e2e/operating-overhead.ts +++ b/tests/runner-e2e/operating-overhead.ts @@ -6,7 +6,7 @@ export interface EvalOperatingOverhead { version: 1; accountingReadiness?: "model_requests"; reference: string; - runner: "github-hosted" | "aws-fleet"; + runner: "github-hosted" | "aws-fleet" | "local"; verifiedAt: string; expiresAt: string; components: Array<{ component: typeof EVAL_OVERHEAD_COMPONENTS[number]; maxUsd: string; reference: string; provisionalBound?: EvalCostBoundEvidence }>; @@ -30,7 +30,7 @@ export function operatingReservationBound(allowance: EvalOperatingOverhead, comp export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint { object(value, ["version", "accountingReadiness", "reference", "runner", "verifiedAt", "expiresAt", "components"]); if (value.accountingReadiness !== undefined && value.accountingReadiness !== "model_requests") throw new Error("Invalid accounting readiness policy"); - if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner) + if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet", "local"].includes(value.runner) || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string" || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt)) || Date.parse(value.expiresAt) <= Date.parse(value.verifiedAt)