From cff331f6ca0a652b5e41641c68dfa5fa584c22ae Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 28 Oct 2026 02:05:36 -0500 Subject: [PATCH] fix(evals): retain local controller cost origin (#17990) ## Thinking Path > - Paperclip manages AI agents and their work. > - Its eval journals retain model costs and operating reservations under one cap. > - The operating policy records the controller's execution origin. > - The schema currently accepts only GitHub-hosted and AWS Fleet origins. > - A local controller must therefore use an inaccurate label or fail admission. > - This PR adds the local origin and keeps the hosted-only schedule gate. ## Linked Issues or Issue Description Refs #17986. No duplicate open PR or issue was found in the operating-overhead/local search. **What existing behavior does this improve?** The Product E2E operating allowance and its retained journal policy. **Current behavior** At master `40c1ab36cd`, an unscheduled local controller cannot set `runner: "local"`. The generic allowance validator rejects it. Local qualification can use the same accounting controls but cannot record its origin correctly. **Proposed behavior** Accept the local origin in the generic allowance. Keep all component, price, cap, expiry, and receipt rules. The scheduled validator still requires `github-hosted` and rejects a local allowance. **Reason and benefit** Local qualification can retain an accurate execution origin without claiming that its reservations are billed GitHub charges. ## What Changed - Add `local` to the operating allowance's runner type and validator. - Exercise local journal creation, model admission, retained overhead, restart, and reconciliation in the existing boundary test. - Check that a local allowance fails Product scheduled admission. - Pin new hosted Runner controller admission to `github-hosted`. Preserve original labels when reading historical manifests; those reads cannot authorize a new worker. - Exercise the hosted-origin rejection and historical/local reads in both source and copied portable runtimes. - Document the local label and its accounting limits. ## Verification - Before the fix: the new local persistence case and schedule-boundary assertion failed; eight tests passed. - Focused ledger, overhead, operations, and scheduled finalization tests: 71 passed across five files. - Controlled Runner source and portable tests: 86 passed. The new hosted-origin regression failed before the fix. - Full Product support suite at the first commit: 4,524 passed, three skipped, and one retained embedded PostgreSQL bootstrap failure. The same database failure was previously reproduced on a clean baseline. - Separate Node test phase on the host: 137 passed. - Actual Linux controller: database lifecycle tests pass. After installing its missing `zip` utility, the full Product suite has 4,527 passes and one Copilot parent-version coverage failure. That same failure reproduces on clean master `40c1ab36cd` with the same Linux filesystem and dependencies (14 passes, one failure). Copilot files are unchanged. The initial seven ZIP-related failures and the watcher failure remain retained. - The final fresh Linux semantic check passes: `GOMEMLIMIT=2GiB GOGC=50 GOMAXPROCS=1 node node_modules/typescript/bin/tsc --singleThreaded -p tests/runner-e2e/tsconfig.json`. The parallel compiler OOM is retained. No semantic checks were omitted. - Both fresh reviews are 5/5, with the addressed thread resolved. Hosted Build first failed during unchanged Pi setup prefetch with an operation timeout. One job rerun recovered; the original failure remains retained. All 58 latest-head checks are now acceptable. - No paid model requests, sandbox creation, schedule changes, or historical journal edits. ## Risks - The caller supplies this origin. It must describe the actual controller. - `local` does not prove zero compute cost, provide a default allowance, or release a hold. - Both maintained hosted admission paths require the GitHub-hosted controller origin. A previously accepted hosted policy with an AWS/local label now fails new admission. Historical reads preserve the original evidence. ## Model Used - OpenAI GPT-6 through Codex, with tool use and code execution. The exact serving model ID and context window are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --------- Co-authored-by: Paperclip --- .../scripts/controlled-evals/hosted.mjs | 1 + .../scripts/controlled-evals/hosted.test.mjs | 21 ++++++++++++++++++- .../scripts/controlled-evals/manifest.mjs | 5 +++-- .../controlled-evals/portable.test.mjs | 2 +- tests/runner-e2e/OPERATIONS.md | 7 +++++++ tests/runner-e2e/operating-overhead.test.ts | 5 +++-- tests/runner-e2e/operating-overhead.ts | 4 ++-- 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs index 4a9f42cb95a..7a6f15dcc55 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.mjs @@ -95,6 +95,7 @@ const safeAdmissionReasons = new Set([ 'Invalid operating overhead component', 'Invalid operating overhead fields', 'Invalid controller observability allocation', 'Observability capacity requires its operating bound evidence', 'Hosted manifest bytes changed', 'Cell is outside the controlled selection', 'Hosted controller admission does not match the current first-attempt worker', + 'Hosted controller requires GitHub-hosted operating origin', ]); export function hostedAdmissionDiagnostic(error) { if (error instanceof Error && safeAdmissionReasons.has(error.message)) return error.message; diff --git a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs index ba64159479d..89b14b2bb39 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/hosted.test.mjs @@ -6,7 +6,7 @@ import { cp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { fixture, success } from './controller-fixture.mjs'; -import { canonical, sha256 } from './manifest.mjs'; +import { canonical, sha256, validateManifest } from './manifest.mjs'; const runtime = process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME ? pathToFileURL(resolve(process.env.PAPERCLIP_CONTROLLER_TEST_RUNTIME) + '/') : import.meta.url; const { hostedPaths, hostedPolicyConfiguration, prepareHostedManifest, verifyHostedCell, hostedAdmissionDiagnostic } = await import(new URL('./hosted.mjs', runtime)); const { packControlledAttempt, summarizeControlledManifest } = await import(new URL('./archive.mjs', runtime)); @@ -96,6 +96,25 @@ test('hosted preparation rejects selection drift, unsupported cells and unreview assert.throws(() => prepareHostedManifest(input), /budget/); }); +test('hosted admission rejects local and fleet origins while preserving retained manifests', async t => { + const c = await campaign(t); + c.input.policy.overhead.accountingReadiness = 'model_requests'; + for (const component of c.input.policy.overhead.components) delete component.provisionalBound; + delete c.input.policy.capacityBound; + const original = c.build().manifest; + for (const runner of ['local', 'aws-fleet']) { + const input = structuredClone(c.input); input.policy.overhead.runner = runner; + assert.throws(() => prepareHostedManifest(input), /GitHub-hosted operating origin/); + const retained = structuredClone(original); retained.overhead.runner = runner; + assert.equal(validateManifest(retained, Date.now(), { admission: false }).manifest.overhead.runner, runner); + await writeFile(c.paths.manifest, canonical(retained)); + await assert.rejects(verifyHostedCell({ manifestPath: c.paths.manifest, manifestSha256: sha256(canonical(retained)), + cellId: 'execution-0', environment: { ...c.environment, PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: 'execution-0' } }), /GitHub-hosted operating origin/); + } + const local = structuredClone(original); delete local.hosted; local.overhead.runner = 'local'; + assert.equal(validateManifest(local).manifest.overhead.runner, 'local'); +}); + test('worker preflight verifies identical manifest bytes and current cell/run before credentials', async t => { const c = await campaign(t), { manifest, manifestSha256 } = c.build(); await writeFile(c.paths.manifest, canonical(manifest)); diff --git a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs index 4149afe37da..c85cd3bd294 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/manifest.mjs @@ -46,7 +46,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } = fields(artifact, ['path', 'sha256']); if (!isAbsolute(artifact.path ?? '') || !hash(artifact.sha256)) throw new Error('Invalid controller executable provenance'); } - validateHostedIdentity(value); + validateHostedIdentity(value, { admission }); const overhead = costRuntime.validateOperatingOverhead(value.overhead, value.maxUsd); if (admission && Date.parse(value.overhead.verifiedAt) > now || Date.parse(value.overhead.expiresAt) <= deadline || value.overhead.accountingReadiness !== 'model_requests' && value.overhead.components.some(component => !component.provisionalBound)) throw new Error('Controller requires current full-lifecycle operating bounds'); @@ -83,7 +83,7 @@ export function validateManifest(value, now = Date.now(), { admission = true } = validateObservability(value); return { manifest: structuredClone(value), manifestSha256: sha256(canonical(value)), allocatedUsd: usd(allocated), overheadUsd: usd(overhead) }; } -function validateHostedIdentity(manifest) { +function validateHostedIdentity(manifest, { admission = true } = {}) { const hosted = manifest.hosted; if (hosted === undefined) return; fields(hosted, ['repository', 'runId', 'runAttempt', 'workflowRevision', 'policySha256', 'controllerLockSha256']); @@ -93,6 +93,7 @@ function validateHostedIdentity(manifest) { || hosted.controllerLockSha256 !== undefined && !hash(hosted.controllerLockSha256) || hosted.runAttempt !== 1 || typeof hosted.workflowRevision !== 'string' || !/^[a-f0-9]{40}$/.test(hosted.workflowRevision) || manifest.campaignId !== `gha-${hosted.runId}-1`) throw new Error('Invalid hosted controller identity'); + if (admission && manifest.overhead?.runner !== 'github-hosted') throw new Error('Hosted controller requires GitHub-hosted operating origin'); } export function assertHostedAdmission(manifest, attempt, environment) { diff --git a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs index 334b6a0a7ff..9e1ef440986 100644 --- a/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs +++ b/packages/paperclip-runner/scripts/controlled-evals/portable.test.mjs @@ -92,7 +92,7 @@ test('copied controller runs without source dependencies, retains native evidenc const hosted = await exec(process.execPath, ['--test', '--test-reporter=tap', join(source, 'hosted.test.mjs')], { cwd: root, env: { ...environment, PAPERCLIP_CONTROLLER_TEST_RUNTIME: join(directory, 'controlled-evals') }, timeout: 30000, maxBuffer: 1_048_576, }); - assert.match(hosted.stdout, /# pass 6\b/); + assert.match(hosted.stdout, /# pass 7\b/); const cli = await exec(process.execPath, [join(directory, 'controlled-evals/archive.mjs'), 'report-attempts', join(root, 'empty-attempts')], { diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 1a860869e37..b99c85ec149 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -512,6 +512,13 @@ existing Braintrust subscription with a reference to that policy and current plan evidence. Any newly incurred subscription or capacity charge needs its own incremental allocation; the existing fee must remain visible separately. +Unscheduled local controllers use `runner: "local"` to retain their actual +execution origin. The same component, cap, expiry and receipt rules apply. This +label supplies no default zero charge and releases no reservation. Scheduled +allowances still require `github-hosted`; a local allowance fails that gate. The +separate hosted Runner controller enforces the same origin when admitting work. +Historical manifest reads preserve their original label; they grant no new admission. + The catalog checks that this allowance matches the selected runner, covers at least the next four hours, and leaves room for a full conservative model-request reservation. This happens before the build and test runners start. The workflow passes diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts index 85bdeedf1a1..dd8b3a2a1cf 100644 --- a/tests/runner-e2e/operating-overhead.test.ts +++ b/tests/runner-e2e/operating-overhead.test.ts @@ -17,11 +17,11 @@ const currentAllowance = (time = Date.now()): EvalOperatingOverhead => ({ ...all const runner = "ubuntu-latest"; const modelIdentity = { executionId: "claude.local.fixture", attemptId: "attempt-1", phase: "agent" as const }; -it("keeps operating overhead inside the same cap through model completion and a journal restart", async () => { +it.each(["aws-fleet", "local"] as const)("keeps %s overhead inside the same cap through model completion and a journal restart", async runnerKind => { const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-")); const file = path.join(directory, "ledger.jsonl"); const limits: EvalSpendLimits = { campaignId: "fixture", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z", - maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() }; + maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: { ...currentAllowance(), runner: runnerKind } }; try { const ledger = await openEvalCostLedger(file, limits); try { @@ -89,6 +89,7 @@ it("requires fresh reviewed allowances tied to the actual CI runner before sched expect(() => scheduledOperatingOverhead(undefined, runner, "20", now)).toThrow("Missing reviewed"); expect(() => scheduledOperatingOverhead("not json", runner, "20", now)).toThrow("JSON"); expect(() => scheduledOperatingOverhead(JSON.stringify(allowance()), runner, "20", now)).toThrow("runner"); + expect(() => scheduledOperatingOverhead(JSON.stringify({ ...hosted, runner: "local" }), runner, "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, "unrecognized", "20", now)).toThrow("runner"); expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-10-19T23:59:59Z"))).toThrow("needs review"); diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts index 40c57283fc5..90b4915a223 100644 --- a/tests/runner-e2e/operating-overhead.ts +++ b/tests/runner-e2e/operating-overhead.ts @@ -6,7 +6,7 @@ export interface EvalOperatingOverhead { version: 1; accountingReadiness?: "model_requests"; reference: string; - runner: "github-hosted" | "aws-fleet"; + runner: "github-hosted" | "aws-fleet" | "local"; verifiedAt: string; expiresAt: string; components: Array<{ component: typeof EVAL_OVERHEAD_COMPONENTS[number]; maxUsd: string; reference: string; provisionalBound?: EvalCostBoundEvidence }>; @@ -30,7 +30,7 @@ export function operatingReservationBound(allowance: EvalOperatingOverhead, comp export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint { object(value, ["version", "accountingReadiness", "reference", "runner", "verifiedAt", "expiresAt", "components"]); if (value.accountingReadiness !== undefined && value.accountingReadiness !== "model_requests") throw new Error("Invalid accounting readiness policy"); - if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner) + if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet", "local"].includes(value.runner) || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string" || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt)) || Date.parse(value.expiresAt) <= Date.parse(value.verifiedAt)