From ed3f067dbd71bb1506eaed303e12e46707145e67 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:52:41 -0700 Subject: [PATCH] fix(evals): select operating allowance by cadence (#17326) ## Thinking Path > - Paperclip manages AI agents and their operating budgets. > - Product E2E runs exercise real agents through a bounded gateway. > - The daily and weekly selections share one repository configuration variable. > - Their different request limits need different operating allowances. > - The existing parser accepts only one allowance, which cannot represent both reviewed limits. > - This change selects the allowance for the actual cadence and retains all existing validation. ## Linked Issues or Issue Description Refs #17009. **What happened?** The daily selection needs an $11 operating reservation; the weekly selection needs $21. A shared $21 allowance cannot fit the $20 daily cap. A shared $11 allowance does not cover the reviewed weekly resource bound. A configuration containing both allowances was rejected before paid admission. **Expected behavior** Each cadence uses its own reviewed allowance. Missing or malformed selected allowances stop admission before paid work. **Steps to reproduce** Set the operating allowance JSON to an object with daily and weekly allowances. Prepare either campaign through prepareScheduledCampaign. Before this change, both calls reject the outer object. **Paperclip version or commit** 3bd709df8b83aa855dbf300dcc57383b03ec1c88. ## What Changed - Accept daily and weekly allowance entries in the existing configuration variable. - Select the actual cadence before validating its budget, runner and expiration. - Pass only the selected allowance through the existing workflow output. - Preserve single-allowance compatibility and document the configuration format. ## Verification - Reproduced both cadence failures before the fix: 2 failed, 12 passed. - Focused scheduled operations and operating overhead tests: 20 passed. - Eval TypeScript check passed. - Missing selected entries and unknown outer fields fail before creating a manifest. - Hosted CI passed at 1c62cc8aeb2bfa610bf1216b20a8a157d753b19e, including full typecheck, tests, build and browser shards. - Greptile and Paperclip reviews both rated the exact commit 5/5 with no actionable findings. ## Risks - A map with a missing selected cadence now fails admission. It never substitutes the other cadence. - Operators must still provide reviewed rates and resource bounds. This parser does not prove actual charges. - This PR does not change limits, credentials, workflow permissions or enable gates. ## Model Used - OpenAI GPT-6 in Codex, with reasoning, repository inspection, code execution and test tools. The exact backend model identifier and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- tests/runner-e2e/OPERATIONS.md | 8 +++++-- tests/runner-e2e/operating-overhead.ts | 12 +++++++--- tests/runner-e2e/scheduled-operations.test.ts | 23 +++++++++++++++++++ tests/runner-e2e/scheduled-operations.ts | 4 ++-- 4 files changed, 40 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 31b36ed856f..5ec1bc48902 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -375,8 +375,12 @@ maximum allowances, not forecasts of actual spending. This is an admission limit with retained reservations, not a reconciled invoice total or a provider-enforced billing limit. -Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed -allowance with `version: 1`, an evidence `reference`, the actual `runner` +Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to an object +with `daily` and `weekly` reviewed allowances. The catalog selects the allowance +for the actual trigger, then passes only that validated allowance to preparation. +A missing selected cadence or unknown outer field fails admission. A single +allowance remains supported when the same bounds fit both schedules. Each +allowance has `version: 1`, an evidence `reference`, the actual `runner` (`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more than 31 days apart. Its `components` array must contain exactly `ci_compute`, `artifact_storage`, `observability`, and `shared_subscription`. Each component diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts index 08396df0ef3..4f53cd7517e 100644 --- a/tests/runner-e2e/operating-overhead.ts +++ b/tests/runner-e2e/operating-overhead.ts @@ -40,10 +40,16 @@ export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd return reserved; } -export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead { +export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number, cadence?: "daily" | "weekly"): EvalOperatingOverhead { if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance"); - let value: EvalOperatingOverhead; - try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); } + let parsed: unknown; + try { parsed = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); } + if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && !("version" in parsed)) { + object(parsed, ["daily", "weekly"]); + if (!cadence) throw new Error("Missing operating overhead cadence"); + parsed = (parsed as Record)[cadence]; + } + const value = parsed as EvalOperatingOverhead; validateOperatingOverhead(value, totalUsd); if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance"); if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) { diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts index 72ebe6d4d10..a9856edd9cc 100644 --- a/tests/runner-e2e/scheduled-operations.test.ts +++ b/tests/runner-e2e/scheduled-operations.test.ts @@ -75,3 +75,26 @@ it("rejects an overhead allowance that cannot leave room for a full selected req expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance"); expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") }); }); + + +it.each(["daily", "weekly"] as const)("prepares %s with its own reviewed operating allowance", async cadence => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-cadence-overhead-")); + const allowance = (value: string): EvalOperatingOverhead => ({ ...overhead, reference: `fixture-${value}`, + components: overhead.components.map(component => ({ ...component, + maxUsd: component.component === "observability" ? value : component.component === "artifact_storage" ? "1" : "0" })) }); + const allowances = { daily: allowance("10"), weekly: allowance("20") }; + const env = { EVAL_EVENT: "workflow_dispatch", EVAL_CADENCE: cadence, EVAL_OPERATIONS_ENABLED: "true", + GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory, + EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(allowances), EVAL_TEST_RUNNER: "ubuntu-latest", + PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId }; + try { + await expect(prepareScheduledCampaign({ ...env, + EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, [cadence]: undefined }) }, input.now)).rejects.toThrow(); + await expect(prepareScheduledCampaign({ ...env, + EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, typo: allowances.daily }) }, input.now)).rejects.toThrow(); + const prepared = await prepareScheduledCampaign(env, input.now); + expect(prepared.manifest.limits.overhead).toEqual(allowances[cadence]); + expect(prepared.manifest.limits.maxUsd).toBe(cadence === "daily" ? "20" : "100"); + expect(prepared.manifest.executionIds).toEqual(scheduledSelection(cadence)); + } finally { await rm(directory, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts index 84bb950e085..f92c8ff135e 100644 --- a/tests/runner-e2e/scheduled-operations.ts +++ b/tests/runner-e2e/scheduled-operations.ts @@ -87,7 +87,7 @@ export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Dat const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled"); const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"), projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "", - overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) }); + overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now, cadence) }); await mkdir(directory, { recursive: true, mode: 0o700 }); const manifestPath = path.join(directory, "manifest.json"); await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" }); @@ -109,7 +109,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE); if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled"); const now = Date.now(); - const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now); + const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now, cadence); validateScheduledBudget({ cadence, overhead, now }); console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead })); } else if (command === "prepare") {