fix(evals): select operating allowance by cadence

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/OPERATIONS.md

@@ -375,8 +375,12 @@ maximum allowances, not forecasts of actual spending.
 This is an admission limit with retained reservations, not a reconciled invoice
 total or a provider-enforced billing limit.
 
-Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed
-allowance with `version: 1`, an evidence `reference`, the actual `runner`
+Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to an object
+with `daily` and `weekly` reviewed allowances. The catalog selects the allowance
+for the actual trigger, then passes only that validated allowance to preparation.
+A missing selected cadence or unknown outer field fails admission. A single
+allowance remains supported when the same bounds fit both schedules. Each
+allowance has `version: 1`, an evidence `reference`, the actual `runner`
 (`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more
 than 31 days apart. Its `components` array must contain exactly `ci_compute`,
 `artifact_storage`, `observability`, and `shared_subscription`. Each component

tests/runner-e2e/operating-overhead.ts

@@ -40,10 +40,16 @@ export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd
   return reserved;
 }
 
-export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead {
+export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number, cadence?: "daily" | "weekly"): EvalOperatingOverhead {
   if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance");
-  let value: EvalOperatingOverhead;
-  try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }
+  let parsed: unknown;
+  try { parsed = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); }
+  if (parsed && typeof parsed === "object" && !Array.isArray(parsed) && !("version" in parsed)) {
+    object(parsed, ["daily", "weekly"]);
+    if (!cadence) throw new Error("Missing operating overhead cadence");
+    parsed = (parsed as Record<string, unknown>)[cadence];
+  }
+  const value = parsed as EvalOperatingOverhead;
   validateOperatingOverhead(value, totalUsd);
   if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance");
   if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) {

tests/runner-e2e/scheduled-operations.test.ts

@@ -75,3 +75,26 @@ it("rejects an overhead allowance that cannot leave room for a full selected req
   expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");
   expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });
 });
+
+
+it.each(["daily", "weekly"] as const)("prepares %s with its own reviewed operating allowance", async cadence => {
+  const directory = await mkdtemp(path.join(tmpdir(), "eval-cadence-overhead-"));
+  const allowance = (value: string): EvalOperatingOverhead => ({ ...overhead, reference: `fixture-${value}`,
+    components: overhead.components.map(component => ({ ...component,
+      maxUsd: component.component === "observability" ? value : component.component === "artifact_storage" ? "1" : "0" })) });
+  const allowances = { daily: allowance("10"), weekly: allowance("20") };
+  const env = { EVAL_EVENT: "workflow_dispatch", EVAL_CADENCE: cadence, EVAL_OPERATIONS_ENABLED: "true",
+    GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,
+    EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(allowances), EVAL_TEST_RUNNER: "ubuntu-latest",
+    PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };
+  try {
+    await expect(prepareScheduledCampaign({ ...env,
+      EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, [cadence]: undefined }) }, input.now)).rejects.toThrow();
+    await expect(prepareScheduledCampaign({ ...env,
+      EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, typo: allowances.daily }) }, input.now)).rejects.toThrow();
+    const prepared = await prepareScheduledCampaign(env, input.now);
+    expect(prepared.manifest.limits.overhead).toEqual(allowances[cadence]);
+    expect(prepared.manifest.limits.maxUsd).toBe(cadence === "daily" ? "20" : "100");
+    expect(prepared.manifest.executionIds).toEqual(scheduledSelection(cadence));
+  } finally { await rm(directory, { recursive: true, force: true }); }
+});

tests/runner-e2e/scheduled-operations.ts

@@ -87,7 +87,7 @@ export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Dat
   const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled");
   const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"),
     projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "",
-    overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) });
+    overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now, cadence) });
   await mkdir(directory, { recursive: true, mode: 0o700 });
   const manifestPath = path.join(directory, "manifest.json");
   await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" });
@@ -109,7 +109,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
     const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE);
     if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled");
     const now = Date.now();
-    const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now);
+    const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now, cadence);
     validateScheduledBudget({ cadence, overhead, now });
     console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead }));
   } else if (command === "prepare") {