PaperClipAI/tests/runner-e2e/scheduled-operations.test.ts

import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { expect, it } from "vitest";
import { prepareScheduledCampaign, scheduledCadence, scheduledManifest, scheduledMatrix, scheduledSelection, validateScheduledBudget } from "./scheduled-operations.js";
import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
import { runnerMatrix } from "./catalog.js";
import { evalRequestReservation } from "./eval-gateway.js";
import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js";

const overhead: EvalOperatingOverhead = { version: 1, reference: "fixture-only", runner: "github-hosted",
  verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",
  components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) };

const input = { cadence: "daily" as const, campaignId: "scheduled-fixture", ledgerPath: "/tmp/scheduled-fixture/ledger.jsonl",
  projectId: "11111111-1111-4111-8111-111111111111", now: Date.parse("2026-10-21T00:00:00Z"), runAttempt: "1", overhead };

it.each(["daily", "weekly"] as const)("runs the exact %s selection through one shared budget and catalog job", cadence => {
  const manifest = scheduledManifest({ ...input, cadence });
  const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix);
  expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort());
  expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxRequests: cadence === "daily" ? 30 : 60, maxConcurrent: 1, maxAttemptsPerExecution: 1 });
  expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000);
  expect(scheduledMatrix(cadence).include).toHaveLength(1);
  expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4);
  expect(manifest.models.every(model => Number(evalRequestReservation({ ...model, mode: "braintrust", apiKey: "fixture" })) <= Number(manifest.limits.maxUsd))).toBe(true);
});

it("does not reduce Codex's verified context ceiling to fit the daily allowance", () => {
  const codex = scheduledManifest({ ...input, cadence: "weekly" }).models.find(row => row.provider === "openai")!;
  expect(codex.maximumInputTokens).toBe(1_050_000);
  expect(Number(evalRequestReservation({ ...codex, mode: "braintrust", apiKey: "fixture" }))).toBeGreaterThan(20);
  expect(scheduledManifest(input).models.map(model => model.provider)).toEqual(["anthropic"]);
});

it.each(["2", "0", "", "01"])("refuses workflow attempt %s before creating a fresh allowance", runAttempt => {
  expect(() => scheduledManifest({ ...input, runAttempt })).toThrow("fresh allowance");
});

it.each(["2026-10-19T23:59:59Z", "2026-11-19T00:00:00Z"])("rejects pricing outside its reviewed window at %s", time => {
  expect(() => scheduledManifest({ ...input, now: Date.parse(time) })).toThrow("pricing needs review");
});

it("rejects unknown triggers and maps the reviewed schedule expressions", () => {
  expect(scheduledCadence("schedule", "47 8 * * *")).toBe("daily");
  expect(scheduledCadence("schedule", "47 9 * * 0")).toBe("weekly");
  expect(scheduledCadence("workflow_dispatch", "", "manual")).toBeUndefined();
  expect(() => scheduledCadence("schedule", "* * * * *")).toThrow();
  expect(() => scheduledCadence("pull_request", "", "daily")).toThrow();
});

it("fails closed before paid work if disabled, missing project identity, or preflight is repeated", async () => {
  const directory = await mkdtemp(path.join(tmpdir(), "eval-schedule-"));
  const env = { EVAL_EVENT: "schedule", EVAL_SCHEDULE: "47 8 * * *", EVAL_OPERATIONS_ENABLED: "true",
    GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,
    EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(overhead), EVAL_TEST_RUNNER: "ubuntu-latest",
    PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };
  try {
    await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATIONS_ENABLED: "false" }, input.now)).rejects.toThrow("not enabled");
    await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATING_OVERHEAD_JSON: "" }, input.now)).rejects.toThrow("Missing reviewed");
    await expect(prepareScheduledCampaign({ ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "" }, input.now)).rejects.toThrow("UUID");
    const prepared = await prepareScheduledCampaign(env, input.now);
    expect(prepared.manifest.limits.campaignId).toBe(prepared.campaignId);
    const forecast = JSON.parse(await readFile(path.join(path.dirname(prepared.manifestPath), "forecast.json"), "utf8"));
    expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.428650100", summedAttemptMs: 107286 });
    expect(forecast.measurements.historicalHostedSelection).toEqual({ modelUsd: "0.428650100", summedAttemptMs: 107286 });
    expect(forecast.measurements.hostedAttemptMs).toBeNull();
    expect(forecast.maxUsd).toBe("20");
    await expect(prepareScheduledCampaign(env, input.now)).rejects.toThrow("EEXIST");
  } finally { await rm(directory, { recursive: true, force: true }); }
});

it("rejects an overhead allowance that cannot leave room for a full selected request before emitting the matrix", () => {
  const expensive = structuredClone(overhead);
  expensive.components[0]!.maxUsd = "13";
  expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance");
  expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") });
});


it.each(["daily", "weekly"] as const)("prepares %s with its own reviewed operating allowance", async cadence => {
  const directory = await mkdtemp(path.join(tmpdir(), "eval-cadence-overhead-"));
  const allowance = (value: string): EvalOperatingOverhead => ({ ...overhead, reference: `fixture-${value}`,
    components: overhead.components.map(component => ({ ...component,
      maxUsd: component.component === "observability" ? value : component.component === "artifact_storage" ? "1" : "0" })) });
  const allowances = { daily: allowance("10"), weekly: allowance("20") };
  const env = { EVAL_EVENT: "workflow_dispatch", EVAL_CADENCE: cadence, EVAL_OPERATIONS_ENABLED: "true",
    GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory,
    EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(allowances), EVAL_TEST_RUNNER: "ubuntu-latest",
    PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId };
  try {
    await expect(prepareScheduledCampaign({ ...env,
      EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, [cadence]: undefined }) }, input.now)).rejects.toThrow();
    await expect(prepareScheduledCampaign({ ...env,
      EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify({ ...allowances, typo: allowances.daily }) }, input.now)).rejects.toThrow();
    const prepared = await prepareScheduledCampaign(env, input.now);
    expect(prepared.manifest.limits.overhead).toEqual(allowances[cadence]);
    expect(prepared.manifest.limits.maxUsd).toBe(cadence === "daily" ? "20" : "100");
    expect(prepared.manifest.executionIds).toEqual(scheduledSelection(cadence));
  } finally { await rm(directory, { recursive: true, force: true }); }
});