From 59d1b7b43d19c8aedce78fd9d9ab4b215d227ac2 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:35:35 -0700 Subject: [PATCH] fix(evals): collect usage when GitHub skips jobs without a runner (#17352) ## Thinking Path > - Paperclip manages AI agents and their operating budgets. > - Product E2E runs retain accounting for each campaign. > - The usage collector reads completed GitHub jobs without rerunning paid work. > - GitHub returns a null runner ID for jobs that it skips before assigning a runner. > - The collector rejected those jobs and could not retain the completed campaign inventory. > - This change accepts that skipped-job representation while preserving started-job checks. ## Linked Issues or Issue Description Refs #17009. **What happened?** The usage collector rejected completed qualification run 39971835384. GitHub returned skipped jobs with null runner IDs. The collector accepted only zero for jobs without a runner. **Expected behavior** Retain completed run usage when skipped jobs have a null or zero runner ID. Do not count VM time for those jobs. Continue to reject missing runner identity for started jobs. **Steps to reproduce** Read the completed first-attempt Product E2E run with operating-receipts.ts github. Before this fix, a skipped job with runner_id set to null throws Invalid workflow job identity. **Paperclip version or commit** ed3f067dbd71bb1506eaed303e12e46707145e67. ## What Changed - Accept a null or zero runner ID only for completed skipped jobs. - Keep run, attempt, labels and started-job identity validation. - Test both skipped-job forms and reject absent runner IDs. - Document the treatment of skipped jobs. ## Verification - Reproduced the observed null-runner failure: 1 failed, 4 passed. - Usage collector and operating receipt tests: 17 passed. - Eval TypeScript check passed. - Read the same completed hosted run successfully without dispatching or retrying tests. The inventory retained 1,303 job seconds and 290,156,153 artifact bytes. - Full hosted CI passed at fb57826c22dd98d9fcdd4ea55b32b6f0a5ec9796, including typecheck, tests, build and browser shards. - Greptile and Paperclip reviews rated the exact commit 5/5 with no actionable findings. ## Risks - The exception applies only when GitHub reports a completed skipped job and an explicit null or zero runner ID. - Current usage remains an inventory. This change does not convert it to a bill, release cost holds, or enable recurrence. ## Model Used - OpenAI GPT-6 in Codex, with reasoning, repository inspection, code execution and test tools. The exact backend model identifier and context-window size are not exposed in this session. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- tests/runner-e2e/OPERATIONS.md | 4 +++- tests/runner-e2e/github-operating-usage.test.ts | 8 +++++--- tests/runner-e2e/github-operating-usage.ts | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 5ec1bc48902..a51bd022762 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -435,7 +435,9 @@ The collector uses the existing `gh` authentication and only reads revision and repository, reads every returned job/artifact page up to 1,000 records per inventory, and rejects changed or incomplete inventories. It retains job identities, summed job seconds, run-start delay, artifact sizes and expiration -dates. It omits arbitrary job names, artifact names and upstream responses. +dates. Skipped jobs with a null or zero runner ID count as not started and use +zero VM time. Started jobs still require a numeric runner identity. It omits +arbitrary job names, artifact names and upstream responses. Existing output files cannot be replaced. This is a current inventory, not an invoice: deleted artifacts are absent, compressed bytes are not billed byte-hours, job duration is not VM lifetime, and present repository visibility diff --git a/tests/runner-e2e/github-operating-usage.test.ts b/tests/runner-e2e/github-operating-usage.test.ts index dac23f7ba35..fa4cd125429 100644 --- a/tests/runner-e2e/github-operating-usage.test.ts +++ b/tests/runner-e2e/github-operating-usage.test.ts @@ -49,15 +49,17 @@ it("paginates all jobs and preserves nonstandard compute as unpriced", async () it("rejects changing, truncated, duplicated, misattributed, rerun and uncompleted evidence", async () => { for (const override of [{ run: { run_attempt: 2 } }, { run: { status: "in_progress" } }, { run: { path: "other.yml" } }, { finalRun: { head_sha: "b".repeat(40) } }, { jobPage: { total_count: 2 } }, { jobs: [job(), job()] }, - { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } }, + { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), runner_id: null }] }, + { jobs: [{ ...job(), runner_id: undefined }] }, { jobs: [{ ...job(), conclusion: "skipped", runner_id: undefined }] }, + { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } }, { artifactPage: { artifacts: [{ ...artifact, workflow_run: { id: 999, head_sha: sha } }] } }, { artifactPage: { artifacts: [{ ...artifact, size_in_bytes: -1 }] } }]) { await expect(readGitHubOperatingUsage("123", api(override).read, now)).rejects.toThrow(); } }); -it("accounts skipped jobs without requiring a fictitious VM and keeps expired artifact identity", async () => { - const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: 0, started_at: null, completed_at: null }], +it.each([0, null])("accounts skipped jobs with runner ID %s without requiring a fictitious VM", async (runnerId) => { + const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: runnerId, started_at: null, completed_at: null }], artifactPage: { artifacts: [{ ...artifact, expired: true }] } }); expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 0, jobs: [{ runner: "not_started" }], artifacts: { retainedBytes: 0, inventory: [{ id: 9, expired: true }] } }); diff --git a/tests/runner-e2e/github-operating-usage.ts b/tests/runner-e2e/github-operating-usage.ts index e55ead9a50e..6655f400de2 100644 --- a/tests/runner-e2e/github-operating-usage.ts +++ b/tests/runner-e2e/github-operating-usage.ts @@ -42,9 +42,9 @@ export async function readGitHubOperatingUsage(runId: string, read: ReadGitHub, const rawJobs = await inventory(read, `${prefix}/actions/runs/${runId}/attempts/1/jobs`, "jobs"); if (!rawJobs.length) throw new Error("Missing workflow jobs"); const jobs = rawJobs.map(job => { - if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !integer(job.runner_id) + const skipped = job.conclusion === "skipped" && (job.runner_id === 0 || job.runner_id === null); + if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !skipped && !integer(job.runner_id) || !Array.isArray(job.labels) || job.labels.some((label: unknown) => typeof label !== "string")) throw new Error("Invalid workflow job identity"); - const skipped = job.conclusion === "skipped" && job.runner_id === 0; if (!skipped && (!timestamp(job.started_at) || !timestamp(job.completed_at) || Date.parse(job.started_at) < Date.parse(run.created_at) || Date.parse(job.completed_at) > now || Date.parse(job.completed_at) < Date.parse(job.started_at))) throw new Error("Incomplete workflow job timing");