fix(evals): accept unassigned skipped GitHub jobs

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/OPERATIONS.md

@@ -435,7 +435,9 @@ The collector uses the existing `gh` authentication and only reads
 revision and repository, reads every returned job/artifact page up to 1,000
 records per inventory, and rejects changed or incomplete inventories. It retains
 job identities, summed job seconds, run-start delay, artifact sizes and expiration
-dates. It omits arbitrary job names, artifact names and upstream responses.
+dates. Skipped jobs with a null or zero runner ID count as not started and use
+zero VM time. Started jobs still require a numeric runner identity. It omits
+arbitrary job names, artifact names and upstream responses.
 Existing output files cannot be replaced. This is a current inventory, not an
 invoice: deleted artifacts are absent, compressed bytes are not billed
 byte-hours, job duration is not VM lifetime, and present repository visibility

tests/runner-e2e/github-operating-usage.test.ts

@@ -49,15 +49,17 @@ it("paginates all jobs and preserves nonstandard compute as unpriced", async ()
 it("rejects changing, truncated, duplicated, misattributed, rerun and uncompleted evidence", async () => {
   for (const override of [{ run: { run_attempt: 2 } }, { run: { status: "in_progress" } }, { run: { path: "other.yml" } },
     { finalRun: { head_sha: "b".repeat(40) } }, { jobPage: { total_count: 2 } }, { jobs: [job(), job()] },
-    { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } },
+    { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), runner_id: null }] },
+    { jobs: [{ ...job(), runner_id: undefined }] }, { jobs: [{ ...job(), conclusion: "skipped", runner_id: undefined }] },
+    { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } },
     { artifactPage: { artifacts: [{ ...artifact, workflow_run: { id: 999, head_sha: sha } }] } },
     { artifactPage: { artifacts: [{ ...artifact, size_in_bytes: -1 }] } }]) {
     await expect(readGitHubOperatingUsage("123", api(override).read, now)).rejects.toThrow();
   }
 });
 
-it("accounts skipped jobs without requiring a fictitious VM and keeps expired artifact identity", async () => {
-  const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: 0, started_at: null, completed_at: null }],
+it.each([0, null])("accounts skipped jobs with runner ID %s without requiring a fictitious VM", async (runnerId) => {
+  const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: runnerId, started_at: null, completed_at: null }],
     artifactPage: { artifacts: [{ ...artifact, expired: true }] } });
   expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 0,
     jobs: [{ runner: "not_started" }], artifacts: { retainedBytes: 0, inventory: [{ id: 9, expired: true }] } });

tests/runner-e2e/github-operating-usage.ts

@@ -42,9 +42,9 @@ export async function readGitHubOperatingUsage(runId: string, read: ReadGitHub,
   const rawJobs = await inventory(read, `${prefix}/actions/runs/${runId}/attempts/1/jobs`, "jobs");
   if (!rawJobs.length) throw new Error("Missing workflow jobs");
   const jobs = rawJobs.map(job => {
-    if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !integer(job.runner_id)
+    const skipped = job.conclusion === "skipped" && (job.runner_id === 0 || job.runner_id === null);
+    if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !skipped && !integer(job.runner_id)
         || !Array.isArray(job.labels) || job.labels.some((label: unknown) => typeof label !== "string")) throw new Error("Invalid workflow job identity");
-    const skipped = job.conclusion === "skipped" && job.runner_id === 0;
     if (!skipped && (!timestamp(job.started_at) || !timestamp(job.completed_at)
         || Date.parse(job.started_at) < Date.parse(run.created_at) || Date.parse(job.completed_at) > now
         || Date.parse(job.completed_at) < Date.parse(job.started_at))) throw new Error("Incomplete workflow job timing");