#!/usr/bin/env node /** * runner-loss-rerun.mjs * * Re-runs the failed jobs of a first PR CI attempt once, but only when every * failed job failed because its runner was lost. Runs from * .github/workflows/ci-runner-loss-rerun.yml on workflow_run. It reads GitHub * API metadata only and never touches the PR's code or artifacts. * * Local dry run against a real run (no rerun is requested): * GITHUB_TOKEN=$(gh auth token) node .github/scripts/runner-loss-rerun.mjs --run-id --dry-run */ import { appendFileSync, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; export const REPOSITORY = "paperclipai/paperclip"; export const REPOSITORY_ID = 1170821064; export const PR_WORKFLOW_PATH = ".github/workflows/pr.yml"; // The runner's own messages when an AWS instance disappears mid-job. export const RUNNER_LOSS_PATTERN = /lost communication with the server|received a shutdown signal/; // `verify` and `e2e` in pr-trusted.yml only assert their `needs` results. They // fail whenever a dependency failed and re-run with it. export const AGGREGATE_JOBS = new Set(["ci / verify", "ci / e2e"]); // ci-routing.yml checks the attempt's triggering actor. A bot-triggered attempt // would move the whole run to GitHub-hosted runners, so leave it to a human. export const GATE_JOB_PREFIX = "ci / gate / "; // A shutdown signal often reaches the check run only as this annotation; the // signal itself is logged as an error line in the job log. const CANCELLED_MESSAGE = "The operation was canceled."; export function createGithubClient({ token, fetcher = fetch, apiUrl = "https://api.github.com" }) { if (!token) throw new Error("GITHUB_TOKEN is required"); const origin = new URL(apiUrl).origin; const request = async (target, { method = "GET" } = {}) => { const url = new URL(target, apiUrl); if (url.origin !== origin) throw new Error(`Refusing to call ${url.origin}`); const response = await fetcher(url.href, { method, headers: { Accept: "application/vnd.github+json", Authorization: `Bearer ${token}`, "X-GitHub-Api-Version": "2022-11-28" }, signal: AbortSignal.timeout(60_000), }); if (!response.ok) throw new Error(`GitHub API ${method} ${url.pathname} failed (HTTP ${response.status})`); return response; }; return { json: async (target) => (await request(target)).json(), // Follows Link rel="next" until the last page. `field` selects the array // in wrapped responses such as { total_count, jobs }. paginate: async (target, field) => { const items = []; let totalCount; for (let next = target, pages = 0; next; pages += 1) { if (pages >= 100) throw new Error(`Too many pages for ${target}`); const response = await request(next); const body = await response.json(); const page = field ? body[field] : body; if (!Array.isArray(page)) throw new Error(`Unexpected page shape for ${target}`); items.push(...page); if (field && Number.isInteger(body.total_count)) totalCount = body.total_count; next = /<([^>]+)>;\s*rel="next"/.exec(response.headers.get("link") ?? "")?.[1]; } if (totalCount !== undefined && items.length !== totalCount) { throw new Error(`Incomplete listing for ${target}: ${items.length} of ${totalCount}`); } return items; }, // Job logs answer with a redirect to a signed download URL. text: async (target) => (await request(target)).text(), post: async (target) => { await request(target, { method: "POST" }); }, }; } export function logShowsRunnerLoss(log) { return log.split("\n").some((line) => line.includes("##[error]") && RUNNER_LOSS_PATTERN.test(line)); } /** Classifies one failed job. Logs are read only when annotations are inconclusive. */ export async function classifyJob(job, { annotations, readLog }) { const failures = annotations.filter((annotation) => annotation.annotation_level === "failure"); const isRunnerLoss = (annotation) => RUNNER_LOSS_PATTERN.test(annotation.message ?? ""); // A step can fail for a real reason before the runner is lost (for example a // test failure, then a loss during an always() upload step). Any failure // annotation other than runner loss or cancellation marks a real failure. const real = failures.find((annotation) => !isRunnerLoss(annotation) && annotation.message !== CANCELLED_MESSAGE); if (real) return { lost: false, evidence: firstLine(real.message) }; const match = failures.find(isRunnerLoss); if (match) return { lost: true, evidence: firstLine(match.message) }; // Only cancellations are left: the shutdown signal, if any, is in the log. if (failures.length > 0) { if (logShowsRunnerLoss(await readLog(job))) { return { lost: true, evidence: "job log: runner received a shutdown signal or lost communication" }; } return { lost: false, evidence: `${CANCELLED_MESSAGE} (no runner-loss error in the job log)` }; } return { lost: false, evidence: "no failure annotation" }; } function firstLine(message) { const line = String(message).split("\n")[0]; return line.length > 160 ? `${line.slice(0, 157)}...` : line; } export function eligibleRun(run) { if (run.repository?.full_name !== REPOSITORY || run.repository?.id !== REPOSITORY_ID) return "run is not in the canonical repository"; if (run.path !== PR_WORKFLOW_PATH) return `workflow is ${run.path}, not ${PR_WORKFLOW_PATH}`; if (run.event !== "pull_request") return `event is ${run.event}, not pull_request`; if (run.status !== "completed" || run.conclusion !== "failure") return `run is ${run.status}/${run.conclusion}, not completed/failure`; // One automatic re-run only. A higher attempt also means someone re-ran it. if (run.run_attempt !== 1) return `run is on attempt ${run.run_attempt}; only attempt 1 is re-run automatically`; return undefined; } /** * Re-running an older run joins the PR's `cancel-in-progress` concurrency * group and would cancel newer CI. Require the run to be the latest CI for the * PR's current head. */ export async function staleReason(client, run) { const pulls = await client.paginate(`/repos/${REPOSITORY}/commits/${run.head_sha}/pulls?per_page=100`); const open = pulls.filter((pr) => pr.state === "open" && pr.base?.repo?.id === REPOSITORY_ID && pr.head?.repo?.id === run.head_repository?.id && pr.head?.sha === run.head_sha); if (open.length === 0) return "no open pull request still has this run's head commit"; const runs = await client.paginate( `/repos/${REPOSITORY}/actions/workflows/pr.yml/runs?event=pull_request&head_sha=${run.head_sha}&per_page=100`, "workflow_runs", ); const newer = runs.find((candidate) => candidate.id > run.id && candidate.conclusion !== "skipped"); if (newer) return `a newer PR run (${newer.id}) exists for this head commit`; return undefined; } export async function decide(client, run) { const decision = { runId: run.id, attempt: run.run_attempt, rerun: false, jobs: [] }; const ineligible = eligibleRun(run); if (ineligible) return { ...decision, reason: ineligible }; const jobs = await client.paginate(`/repos/${REPOSITORY}/actions/runs/${run.id}/attempts/1/jobs?per_page=100`, "jobs"); const failed = jobs.filter((job) => job.conclusion === "failure" && !AGGREGATE_JOBS.has(job.name)); if (failed.length === 0) return { ...decision, reason: "no failed jobs other than the verify/e2e aggregates" }; for (const job of failed) { if (job.name.startsWith(GATE_JOB_PREFIX)) { decision.jobs.push({ id: job.id, name: job.name, lost: false, evidence: "routing gate failed; a bot re-run would lose AWS routing" }); continue; } const annotations = await client.paginate(`/repos/${REPOSITORY}/check-runs/${job.id}/annotations?per_page=100`); const readLog = (entry) => client.text(`/repos/${REPOSITORY}/actions/jobs/${entry.id}/logs`); decision.jobs.push({ id: job.id, name: job.name, ...(await classifyJob(job, { annotations, readLog })) }); } const real = decision.jobs.filter((job) => !job.lost); if (real.length) return { ...decision, reason: `${real.length} of ${failed.length} failed jobs did not lose their runner` }; const stale = await staleReason(client, run); if (stale) return { ...decision, reason: stale }; return { ...decision, rerun: true, reason: `all ${failed.length} failed jobs lost their runner` }; } export function renderSummary(decision, { dryRun = false } = {}) { const action = decision.rerun ? (dryRun ? "Would re-run failed jobs (dry run)" : "Re-ran failed jobs") : "No automatic re-run"; const lines = [ "### CI runner-loss re-run", "", `**${action}** for run ${decision.runId} attempt ${decision.attempt}: ${decision.reason}.`, "", ]; if (decision.jobs.length) { lines.push("| Job | Runner lost | Evidence |", "| --- | --- | --- |"); for (const job of decision.jobs) { lines.push(`| ${escapeCell(job.name)} | ${job.lost ? "yes" : "no"} | ${escapeCell(job.evidence)} |`); } lines.push(""); } return `${lines.join("\n")}\n`; } const escapeCell = (value) => String(value).replace(/\|/g, "\\|").replace(/[\r\n]+/g, " "); export async function main({ env = process.env, argv = process.argv.slice(2), fetcher = fetch, log = console.log } = {}) { const dryRun = argv.includes("--dry-run"); const argRunId = argv[argv.indexOf("--run-id") + 1]; const event = env.GITHUB_EVENT_PATH ? JSON.parse(readFileSync(env.GITHUB_EVENT_PATH, "utf8")) : undefined; const runId = argv.includes("--run-id") ? argRunId : event?.workflow_run?.id; if (!/^[1-9]\d*$/.test(String(runId))) throw new Error("Provide --run-id or a workflow_run event"); const client = createGithubClient({ token: env.GITHUB_TOKEN, fetcher }); // Read the run's current state rather than the event snapshot: its attempt // number is above 1 if anyone re-ran it after this event was queued. const run = await client.json(`/repos/${REPOSITORY}/actions/runs/${runId}`); const decision = await decide(client, run); if (decision.rerun && !dryRun) { await client.post(`/repos/${REPOSITORY}/actions/runs/${run.id}/rerun-failed-jobs`); } const summary = renderSummary(decision, { dryRun }); log(summary); if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, summary); return decision; } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { main().catch((error) => { console.error(error.message); process.exitCode = 1; }); }