diff --git a/.github/scripts/runner-loss-rerun.mjs b/.github/scripts/runner-loss-rerun.mjs index f581cdbbca5..0384e1b0866 100644 --- a/.github/scripts/runner-loss-rerun.mjs +++ b/.github/scripts/runner-loss-rerun.mjs @@ -27,9 +27,10 @@ export const AGGREGATE_JOBS = new Set(["ci / verify", "ci / e2e"]); // would move the whole run to GitHub-hosted runners, so leave it to a human. export const GATE_JOB_PREFIX = "ci / gate / "; export const RERUN_CONCLUSIONS = new Set(["failure", "cancelled"]); -// A shutdown signal often reaches the check run only as this annotation; the -// signal itself is logged as an error line in the job log. -const CANCELLED_MESSAGE = "The operation was canceled."; +// A shutdown signal often reaches the check run only as one of these +// annotations; the signal itself is logged as an error line in the job log. +const CANCELLED_MESSAGES = new Set(["The operation was canceled.", "A task was canceled."]); +const isCancellation = (annotation) => CANCELLED_MESSAGES.has(annotation.message); export function createGithubClient({ token, fetcher = fetch, apiUrl = "https://api.github.com" }) { if (!token) throw new Error("GITHUB_TOKEN is required"); @@ -83,12 +84,12 @@ export async function classifyJob(job, { annotations, readLog }) { const isRunnerLoss = (annotation) => RUNNER_LOSS_PATTERN.test(annotation.message ?? ""); const match = failures.find(isRunnerLoss); if (match) return { lost: true, evidence: firstLine(match.message) }; - if (failures.some((annotation) => annotation.message === CANCELLED_MESSAGE) && logShowsRunnerLoss(await readLog(job))) { + if (failures.some(isCancellation) && logShowsRunnerLoss(await readLog(job))) { return { lost: true, evidence: "job log: runner received a shutdown signal or lost communication" }; } - const real = failures.find((annotation) => annotation.message !== CANCELLED_MESSAGE); + const real = failures.find((annotation) => !isCancellation(annotation)); if (real) return { lost: false, evidence: firstLine(real.message) }; - if (failures.length > 0) return { lost: false, evidence: `${CANCELLED_MESSAGE} (no runner-loss error in the job log)` }; + if (failures.length > 0) return { lost: false, evidence: `${firstLine(failures[0].message)} (no runner-loss error in the job log)` }; return { lost: false, evidence: "no failure annotation" }; } diff --git a/.github/scripts/tests/runner-loss-rerun.test.mjs b/.github/scripts/tests/runner-loss-rerun.test.mjs index 819f160881e..4873c508ad5 100644 --- a/.github/scripts/tests/runner-loss-rerun.test.mjs +++ b/.github/scripts/tests/runner-loss-rerun.test.mjs @@ -344,6 +344,18 @@ test("the client refuses to follow pagination to another host", async () => { await assert.rejects(client.paginate("/repos/paperclipai/paperclip/commits/x/pulls"), /Refusing to call https:\/\/example.com/); }); +test("a step cancelled as \"A task was canceled.\" counts as lost only with shutdown evidence", async () => { + const annotations = [{ annotation_level: "failure", message: "A task was canceled." }]; + const lost = await classifyJob({ id: 1 }, { + annotations, + readLog: async () => "##[error]The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.\n##[error]A task was canceled.\n", + }); + assert.equal(lost.lost, true); + const noSignal = await classifyJob({ id: 1 }, { annotations, readLog: async () => "##[error]A task was canceled.\n" }); + assert.equal(noSignal.lost, false); + assert.match(noSignal.evidence, /A task was canceled\./); +}); + test("classification is conservative about cancelled jobs", async () => { const cancelled = [{ annotation_level: "failure", message: "The operation was canceled." }]; const noSignal = await classifyJob({ id: 1 }, { annotations: cancelled, readLog: async () => "##[error]The operation was canceled.\n" }); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 24a26f1d11f..e07b05ba2dc 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -195,8 +195,8 @@ failed jobs of a PR's first CI attempt once when every failed or cancelled job lost its runner. A spot reclaim often ends the job, and the run, as `cancelled`. A lost runner is recognized by the check-run annotation "lost communication with the server" or "received a shutdown signal". When a job has -the annotation "The operation was canceled.", the job log is checked for the -same messages. This evidence counts even if the job also has other error +the annotation "The operation was canceled." or "A task was canceled.", the job +log is checked for the same messages. This evidence counts even if the job also has other error annotations, so a test that really failed just before a reclaim gets one wasted re-run. A cancelled job without this evidence, such as a job a person cancelled, counts as a real failure. The `verify` and `e2e` aggregates are not