From e6df84a9928edabcad760e409c8f7f69ce58aa3b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 19:41:10 -0600 Subject: [PATCH] fix(ci): count a step cancelled as "A task was canceled." as runner loss when the log shows the shutdown (#21514) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - PR CI runs on AWS spot runners, and `ci-runner-loss-rerun.yml` re-runs a PR's first attempt once when every failed job lost its runner (#21291) > - On 2026-11-13 a watcher was added to the runner image that stops the GitHub runner about 10 seconds after AWS announces a spot reclaim, so the lost job ends in about 28 seconds instead of up to 10 minutes > - When the runner is stopped this way, the cancelled step's check annotation is usually "The operation was canceled.", but a step that was still starting up gets "A task was canceled." instead > - The classifier only recognised the first text, so it counted such a job as a real failure and refused the re-run, even though the job log has the shutdown-signal line > - This pull request recognises both cancellation texts, still only together with shutdown evidence in the log > - The benefit is that lost jobs stopped early in a step are re-run automatically too ## Linked Issues or Issue Description Refs #21291 **What happened?** In the first hour after the spot-notice watcher went live, a `ci / Build` job was stopped 13 s after it started. Its job log has `##[error]The runner has received a shutdown signal...`, but its only failure annotation was `A task was canceled.`. `classifyJob` treated that annotation as a real failure, so the whole run was not re-run. **Expected behavior** A job whose log shows the shutdown signal is counted as runner loss, whichever cancellation text the step was given. **Actual behavior** It is counted as a real failure, and the run stays red until a person re-runs it. ## What Changed - `.github/scripts/runner-loss-rerun.mjs`: the cancellation annotation check accepts "The operation was canceled." and "A task was canceled.". The log check still decides. A cancellation with no shutdown or lost-communication line in the log is still a real failure, and its evidence now quotes the actual annotation text. - `.github/scripts/tests/runner-loss-rerun.test.mjs`: a test for "A task was canceled." with and without shutdown evidence. - `doc/DEVELOPING.md`: the runner-loss paragraph names both texts. ## Verification - Red: the new test fails against the unchanged script (25 pass, 1 fail). - Green: `node --test .github/scripts/tests/runner-loss-rerun.test.mjs`: 26 pass. - `node --test '.github/scripts/tests/*.test.mjs'` (the `policy` job's command): 808 pass. - The workflow runs from `master`, so this PR's own CI cannot exercise the new behaviour; the tests are the evidence. ## Risks - A job cancelled as "A task was canceled." for another reason is still treated as a real failure unless its log has the shutdown or lost-communication line, so a person cancelling a run still blocks the automatic re-run. - No change to triggers, permissions, the once-only rule or the stale-head checks. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code, with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .github/scripts/runner-loss-rerun.mjs | 13 +++++++------ .github/scripts/tests/runner-loss-rerun.test.mjs | 12 ++++++++++++ doc/DEVELOPING.md | 4 ++-- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/scripts/runner-loss-rerun.mjs b/.github/scripts/runner-loss-rerun.mjs index f581cdbbca5..0384e1b0866 100644 --- a/.github/scripts/runner-loss-rerun.mjs +++ b/.github/scripts/runner-loss-rerun.mjs @@ -27,9 +27,10 @@ export const AGGREGATE_JOBS = new Set(["ci / verify", "ci / e2e"]); // would move the whole run to GitHub-hosted runners, so leave it to a human. export const GATE_JOB_PREFIX = "ci / gate / "; export const RERUN_CONCLUSIONS = new Set(["failure", "cancelled"]); -// A shutdown signal often reaches the check run only as this annotation; the -// signal itself is logged as an error line in the job log. -const CANCELLED_MESSAGE = "The operation was canceled."; +// A shutdown signal often reaches the check run only as one of these +// annotations; the signal itself is logged as an error line in the job log. +const CANCELLED_MESSAGES = new Set(["The operation was canceled.", "A task was canceled."]); +const isCancellation = (annotation) => CANCELLED_MESSAGES.has(annotation.message); export function createGithubClient({ token, fetcher = fetch, apiUrl = "https://api.github.com" }) { if (!token) throw new Error("GITHUB_TOKEN is required"); @@ -83,12 +84,12 @@ export async function classifyJob(job, { annotations, readLog }) { const isRunnerLoss = (annotation) => RUNNER_LOSS_PATTERN.test(annotation.message ?? ""); const match = failures.find(isRunnerLoss); if (match) return { lost: true, evidence: firstLine(match.message) }; - if (failures.some((annotation) => annotation.message === CANCELLED_MESSAGE) && logShowsRunnerLoss(await readLog(job))) { + if (failures.some(isCancellation) && logShowsRunnerLoss(await readLog(job))) { return { lost: true, evidence: "job log: runner received a shutdown signal or lost communication" }; } - const real = failures.find((annotation) => annotation.message !== CANCELLED_MESSAGE); + const real = failures.find((annotation) => !isCancellation(annotation)); if (real) return { lost: false, evidence: firstLine(real.message) }; - if (failures.length > 0) return { lost: false, evidence: `${CANCELLED_MESSAGE} (no runner-loss error in the job log)` }; + if (failures.length > 0) return { lost: false, evidence: `${firstLine(failures[0].message)} (no runner-loss error in the job log)` }; return { lost: false, evidence: "no failure annotation" }; } diff --git a/.github/scripts/tests/runner-loss-rerun.test.mjs b/.github/scripts/tests/runner-loss-rerun.test.mjs index 819f160881e..4873c508ad5 100644 --- a/.github/scripts/tests/runner-loss-rerun.test.mjs +++ b/.github/scripts/tests/runner-loss-rerun.test.mjs @@ -344,6 +344,18 @@ test("the client refuses to follow pagination to another host", async () => { await assert.rejects(client.paginate("/repos/paperclipai/paperclip/commits/x/pulls"), /Refusing to call https:\/\/example.com/); }); +test("a step cancelled as \"A task was canceled.\" counts as lost only with shutdown evidence", async () => { + const annotations = [{ annotation_level: "failure", message: "A task was canceled." }]; + const lost = await classifyJob({ id: 1 }, { + annotations, + readLog: async () => "##[error]The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.\n##[error]A task was canceled.\n", + }); + assert.equal(lost.lost, true); + const noSignal = await classifyJob({ id: 1 }, { annotations, readLog: async () => "##[error]A task was canceled.\n" }); + assert.equal(noSignal.lost, false); + assert.match(noSignal.evidence, /A task was canceled\./); +}); + test("classification is conservative about cancelled jobs", async () => { const cancelled = [{ annotation_level: "failure", message: "The operation was canceled." }]; const noSignal = await classifyJob({ id: 1 }, { annotations: cancelled, readLog: async () => "##[error]The operation was canceled.\n" }); diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 24a26f1d11f..e07b05ba2dc 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -195,8 +195,8 @@ failed jobs of a PR's first CI attempt once when every failed or cancelled job lost its runner. A spot reclaim often ends the job, and the run, as `cancelled`. A lost runner is recognized by the check-run annotation "lost communication with the server" or "received a shutdown signal". When a job has -the annotation "The operation was canceled.", the job log is checked for the -same messages. This evidence counts even if the job also has other error +the annotation "The operation was canceled." or "A task was canceled.", the job +log is checked for the same messages. This evidence counts even if the job also has other error annotations, so a test that really failed just before a reclaim gets one wasted re-run. A cancelled job without this evidence, such as a job a person cancelled, counts as a real failure. The `verify` and `e2e` aggregates are not