diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e1b3e712cd9..e61d8ad3077 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -85,6 +85,7 @@ jobs: permissions: contents: read actions: write + checks: read # runner-loss evidence (annotations) for readiness retries outputs: verified: ${{ steps.watch.outputs.verified }} steps: @@ -494,6 +495,7 @@ jobs: permissions: contents: read actions: write # restart only exact-source producers deferred before admission + checks: read # runner-loss evidence (annotations) for readiness retries steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 with: diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 04f31734e21..e8a2df28761 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -249,7 +249,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/cloud-readiness-runner-loss.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d9bdc1824f5..f64604741c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -322,10 +322,14 @@ jobs: name: Reuse exact-source verification if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest - timeout-minutes: 70 + # The source wait allows 90 minutes: a deferred full readiness run plus two + # retries of jobs whose Spot runner was reclaimed. + timeout-minutes: 100 permissions: contents: read + # Also re-runs readiness jobs whose runner was lost, never a failed check. actions: write # restart only an exact-source push deferred before admission + checks: read # runner-loss evidence (annotations) for readiness retries steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 3f4bf936270..58f38ff8d60 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -108,6 +108,27 @@ It: - fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional - creates a git tag `canary/vYYYY.MDD.P-canary.N` +#### Busy master and lost runners + +Master often receives a commit every few minutes, faster than one canary +publishes (about 60-90 minutes). The release workflow keeps one running canary +and one pending canary. A newer push replaces the pending one, so the canary +that runs next is always the newest master commit. Commits in between are not +published on their own; they ship inside the next canary. + +Each canary waits for `Cloud source verified v1` from the Cloud readiness run +of its exact SHA (up to 90 minutes). Cloud readiness runs on AWS Spot workers. +When an attempt fails only because its workers were reclaimed, the canary gate +and the master image watcher re-run that attempt's failed jobs instead of +failing the release +([`scripts/cloud-readiness-runner-loss.mjs`](../scripts/cloud-readiness-runner-loss.mjs)). +A job counts as lost when its log shows the runner's shutdown or lost +communication error. Retries continue up to attempt 4 of the readiness run. A +real check failure, including a flaky test, is never retried: that canary fails +and the next push publishes. If a canary is missing after a quiet period, look +at the failed `Reuse exact-source verification` job. It names the readiness run +and lists each failed job as a lost runner or a failure. + After publication, the onboarding smoke checks out the same source SHA and installs its test dependencies from the publisher's resolved lockfile with `--frozen-lockfile`. The same-run artifact is captured before tracked files are @@ -316,7 +337,9 @@ latest attempt. A green admission-only run with skipped build and publish jobs is pending work. The source observer retries only producers that were strictly deferred, continues watching after build admission, and waits for publication. The final publication step rechecks both proofs before it creates image tags. -Failed or cancelled producers require inspection and are not retried automatically. +Failed or cancelled producers require inspection and are not retried automatically, +with one exception: a Cloud readiness attempt that failed only because Spot +runners were reclaimed is retried (see "Busy master and lost runners" below). [`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 396a9b2667a..1eebb910c7e 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -123,6 +123,8 @@ test("canary can admit deferred exact-source checks while stable keeps full veri assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); + // Two runner-loss retries of a deferred full readiness run fit inside the job. + assert.match(canary, /timeout-minutes: 100\n/); assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/); assert.match(releaseWorkflow, /publish_canary:\n\s+if: .*github\.event_name == 'push'.*\n\s+needs: \[verify_canary, release_runner_source, release_runner_binaries\]/); // The stable lane is gated on the stable channel since the nightly lane diff --git a/scripts/cloud-readiness-runner-loss.mjs b/scripts/cloud-readiness-runner-loss.mjs new file mode 100644 index 00000000000..d4f36bad0b2 --- /dev/null +++ b/scripts/cloud-readiness-runner-loss.mjs @@ -0,0 +1,130 @@ +// A Cloud readiness attempt that failed only because AWS Spot workers were +// reclaimed proves nothing about the source. Before this module, the first +// lost runner failed the release's source gate, the image watcher cancelled +// the readiness run, and nothing retried it: on 2026-11-13 a 13:49:56Z fleet +// reclaim held every release containing #21266 until a person re-ran the run. +// +// Both waiters (the npm canary gate in release.yml and the image watcher in +// docker.yml) call this when the source proof fails. A failure whose every +// causing job lost its runner re-runs the failed jobs instead, up to a fixed +// attempt ceiling. A real check failure still fails closed: it is never +// retried here. +import { classifyJob, createGithubClient } from "../.github/scripts/runner-loss-rerun.mjs"; + +const repository = "paperclipai/paperclip"; +const workflowPath = ".github/workflows/cloud-readiness.yml"; +// Attempt 1 is often the deferred admission-only pass and attempt 2 the +// admitted full run, so 4 allows at least two runner-loss retries. The ceiling +// is stateless: every waiter of every release attempt agrees on it. +export const maxRunnerLossAttempt = 4; +// A causing job ended the attempt on its own. Cancelled jobs are fallout from +// that failure (the image watcher or a reusable workflow cancelled them). +const causeConclusions = new Set(["failure", "timed_out", "startup_failure"]); +const nonGating = name => name.startsWith("Runner chaos evals (not a publication gate) / "); + +function trusted(run, sha, runId) { + return run?.id === runId && run.path === workflowPath && run.head_sha === sha && run.head_branch === "master" && + run.event === "push" && run.repository?.full_name === repository && run.head_repository?.full_name === repository && + Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0; +} + +async function attemptJobs(api, runId, attempt) { + const jobs = []; + for (let page = 1; page <= 10; page += 1) { + const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100&page=${page}`); + if (!Array.isArray(batch.jobs)) throw new Error("Cloud readiness job listing is malformed."); + jobs.push(...batch.jobs); + if (batch.jobs.length < 100) return jobs; + } + throw new Error("Cloud readiness job listing is incomplete."); +} + +/** Which failed jobs lost their runner. `lost` is true only when at least one + * job lost its runner and no job failed for any other reason. */ +export async function classifyAttempt(jobs, { readAnnotations, readLog }) { + const ended = jobs.filter(job => job.status === "completed" && !nonGating(job.name ?? "")); + const causes = ended.filter(job => causeConclusions.has(job.conclusion)); + // A reclaimed worker sometimes concludes its job `cancelled`. With no other + // cause, a cancelled job counts only when its own log shows the loss. + const candidates = causes.length ? causes : ended.filter(job => job.conclusion === "cancelled"); + const results = []; + for (const job of candidates) { + try { + results.push({ id: job.id, name: job.name, conclusion: job.conclusion, + ...(await classifyJob(job, { annotations: await readAnnotations(job), readLog })) }); + } catch (error) { + results.push({ id: job.id, name: job.name, conclusion: job.conclusion, lost: false, evidence: `could not read job evidence: ${error.message}` }); + } + } + const lost = causes.length ? results.length > 0 && results.every(job => job.lost) : results.some(job => job.lost); + return { lost, jobs: results }; +} + +/** + * Called after a source proof read failed for `failure.runId`/`failure.attempt`. + * Returns "retrying" when the waiter should keep waiting (a runner-loss retry + * was requested, is pending, or a newer attempt already exists) and "failed" + * when the failure stands. + */ +export async function retryRunnerLoss({ sha, failure, api, readAnnotations, readLog, cancel, rerunFailed, log = console.log }) { + const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`); + if (!trusted(run, sha, failure.runId)) throw new Error("Cloud readiness identity changed before a runner-loss retry."); + // Someone (another waiter or a person) already started the next attempt. + if (run.run_attempt > failure.attempt) return "retrying"; + if (run.run_attempt !== failure.attempt) return "failed"; + const jobs = (await attemptJobs(api, run.id, run.run_attempt)) + .filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha); + const verdict = await classifyAttempt(jobs, { readAnnotations, readLog }); + for (const job of verdict.jobs) log(`Cloud readiness ${run.id} attempt ${run.run_attempt}: ${job.name} ${job.lost ? "lost its runner" : "failed"} (${job.evidence}).`); + if (!verdict.lost) return "failed"; + if (run.run_attempt >= maxRunnerLossAttempt) { + log(`Cloud readiness ${run.id} lost runners on attempt ${run.run_attempt}, the last automatic attempt.`); + return "failed"; + } + // Re-running failed jobs needs a completed run. Healthy jobs still running + // would be repeated anyway once they are cancelled, so end the attempt now. + if (run.status !== "completed") { + // Job evidence reads take time. Another waiter may have ended this attempt + // and started the next one meanwhile: never cancel a newer attempt. + const fresh = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trusted(fresh, sha, run.id)) throw new Error("Cloud readiness identity changed before a runner-loss retry."); + if (fresh.run_attempt !== run.run_attempt || fresh.status === "completed") return "retrying"; + log(`Cloud readiness ${run.id} attempt ${run.run_attempt} lost its runner; ending the attempt to retry its failed jobs.`); + try { + await cancel(`/repos/${repository}/actions/runs/${run.id}/cancel`); + } catch (error) { + // The attempt finished or was cancelled by the other waiter first. + const latest = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trusted(latest, sha, run.id)) throw error; + if (latest.run_attempt > run.run_attempt || latest.status === "completed" || [409, 422].includes(error.status)) return "retrying"; + throw error; + } + return "retrying"; + } + try { + await rerunFailed(`/repos/${repository}/actions/runs/${run.id}/rerun-failed-jobs`); + log(`Cloud readiness ${run.id} attempt ${run.run_attempt} lost its runner; re-ran its failed jobs.`); + } catch (error) { + // The other waiter may have won the same write, or GitHub has not + // finished cancelling the attempt. Both resolve on the next poll. + const latest = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trusted(latest, sha, run.id)) throw error; + if (latest.run_attempt > run.run_attempt || latest.status !== "completed" || [409, 422].includes(error.status)) return "retrying"; + throw error; + } + return "retrying"; +} + +/** The canonical writes and evidence reads `retryRunnerLoss` needs. */ +export function createRunnerLossRetry({ token, fetcher = fetch, log = console.log }) { + const client = createGithubClient({ token, fetcher }); + const write = async path => { + if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/(cancel|rerun-failed-jobs)$/.test(path)) throw new Error("Invalid runner-loss retry path."); + const response = await fetcher(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw Object.assign(new Error(`Could not retry a lost readiness runner: HTTP ${response.status}.`), { status: response.status }); + }; + return ({ sha, failure, api }) => retryRunnerLoss({ sha, failure, api, log, cancel: write, rerunFailed: write, + readAnnotations: job => client.paginate(`/repos/${repository}/check-runs/${job.id}/annotations?per_page=100`), + readLog: job => client.text(`/repos/${repository}/actions/jobs/${job.id}/logs`) }); +} diff --git a/scripts/cloud-readiness-runner-loss.test.mjs b/scripts/cloud-readiness-runner-loss.test.mjs new file mode 100644 index 00000000000..258584667f4 --- /dev/null +++ b/scripts/cloud-readiness-runner-loss.test.mjs @@ -0,0 +1,284 @@ +import assert from "node:assert/strict"; +import test, { describe } from "node:test"; +import { classifyAttempt, createRunnerLossRetry, maxRunnerLossAttempt, retryRunnerLoss } from "./cloud-readiness-runner-loss.mjs"; +import { waitForSourceVerification } from "./cloud-source-verification.mjs"; +import { watchReleaseSource } from "./watch-release-source.mjs"; + +const sha = "a".repeat(40); +const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; +const lostLog = "2026-11-13T13:49:56.0521573Z ##[error]The runner has received a shutdown signal. This can happen when the runner service is stopped, or a manually started runner is canceled.\n"; +const testLog = "2026-11-13T14:19:18.2551007Z ##[error]AssertionError: expected \"vi.fn()\" to be called 2 times, but got 1 times\n"; +const cancelledNote = { annotation_level: "failure", message: "The operation was canceled." }; +const assertionNote = { annotation_level: "failure", message: "AssertionError: expected \"vi.fn()\" to be called 2 times, but got 1 times" }; + +/** + * A stateful fake of the GitHub Actions API for one Cloud readiness run. Each + * attempt is a list of jobs; `rerun-failed-jobs` starts the next attempt from + * `script`, and `cancel` completes the current one. Job evidence (annotations + * and logs) is keyed by job id, as GitHub serves it. + */ +function github({ attempts, script = [], startAttempt = 1, status = "completed" }) { + const state = { attempt: startAttempt, status, attempts: { ...attempts }, writes: [], evidence: {}, script }; + const run = () => ({ id: 99, workflow_id: 123, path: ".github/workflows/cloud-readiness.yml", run_attempt: state.attempt, + repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push", + status: state.status, conclusion: state.status === "completed" ? "failure" : null }); + state.api = async path => { + if (path.endsWith("/workflows/cloud-readiness.yml")) return { id: 123, path: ".github/workflows/cloud-readiness.yml" }; + if (path.includes("/workflows/123/runs?")) return { total_count: 1, workflow_runs: [run()] }; + const jobs = path.match(/\/runs\/99\/attempts\/(\d+)\/jobs\?/); + if (jobs) return { total_count: (state.attempts[jobs[1]] ?? []).length, jobs: state.attempts[jobs[1]] ?? [] }; + if (path.endsWith("/runs/99")) return run(); + // The image watcher's own run and the migrator lookup. + if (path.includes("/runs/50/attempts/1/jobs?")) return { total_count: 0, jobs: [] }; + if (path.endsWith("/runs/50")) return { id: 50, run_attempt: 1, head_sha: sha, head_branch: "master", event: "push", + repository, head_repository: repository, path: ".github/workflows/docker.yml", status: "in_progress" }; + if (path.endsWith("/workflows/cloud-migrator-artifacts.yml")) return { id: 124, path: ".github/workflows/cloud-migrator-artifacts.yml" }; + if (path.includes("/workflows/124/runs?")) return { total_count: 0, workflow_runs: [] }; + throw new Error(`Unexpected API path: ${path}`); + }; + const write = async path => { + state.writes.push(path); + if (path.endsWith("/cancel")) state.status = "completed"; + if (path.endsWith("/rerun-failed-jobs")) { + if (state.status !== "completed") throw Object.assign(new Error("HTTP 409"), { status: 409 }); + state.attempt += 1; + state.attempts[state.attempt] = state.script.shift()?.(state.attempt) ?? []; + state.status = "in_progress"; + } + }; + state.cancel = write; + state.rerunFailed = write; + state.readAnnotations = async job => state.evidence[job.id]?.annotations ?? []; + state.readLog = async job => state.evidence[job.id]?.log ?? ""; + state.retry = ({ failure, api }) => retryRunnerLoss({ sha, failure, api, log: () => {}, cancel: path => state.cancel(path), + rerunFailed: path => state.rerunFailed(path), readAnnotations: job => state.readAnnotations(job), readLog: job => state.readLog(job) }); + state.job = (attempt, id, name, conclusion, evidence) => { + if (evidence) state.evidence[id] = evidence; + return { id, name, run_id: 99, run_attempt: attempt, head_sha: sha, status: conclusion ? "completed" : "in_progress", conclusion: conclusion ?? null }; + }; + return state; +} + +const proofName = "Cloud source verified v1"; + +// The 2026-11-13 13:49:56Z reclaim of readiness run 43435593630 attempt 2: +// several jobs lost their Spot runners at once, siblings were cancelled, and +// the source proof was cancelled. +function reclaimedAttempt(g, attempt) { + return [ + g.job(attempt, attempt * 100 + 1, "verify / Serialized tests (1/5)", "failure", { annotations: [cancelledNote], log: lostLog }), + g.job(attempt, attempt * 100 + 2, "verify / Verify Paperclip Runner (rust)", "failure", { annotations: [cancelledNote], log: lostLog }), + g.job(attempt, attempt * 100 + 3, "verify / General tests (server (1/10))", "cancelled", { annotations: [cancelledNote], log: "" }), + g.job(attempt, attempt * 100 + 4, "verify / Build", "success"), + g.job(attempt, attempt * 100 + 5, proofName, "cancelled"), + ]; +} +const greenAttempt = g => attempt => [g.job(attempt, attempt * 100 + 4, "verify / Build", "success"), g.job(attempt, attempt * 100 + 5, proofName, "success")]; +// A genuine test failure, as in readiness run 43439942063 (decision-models). +const flakyAttempt = g => attempt => [ + g.job(attempt, attempt * 100 + 1, "verify / General tests (server (4/10))", "failure", { annotations: [assertionNote], log: testLog }), + g.job(attempt, attempt * 100 + 3, "verify / General tests (server (1/10))", "cancelled", { annotations: [cancelledNote] }), + g.job(attempt, attempt * 100 + 5, proofName, "cancelled"), +]; + +const clock = () => { let time = 0; return { now: () => time, sleep: async ms => { time += ms; } }; }; + +describe("runner-loss classification", () => { + test("an attempt whose every failed job lost its runner is a runner loss; cancelled siblings are fallout", async () => { + const g = github({ attempts: {} }); + const verdict = await classifyAttempt(reclaimedAttempt(g, 2), g); + assert.equal(verdict.lost, true); + assert.deepEqual(verdict.jobs.map(job => job.conclusion), ["failure", "failure"]); + }); + + test("one real failure beside lost runners is not a runner loss", async () => { + const g = github({ attempts: {} }); + const jobs = [...reclaimedAttempt(g, 2), ...flakyAttempt(g)(2).slice(0, 1)]; + assert.equal((await classifyAttempt(jobs, g)).lost, false); + }); + + test("a failing test is not a runner loss", async () => { + const g = github({ attempts: {} }); + assert.equal((await classifyAttempt(flakyAttempt(g)(2), g)).lost, false); + }); + + test("a reclaimed job that concluded cancelled counts only with a runner-loss log", async () => { + const g = github({ attempts: {} }); + const lost = [g.job(2, 1, "verify / Build", "cancelled", { annotations: [cancelledNote], log: lostLog }), g.job(2, 2, proofName, "cancelled")]; + assert.equal((await classifyAttempt(lost, g)).lost, true); + // A person cancelling the run is not a runner loss. + const manual = [g.job(2, 3, "verify / Build", "cancelled", { annotations: [cancelledNote, { annotation_level: "failure", message: "The run was canceled by @someone." }] }), g.job(2, 4, proofName, "cancelled")]; + assert.equal((await classifyAttempt(manual, g)).lost, false); + }); + + test("non-gating Runner chaos evals neither cause nor block a retry", async () => { + const g = github({ attempts: {} }); + const chaos = g.job(2, 9, "Runner chaos evals (not a publication gate) / Restart, replay, trace, and recovery faults", "failure", { annotations: [assertionNote], log: testLog }); + assert.equal((await classifyAttempt([...reclaimedAttempt(g, 2), chaos], g)).lost, true); + }); + + test("unreadable job evidence fails closed", async () => { + const g = github({ attempts: {} }); + const verdict = await classifyAttempt(reclaimedAttempt(g, 2), { ...g, readLog: async () => { throw new Error("HTTP 404"); } }); + assert.equal(verdict.lost, false); + }); +}); + +describe("runner-loss retry", () => { + test("a completed reclaimed attempt re-runs only its failed jobs", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.deepEqual(g.writes, ["/repos/paperclipai/paperclip/actions/runs/99/rerun-failed-jobs"]); + }); + + test("a reclaimed attempt still running is ended first, then re-run on the next poll", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = reclaimedAttempt(g, 2); + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.deepEqual(g.writes, ["/repos/paperclipai/paperclip/actions/runs/99/cancel"]); + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.equal(g.writes.at(-1), "/repos/paperclipai/paperclip/actions/runs/99/rerun-failed-jobs"); + }); + + test("a newer attempt started while evidence was read is never cancelled", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = reclaimedAttempt(g, 2); + const readLog = g.readLog; + // The other waiter ends attempt 2 and starts attempt 3 during our log read. + g.readLog = async job => { g.attempt = 3; g.status = "in_progress"; return readLog(job); }; + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.deepEqual(g.writes, []); + }); + + test("a cancellation conflict after the attempt completed keeps waiting", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.cancel = async () => { g.status = "completed"; throw Object.assign(new Error("HTTP 409"), { status: 409 }); }; + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + }); + + test("an unexpected cancellation error with the attempt still running is surfaced", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.cancel = async () => { throw Object.assign(new Error("HTTP 500"), { status: 500 }); }; + await assert.rejects(g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), /HTTP 500/); + }); + + test("a real failure is never retried", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = flakyAttempt(g)(2); + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "failed"); + assert.deepEqual(g.writes, []); + }); + + test("retries stop at the attempt ceiling", async () => { + const g = github({ attempts: {}, startAttempt: maxRunnerLossAttempt }); + g.attempts[maxRunnerLossAttempt] = reclaimedAttempt(g, maxRunnerLossAttempt); + assert.equal(await g.retry({ failure: { runId: 99, attempt: maxRunnerLossAttempt }, api: g.api }), "failed"); + assert.deepEqual(g.writes, []); + }); + + test("a newer attempt started by the other waiter or a person is waited on, not re-run", async () => { + const g = github({ attempts: {}, startAttempt: 3, status: "in_progress" }); + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.deepEqual(g.writes, []); + }); + + test("the other waiter winning the same re-run is not a failure", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.rerunFailed = async () => { g.attempt += 1; g.status = "in_progress"; throw Object.assign(new Error("HTTP 403"), { status: 403 }); }; + assert.equal(await g.retry({ failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + }); + + test("a changed run identity refuses to write", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + await assert.rejects(retryRunnerLoss({ sha: "b".repeat(40), failure: { runId: 99, attempt: 2 }, api: g.api, log: () => {}, + cancel: g.cancel, rerunFailed: g.rerunFailed, readAnnotations: g.readAnnotations, readLog: g.readLog }), /identity changed/); + assert.deepEqual(g.writes, []); + }); + + test("every waiter job that may retry can read runner-loss evidence", async () => { + const { readFileSync } = await import("node:fs"); + const read = file => readFileSync(new URL(`../.github/workflows/${file}`, import.meta.url), "utf8"); + const job = (text, name) => text.match(new RegExp(`\\n ${name}:\\n[\\s\\S]*?(?=\\n [a-z_-]+:\\n)`))[0]; + for (const [file, name] of [["release.yml", "verify_canary"], ["docker.yml", "source_verification"], ["docker.yml", "publication_verification"]]) { + const text = job(read(file), name); + assert.match(text, /\n checks: read\b/, `${file} ${name} needs checks: read for annotations`); + assert.match(text, /\n actions: write\b/, `${file} ${name} needs actions: write for logs and retries`); + } + }); + + test("the production writer only posts cancel or rerun-failed-jobs to a canonical run", async () => { + const posts = []; + const fetcher = async (url, init) => { posts.push([url, init.method]); return new Response(null, { status: 201 }); }; + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + const evidence = async url => url.includes("/annotations") ? Response.json([cancelledNote]) : new Response(lostLog); + const retry = createRunnerLossRetry({ token: "t", log: () => {}, fetcher: async (url, init = {}) => init.method === "POST" ? fetcher(url, init) : evidence(url) }); + assert.equal(await retry({ sha, failure: { runId: 99, attempt: 2 }, api: g.api }), "retrying"); + assert.deepEqual(posts, [["https://api.github.com/repos/paperclipai/paperclip/actions/runs/99/rerun-failed-jobs", "POST"]]); + }); +}); + +// Simulations of the 2026-11-13 hold: readiness attempt 2 loses its runners. +// Before this change both waiters failed the release on that attempt. +describe("busy-master simulation", () => { + test("the npm canary gate retries a reclaimed attempt and publishes on the retried proof", async () => { + const g = github({ attempts: {}, startAttempt: 2, script: [] }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.script.push(greenAttempt(g)); + const proof = await waitForSourceVerification(sha, { api: g.api, ...clock(), log: () => {}, retryRunnerLoss: g.retry }); + assert.deepEqual({ runId: proof.runId, attempt: proof.attempt }, { runId: 99, attempt: 3 }); + assert.deepEqual(g.writes, ["/repos/paperclipai/paperclip/actions/runs/99/rerun-failed-jobs"]); + }); + + test("without the retry the same reclaim fails the release (the 2026-11-13 behavior)", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + await assert.rejects(waitForSourceVerification(sha, { api: g.api, ...clock(), log: () => {} }), /did not pass/); + }); + + test("two reclaims in a row still publish; a third stops at the ceiling", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.script.push(attempt => reclaimedAttempt(g, attempt), greenAttempt(g)); + const proof = await waitForSourceVerification(sha, { api: g.api, ...clock(), log: () => {}, retryRunnerLoss: g.retry }); + assert.equal(proof.attempt, 4); + + const h = github({ attempts: {}, startAttempt: 2 }); + h.attempts[2] = reclaimedAttempt(h, 2); + h.script.push(attempt => reclaimedAttempt(h, attempt), attempt => reclaimedAttempt(h, attempt)); + await assert.rejects(waitForSourceVerification(sha, { api: h.api, ...clock(), log: () => {}, retryRunnerLoss: h.retry }), /attempt 4/); + }); + + test("a real test failure still fails the release without a retry", async () => { + const g = github({ attempts: {}, startAttempt: 2 }); + g.attempts[2] = flakyAttempt(g)(2); + await assert.rejects(waitForSourceVerification(sha, { api: g.api, ...clock(), log: () => {}, retryRunnerLoss: g.retry }), /did not pass/); + assert.deepEqual(g.writes, []); + }); + + test("the image watcher keeps its image attempt alive through a reclaim instead of cancelling it", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = reclaimedAttempt(g, 2); + g.script.push(greenAttempt(g)); + const imageCancels = []; + const proof = await watchReleaseSource({ sha, runId: 50, attempt: 1, api: g.api, ...clock(), log: () => {}, + cancel: async path => imageCancels.push(path), retryRunnerLoss: g.retry }); + assert.equal(proof.attempt, 3); + assert.deepEqual(imageCancels, []); + assert.deepEqual(g.writes, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", "/repos/paperclipai/paperclip/actions/runs/99/rerun-failed-jobs"]); + }); + + test("the image watcher still cancels image work on a real failure", async () => { + const g = github({ attempts: {}, startAttempt: 2, status: "in_progress" }); + g.attempts[2] = flakyAttempt(g)(2); + const imageCancels = []; + await assert.rejects(watchReleaseSource({ sha, runId: 50, attempt: 1, api: g.api, ...clock(), log: () => {}, + cancel: async path => imageCancels.push(path), retryRunnerLoss: g.retry }), /General tests/); + assert.deepEqual(imageCancels, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", "/repos/paperclipai/paperclip/actions/runs/50/cancel"]); + }); +}); diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index f2e71314781..db6f34019d8 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -2,6 +2,7 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import { createActionsReader } from "./github-actions-reader.mjs"; import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; +import { createRunnerLossRetry } from "./cloud-readiness-runner-loss.mjs"; const repository = "paperclipai/paperclip"; const workflowPath = ".github/workflows/cloud-readiness.yml"; @@ -96,7 +97,7 @@ export { createActionsReader }; export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun, + timeoutMs = 90 * 60_000, intervalMs = 30_000, log = console.log, rerun, retryRunnerLoss, } = {}) { assertSha(sha); const deadline = now() + timeoutMs; @@ -107,8 +108,13 @@ export async function waitForSourceVerification(sha, { const result = await readSourceVerification(sha, api); if (result) return result; } catch (error) { - if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error; - await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + if (error instanceof SourceVerificationFailed && retryRunnerLoss) { + // A reclaimed Spot worker is not a source failure: retry its jobs. + if (await retryRunnerLoss({ sha, failure: error, api }) !== "retrying") throw error; + } else { + if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error; + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } } const remaining = deadline - now(); if (remaining > 0) await sleep(Math.min(intervalMs, remaining)); @@ -121,11 +127,14 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required."); // One deadline for both layers: the reader stops retrying when the poll it // serves is out of time, instead of extending the wait past its timeout. - const timeoutMs = 60 * 60_000; + // Long enough for a deferred full run plus two runner-loss retries. + const timeoutMs = 90 * 60_000; const deadline = Date.now() + timeoutMs; const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline }); - const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined; - const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun }); + const canonical = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master"; + const rerun = canonical ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined; + const retryRunnerLoss = canonical ? createRunnerLossRetry({ token: process.env.GITHUB_TOKEN }) : undefined; + const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun, retryRunnerLoss }); const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`; console.log(message); if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); diff --git a/scripts/watch-release-source.mjs b/scripts/watch-release-source.mjs index a65dfaa1eea..1dbdfe7b6ca 100644 --- a/scripts/watch-release-source.mjs +++ b/scripts/watch-release-source.mjs @@ -2,6 +2,7 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; import { requestDeferredProducer, readMigratorPublication, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; +import { createRunnerLossRetry } from "./cloud-readiness-runner-loss.mjs"; const repository = "paperclipai/paperclip"; @@ -35,7 +36,7 @@ async function cancelFailedVerification({ api, cancel, sha }, failure) { if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`); } -export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now, +export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, retryRunnerLoss, now = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log, timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) { if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 || @@ -68,6 +69,10 @@ export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rer } catch (error) { if (error instanceof SourceVerificationDeferred && rerun) { await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } else if (error instanceof SourceVerificationFailed && retryRunnerLoss && + await retryRunnerLoss({ sha, failure: error, api }) === "retrying") { + // A reclaimed Spot worker is retried, not treated as a failed source: + // keep this image attempt alive for the retried proof. } else { if (error instanceof SourceVerificationFailed) { await cancelFailedVerification({ api, cancel, sha }, error); @@ -95,6 +100,7 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT), api, rerun: createReleaseRerunner(token), + retryRunnerLoss: createRunnerLossRetry({ token }), cancel: async path => { const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error", headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });