diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index 28444873bb..bd40e33cd3 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -61,6 +61,80 @@ export async function readSourceVerification(sha, api) { return undefined; } +// Statuses that say "ask again", not "the answer is no". A release must not be +// blocked because GitHub returned a gateway error during a 45-minute poll. +const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); + +// A rate-limited read also says "ask again", and GitHub reports both primary +// and secondary rate limits as 403. Only the headers separate that from a +// token that may not read Actions, which must still fail at once. +function rateLimited(response) { + if (response.status !== 403) return false; + const header = (name) => response.headers?.get?.(name) ?? null; + return header("retry-after") !== null || header("x-ratelimit-remaining") === "0"; +} + +/** Milliseconds from a Retry-After header, when it carries a sane delay. */ +function retryAfterMs(response, capMs) { + const value = Number(response?.headers?.get?.("retry-after")); + if (!Number.isFinite(value) || value <= 0) return null; + return Math.min(value * 1_000, capMs); +} + +/** + * The GitHub Actions read used by the polling below, with transient transport + * failures retried: network errors, the statuses above, rate-limited 403s, and + * a body that fails while it is being read. Any other non-OK status throws on + * the first response, because waiting out a wrong token only delays the news. + * + * `deadlineAt` bounds retries by the caller's own polling deadline, so a read + * cannot extend the wait past the timeout it belongs to. + */ +export function createActionsReader({ + token, fetchImpl = fetch, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + attempts = 4, backoffMs = 1_000, log = console.log, + now = Date.now, deadlineAt = () => Infinity, maxRetryAfterMs = 60_000, +} = {}) { + return async (path) => { + for (let attempt = 1; ; attempt += 1) { + // Retry only while both the attempt budget and the caller's deadline + // leave room for the wait this attempt would cost. + const waitMs = backoffMs * attempt; + const retryable = attempt < attempts && now() + waitMs < deadlineAt(); + const pause = (response) => sleep(retryAfterMs(response, maxRetryAfterMs) ?? waitMs); + let response; + try { + response = await fetchImpl(`https://api.github.com${path}`, { + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, + signal: AbortSignal.timeout(30_000), redirect: "error", + }); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + if (response.ok) { + try { + // A 200 whose body is truncated or undecodable is a transport + // failure like any other, so it belongs inside the retry. + return await response.json(); + } catch (cause) { + if (!retryable) throw new Error(`GitHub Actions read failed: ${cause.message}`); + log(`GitHub Actions read body failed (${cause.message}); retrying (${attempt}/${attempts - 1}).`); + await pause(); + continue; + } + } + if (!retryable || !(TRANSIENT_READ_STATUSES.has(response.status) || rateLimited(response))) { + throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); + } + log(`GitHub Actions read failed (HTTP ${response.status}); retrying (${attempt}/${attempts - 1}).`); + await pause(response); + } + }; +} + export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log, @@ -80,15 +154,12 @@ export async function waitForSourceVerification(sha, { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { try { if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required."); - const api = async (path) => { - const response = await fetch(`https://api.github.com${path}`, { - headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" }, - signal: AbortSignal.timeout(30_000), redirect: "error", - }); - if (!response.ok) throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`); - return response.json(); - }; - const proof = await waitForSourceVerification(process.argv[2], { api }); + // One deadline for both layers: the reader stops retrying when the poll it + // serves is out of time, instead of extending the wait past its timeout. + const timeoutMs = 45 * 60_000; + const deadline = Date.now() + timeoutMs; + const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline }); + const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs }); const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`; console.log(message); if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index d75b2baf2f..4a47abfd16 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs"; +import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs"; const sha = "a".repeat(40); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; @@ -123,3 +123,111 @@ test("malformed source refs are rejected before any request", async () => { await assert.rejects(readSourceVerification(ref, () => assert.fail("must not request")), /full lowercase/); } }); + +// A gateway error during the poll must not decide the release. These cover the +// reader's transport only; the verification semantics above are unchanged. +function reader({ responses, attempts = 4, ...options }) { + const seen = []; + const waits = []; + const fetchImpl = async () => { + const next = responses[seen.length]; + seen.push(next); + if (next instanceof Error) throw next; + return { + ok: next.status >= 200 && next.status < 300, + status: next.status, + headers: { get: (name) => next.headers?.[name.toLowerCase()] ?? null }, + json: async () => { + if (next.bodyError) throw next.bodyError; + return next.body ?? { ok: true }; + }, + }; + }; + const api = createActionsReader({ + token: "t", fetchImpl, attempts, backoffMs: 10, + sleep: async (ms) => { waits.push(ms); }, log: () => {}, ...options, + }); + return { api, seen, waits }; +} + +test("a transient gateway error is retried instead of failing the release", async () => { + const { api, seen, waits } = reader({ responses: [{ status: 502 }, { status: 200, body: { id: 1 } }] }); + assert.deepEqual(await api("/repos/x"), { id: 1 }); + assert.equal(seen.length, 2); + assert.deepEqual(waits, [10]); +}); + +test("every transient status is retried, and the backoff grows", async () => { + for (const status of [408, 425, 429, 500, 502, 503, 504]) { + const { api, seen, waits } = reader({ responses: [{ status }, { status }, { status: 200, body: { ok: true } }] }); + await api("/repos/x"); + assert.equal(seen.length, 3, `status ${status} should be retried`); + assert.deepEqual(waits, [10, 20]); + } +}); + +test("a rate-limited 403 is retried, and a forbidden 403 is not", async () => { + // GitHub reports both primary and secondary rate limits as 403; only the + // headers tell them apart from a token that may not read Actions. + for (const headers of [{ "retry-after": "1" }, { "x-ratelimit-remaining": "0" }]) { + const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200, body: { ok: true } }] }); + await api("/repos/x"); + assert.equal(seen.length, 2, `403 with ${JSON.stringify(headers)} should be retried`); + } + for (const headers of [undefined, { "x-ratelimit-remaining": "4999" }]) { + const { api, seen } = reader({ responses: [{ status: 403, headers }, { status: 200 }] }); + await assert.rejects(api("/repos/x"), /HTTP 403/); + assert.equal(seen.length, 1, "a forbidden 403 must fail on the first response"); + } +}); + +test("Retry-After sets the wait, capped so one header cannot stall the poll", async () => { + const { api, waits } = reader({ responses: [{ status: 429, headers: { "retry-after": "5" } }, { status: 200 }] }); + await api("/repos/x"); + assert.deepEqual(waits, [5_000]); + + const capped = reader({ responses: [{ status: 429, headers: { "retry-after": "86400" } }, { status: 200 }], maxRetryAfterMs: 60_000 }); + await capped.api("/repos/x"); + assert.deepEqual(capped.waits, [60_000]); +}); + +test("a body that fails while being read is retried", async () => { + const { api, seen } = reader({ + responses: [{ status: 200, bodyError: new Error("terminated") }, { status: 200, body: { id: 7 } }], + }); + assert.deepEqual(await api("/repos/x"), { id: 7 }); + assert.equal(seen.length, 2); +}); + +test("a network failure is retried, and its message survives exhaustion", async () => { + const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => new Error("fetch failed")) }); + await assert.rejects(api("/repos/x"), /GitHub Actions read failed: fetch failed/); + assert.equal(seen.length, 4); +}); + +test("an authorization failure is not retried", async () => { + for (const status of [401, 404, 422]) { + const { api, seen } = reader({ responses: [{ status }, { status: 200 }] }); + await assert.rejects(api("/repos/x"), new RegExp(`HTTP ${status}`)); + assert.equal(seen.length, 1, `status ${status} must fail on the first response`); + } +}); + +test("a transient status that never clears fails after its attempt budget", async () => { + const { api, seen } = reader({ responses: Array.from({ length: 4 }, () => ({ status: 502 })) }); + await assert.rejects(api("/repos/x"), /HTTP 502/); + assert.equal(seen.length, 4); +}); + +test("retries stop at the caller's deadline rather than outliving the poll", async () => { + // The wait this attempt would cost does not fit before the deadline, so the + // read reports the failure instead of sleeping past the timeout it serves. + let clock = 0; + const { api, seen, waits } = reader({ + responses: [{ status: 502 }, { status: 200 }], + now: () => clock, deadlineAt: () => 5, + }); + await assert.rejects(api("/repos/x"), /HTTP 502/); + assert.equal(seen.length, 1); + assert.deepEqual(waits, []); +});