diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 70e28fe798f..20bb8ee203f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -310,6 +310,14 @@ The workflow: ## Docker Image Tags +Master image publication requires both exact-source verification and a successful +`publish` job from the canonical migrator workflow at the same source SHA and +latest attempt. A green admission-only run with skipped build and publish jobs +is pending work. The source observer retries only producers that were strictly +deferred, continues watching after build admission, and waits for publication. +The final publication step rechecks both proofs before it creates image tags. +Failed or cancelled producers require inspection and are not retried automatically. + [`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: | Build ref | Tags | diff --git a/scripts/cloud-release-admission.mjs b/scripts/cloud-release-admission.mjs index 31ca67fa9db..007f5e0755e 100644 --- a/scripts/cloud-release-admission.mjs +++ b/scripts/cloud-release-admission.mjs @@ -56,14 +56,7 @@ export function deferredProducer(jobs, run, admissionJob) { jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha); } -// Only skipped admission is restartable here. A failed check or build never -// becomes an automatic retry, and every write revalidates canonical identity. -export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) { - if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract."); - const { workflow, run } = await producer(file, sha, api); - if (!run) return "pending"; - const key = `${run.id}/${run.run_attempt}`; - if (requested.has(key)) return "pending"; +async function producerJobs(run, api) { const jobs = []; let total; for (let page = 1; page <= 10; page++) { @@ -73,14 +66,43 @@ export async function requestDeferredProducer({ file, sha, admissionJob, api, re batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list."); total = batch.total_count; jobs.push(...batch.jobs); - if (page * 100 >= total) break; - if (page === 10) throw new Error("Incomplete release producer job list."); + if (page * 100 >= total) return jobs; } + throw new Error("Incomplete release producer job list."); +} + +/** Admission is not publication. The final image gate requires the exact + * canonical migrator publisher, including a fresh read after its job evidence. */ +export async function readMigratorPublication(sha, api) { + const { workflow, run } = await producer("cloud-migrator-artifacts.yml", sha, api); + if (!run) return false; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed") return false; + if (current.conclusion !== "success") throw new Error("Exact-source migrator publication failed; inspect its workflow before retrying."); + const jobs = await producerJobs(current, api); + if (!jobs.every(job => job.run_id === run.id && job.run_attempt === current.run_attempt && job.head_sha === sha)) return false; + const publish = jobs.filter(job => job.name === "publish"); + if (publish.length !== 1 || publish[0].status !== "completed" || publish[0].conclusion !== "success") return false; + const confirmed = await api(`/repos/${repository}/actions/runs/${run.id}`); + return confirmed.id === run.id && trusted(confirmed, sha, workflow) && confirmed.run_attempt === current.run_attempt && + confirmed.status === "completed" && confirmed.conclusion === "success"; +} + +// Only skipped admission is restartable here. A failed check or build never +// becomes an automatic retry, and every write revalidates canonical identity. +export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) { + if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract."); + const { workflow, run } = await producer(file, sha, api); + if (!run) return "pending"; + const key = `${run.id}/${run.run_attempt}`; + if (requested.has(key)) return "pending"; + const jobs = await producerJobs(run, api); if (run.status !== "completed") { const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha); // Admission itself succeeds for both outcomes. Only an actual build is // evidence that its output admitted work; queued jobs may still be skipped. - return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped") + return current.some(job => job.name !== admissionJob && Number.isSafeInteger(job.runner_id) && job.runner_id > 0 && + ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped") ? "admitted" : "pending"; } if (!deferredProducer(jobs, run, admissionJob)) return "finished"; diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 6e055607267..35c07a1ed0d 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -7,7 +7,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; -import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; +import { releaseAdmitted, deferredProducer, requestDeferredProducer, readMigratorPublication, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs"; @@ -63,6 +63,48 @@ describe("admission CLI startup", () => { assert.equal(result.output, ""); }); }); + +test("the final image publication CLI rejects a green workflow that skipped its migrator", () => { + const root = mkdtempSync(join(tmpdir(), "migrator-publication-cli-")); + try { + const preload = join(root, "fixture.mjs"); + const installFixture = () => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const run = (id, file) => ({ id, workflow_id: id, run_attempt: 1, path: `.github/workflows/${file}`, event: "push", + head_branch: "master", head_sha: "a".repeat(40), repository, head_repository: repository, status: "completed", conclusion: "success" }); + const source = run(1, "cloud-readiness.yml"), migrator = run(2, "cloud-migrator-artifacts.yml"); + globalThis.fetch = async (input, init) => { + if (init?.method && init.method !== "GET") throw new Error("Final publication must be read-only"); + const url = new URL(input), path = url.pathname; + if (url.origin !== "https://api.github.com") throw new Error("Unexpected origin"); + if (path.endsWith("/cloud-readiness.yml")) return Response.json({ id: 1, path: source.path }); + if (path.endsWith("/cloud-migrator-artifacts.yml")) return Response.json({ id: 2, path: migrator.path }); + if (path.endsWith("/workflows/1/runs")) return Response.json({ total_count: 1, workflow_runs: [source] }); + if (path.endsWith("/workflows/2/runs")) return Response.json({ total_count: 1, workflow_runs: [migrator] }); + if (path.endsWith("/runs/1")) return Response.json(source); + if (path.endsWith("/runs/2")) return Response.json(migrator); + if (path.endsWith("/jobs")) { + const id = path.includes("/runs/1/") ? 1 : 2; + return Response.json({ total_count: 1, jobs: [{ id: 3, name: id === 1 ? "Cloud source verified v1" : "publish", + run_id: id, run_attempt: 1, head_sha: "a".repeat(40), status: "completed", + conclusion: id === 1 || process.env.FIXTURE_PUBLISHED === "1" ? "success" : "skipped" }] }); + } + throw new Error(`Unexpected path ${path}`); + }; + }; + writeFileSync(preload, "(" + installFixture.toString() + ")();\n"); + for (const published of [false, true]) { + const output = join(root, `output-${published}`); + const result = spawnSync(process.execPath, ["--import", preload, fileURLToPath(new URL("./watch-release-source.mjs", import.meta.url)), "--check-proof"], { + encoding: "utf8", timeout: 5_000, env: { ...process.env, GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", + GITHUB_SHA: sha, GITHUB_TOKEN: "fixture", GITHUB_OUTPUT: output, FIXTURE_PUBLISHED: published ? "1" : "0" }, + }); + assert.equal(result.status, published ? 0 : 1, result.stderr || result.error?.message); + if (published) assert.equal(readFileSync(output, "utf8"), "verified=true\n"); + else assert.match(result.stderr, /Exact-source migrator has not published/); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; const baseRun = { id: 456, workflow_id: workflow.id, path: workflow.path, run_attempt: 2, @@ -583,6 +625,8 @@ describe("release candidate admission", () => { f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build]; assert.equal(await requestDeferredProducer(f), "pending"); build.status = "in_progress"; + assert.equal(await requestDeferredProducer(f), "pending", "labels alone do not prove a worker started"); + build.runner_id = 123; assert.equal(await requestDeferredProducer(f), "admitted"); build.run_attempt = 2; assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work"); @@ -615,7 +659,7 @@ describe("release candidate admission", () => { assert.equal(f.writes.length, 1); }); - test("the image monitor admits a late migrator even when source proof is ready", async () => { + test("the image monitor waits for actual migrator publication after deferred admission", async () => { const f = fixture("cloud-migrator-artifacts.yml"); const source = run("cloud-readiness.yml"); f.runs["cloud-readiness.yml"] = [source]; @@ -627,15 +671,45 @@ describe("release candidate admission", () => { const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"), now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => { time += ms; - const selected = run(f.file, { run_attempt: 2 }); + const selected = run(f.file, { run_attempt: 2, ...(time >= 20 ? { status: "completed", conclusion: "success" } : {}) }); f.runs[f.file] = [selected]; - f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }]; + f.jobs[selected.id] = [time >= 20 ? job(selected, "publish", "success") : + { ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"], runner_id: 123 }]; } }); assert.equal(proof.sha, sha); - assert.equal(time, 10); + assert.equal(time, 20); assert.equal(f.writes.length, 1); }); + test("migrator publication rejects skipped, stale, failed and duplicate publisher evidence", async () => { + const f = fixture("cloud-migrator-artifacts.yml"); + const selected = f.runs[f.file][0]; + assert.equal(await readMigratorPublication(sha, f.api), false); + f.jobs[selected.id] = [job(selected, "publish", "success")]; + assert.equal(await readMigratorPublication(sha, f.api), true); + for (const change of [{ conclusion: "skipped" }, { run_attempt: 2 }, { run_id: 1 }, { head_sha: "b".repeat(40) }, { status: "queued" }]) { + f.jobs[selected.id] = [{ ...job(selected, "publish", "success"), ...change }]; + assert.equal(await readMigratorPublication(sha, f.api), false); + } + f.jobs[selected.id] = [job(selected, "publish", "success"), job(selected, "publish", "success")]; + assert.equal(await readMigratorPublication(sha, f.api), false); + f.current[selected.id] = { ...selected, conclusion: "failure" }; + await assert.rejects(readMigratorPublication(sha, f.api), /migrator publication failed/); + assert.deepEqual(f.writes, [], "a read never retries a failed producer"); + }); + + test("a publication retry racing job evidence invalidates the old success", async () => { + const f = fixture("cloud-migrator-artifacts.yml"); + const selected = f.runs[f.file][0]; + f.jobs[selected.id] = [job(selected, "publish", "success")]; + let reads = 0; + const api = async path => { + if (path.endsWith(`/runs/${selected.id}`) && ++reads === 2) return { ...selected, run_attempt: 2 }; + return f.api(path); + }; + assert.equal(await readMigratorPublication(sha, api), false); + }); + test("the retry transport is a bounded POST to a canonical run", async () => { const calls = []; const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; }); diff --git a/scripts/watch-release-source.mjs b/scripts/watch-release-source.mjs index bf4fe032774..a65dfaa1eea 100644 --- a/scripts/watch-release-source.mjs +++ b/scripts/watch-release-source.mjs @@ -1,7 +1,7 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; -import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; +import { requestDeferredProducer, readMigratorPublication, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; const repository = "paperclipai/paperclip"; @@ -43,7 +43,6 @@ export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rer const deadline = now() + timeoutMs; const warned = new Set(); const requested = new Set(); - let migratorAdmitted = !rerun; const reportWaits = async jobs => { for (const wait of runnerWaits(jobs, now())) { if (wait.seconds > 60 && !warned.has(wait.id)) { @@ -54,15 +53,15 @@ export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rer } }; while (now() < deadline) { - if (!migratorAdmitted) { - migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending"; - } + // Keep observing until publication, even after a build appeared admitted. + // GitHub may transiently label a job in progress before skipping it. + if (rerun) await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }); const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`); if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing."); await reportWaits(batch.jobs); try { const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits }); - if (proof && migratorAdmitted) { + if (proof && (!rerun || await readMigratorPublication(sha, api))) { await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`); return proof; } @@ -82,8 +81,8 @@ export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rer // This early failure observer is not the publication deadline. Healthy // checks can outlive a Spot worker; a separate exact-source gate still // waits for proof before publishing any image. - if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline."); - await summary("Source monitoring window ended; final publication still requires exact-source verification.\n"); + if (requireProof) throw new Error("Exact-source verification and migrator publication did not become ready before the publication deadline."); + await summary("Source monitoring window ended; final publication still requires exact-source verification and migrator publication.\n"); return undefined; } @@ -104,5 +103,6 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(), }); if (checkProof && !proof) throw new Error("Exact-source proof changed before publication."); + if (checkProof && !await readMigratorPublication(process.env.GITHUB_SHA, api)) throw new Error("Exact-source migrator has not published."); if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`); }