perf(ci): admit release candidates and isolate native image capacity (#18869)

Admit full source verification only for exact-source Docker or npm release work, while superseded pending sources complete metadata-only admission. Restart only strictly deferred runs; failed or cancelled checks require an explicit request.

Route canonical master native images to dedicated 32-core Spot profiles and keep source checks on protected verification capacity. Use the four-slot GitHub-hosted publication pool for registry work and signing, with proof waits outside that pool and a fresh source-proof check before tagging. Trim historical checkout blobs and prepared-runner disk cleanup while preserving tag and fork publication behavior.

Validation: release/source and source/artifact regressions, 665 workflow contracts, actionlint, complete latest-head PR CI, and Greptile 5/5 with all comments resolved. Existing CLI and large-file repository timing tests also passed independently; their CI shards received one scoped retry.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

scripts/__tests__/release-verify-workflow.test.mjs

@@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati
     "the registry-only manifest merge does not need a Docker Hub mirror");
 });
 
+test("source proof gates publication without occupying a publication runner", () => {
+  const workflow = readWorkflow("docker.yml");
+  const observer = workflow.split("  source_verification:\n")[1].split("  reuse_canary_image:")[0];
+  const gate = workflow.split("  publication_verification:\n")[1].split("  merge-and-push:")[0];
+  const publisher = workflow.split("  merge-and-push:\n")[1].split("  promote_canary_channel:")[0];
+  assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/);
+  assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/);
+  assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/);
+  assert.match(gate, /needs: \[build-and-push, source_verification\]/);
+  assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/);
+  assert.doesNotMatch(publisher, /--require-proof|actions: write/);
+  assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push"));
+  assert.match(publisher, /actions: read/);
+  assert.match(publisher, /timeout-minutes: 20/);
+  const expression = publisher.match(/^    if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build");
+  for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [
+    ["refs/heads/master", "true", "skipped", true],
+    ["refs/heads/master", "false", "skipped", false],
+    ["refs/heads/master", undefined, "skipped", false],
+    ["refs/heads/master", "false", "success", true],
+    ["refs/heads/master", "false", "failure", false],
+    ["refs/tags/v1", undefined, "skipped", true],
+    ["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"],
+    ["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"],
+  ]) {
+    const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master";
+    assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: {
+      build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } },
+      publication_verification: { result: fallback },
+    } }), expected);
+  }
+});
+
 test("chaos verification isolates callers that verify the same source commit", () => {
   const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
   const group = chaosWorkflow.match(/^  group: (.+)$/m)?.[1];
@@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", (
   assert.match(chaosWorkflow, /cancel-in-progress: true/);
 });
 
-test("canary reuses exact-source proof while stable keeps full verification", () => {
+test("canary can admit deferred exact-source checks while stable keeps full verification", () => {
   const releaseWorkflow = readWorkflow("release.yml");
   const canary = releaseWorkflow.match(/  verify_canary:\n[\s\S]*?(?=\n  [a-z_]+:)/)[0];
-  assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
-  assert.match(canary, /actions: read/);
+  assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
+  assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/);
   assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
   assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
   assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);

scripts/cloud-release-admission.mjs

@@ -0,0 +1,123 @@
+import { appendFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+
+const repository = "paperclipai/paperclip";
+const repositoryId = 1170821064;
+const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]);
+export const sourceAdmissionJob = "Cloud candidate admission v1";
+export const migratorAdmissionJob = "Cloud image candidate admission v1";
+
+function assertSha(sha) {
+  if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA.");
+}
+
+function trusted(run, sha, workflow) {
+  return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha &&
+    run.repository?.id === repositoryId && run.head_repository?.id === repositoryId &&
+    run.repository?.full_name === repository && run.head_repository?.full_name === repository &&
+    run.head_branch === "master" && run.event === "push" &&
+    Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0;
+}
+
+async function producer(file, sha, api, status) {
+  assertSha(sha);
+  if (!workflows.has(file)) throw new Error("Unknown release workflow.");
+  const workflow = await api(`/repos/${repository}/actions/workflows/${file}`);
+  if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch.");
+  const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`);
+  if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 ||
+      list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing.");
+  const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0];
+  return { workflow, run };
+}
+
+// A retry is an explicit request for this exact immutable source. Normal push
+// work starts only for a candidate admitted by an image build or npm canary.
+export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) {
+  assertSha(sha);
+  if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt.");
+  if (!["push", "workflow_dispatch"].includes(event)) return false;
+  if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true;
+  for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) {
+    const { workflow, run } = await producer(file, sha, api, "in_progress");
+    if (!run) continue;
+    const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+    if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true;
+  }
+  return false;
+}
+
+export function deferredProducer(jobs, run, admissionJob) {
+  const admission = jobs.filter(job => job.name === admissionJob);
+  const remaining = jobs.filter(job => job.name !== admissionJob);
+  return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 &&
+    remaining.every(job => job.status === "completed" && job.conclusion === "skipped") &&
+    jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha);
+}
+
+// Only skipped admission is restartable here. A failed check or build never
+// becomes an automatic retry, and every write revalidates canonical identity.
+export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) {
+  if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract.");
+  const { workflow, run } = await producer(file, sha, api);
+  if (!run) return "pending";
+  const key = `${run.id}/${run.run_attempt}`;
+  if (requested.has(key)) return "pending";
+  const jobs = [];
+  let total;
+  for (let page = 1; page <= 10; page++) {
+    const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
+    if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list.");
+    if ((total !== undefined && total !== batch.total_count) ||
+        batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list.");
+    total = batch.total_count;
+    jobs.push(...batch.jobs);
+    if (page * 100 >= total) break;
+    if (page === 10) throw new Error("Incomplete release producer job list.");
+  }
+  if (run.status !== "completed") {
+    const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha);
+    // Admission itself succeeds for both outcomes. Only an actual build is
+    // evidence that its output admitted work; queued jobs may still be skipped.
+    return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped")
+      ? "admitted" : "pending";
+  }
+  if (!deferredProducer(jobs, run, admissionJob)) return "finished";
+  const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+  if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending";
+  try {
+    await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`);
+  } catch (error) {
+    // Docker and npm may select the same source at the same time. A rejected
+    // duplicate write is harmless only when a fresh read proves a retry began.
+    const latest = await api(`/repos/${repository}/actions/runs/${run.id}`);
+    if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error;
+    if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt &&
+      ["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) {
+      // GitHub may reject a duplicate before its read API shows the retry.
+      // Leave it pending for the next poll; this is not proof of admission.
+      if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending";
+      throw error;
+    }
+  }
+  requested.add(key);
+  return "pending";
+}
+
+export function createReleaseRerunner(token, fetchImpl = fetch) {
+  return async path => {
+    if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path.");
+    const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error",
+      headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
+    if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status });
+  };
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required.");
+  const { createActionsReader } = await import("./cloud-source-verification.mjs");
+  const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }),
+    event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" });
+  console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source.");
+  if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`);
+}

scripts/cloud-source-verification.mjs

@@ -1,9 +1,18 @@
 import { appendFile } from "node:fs/promises";
 import { pathToFileURL } from "node:url";
+import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
 
 const repository = "paperclipai/paperclip";
 const workflowPath = ".github/workflows/cloud-readiness.yml";
 export const sourceVerificationJob = "Cloud source verified v1";
+export class SourceVerificationFailed extends Error {
+  constructor(message, run) {
+    super(message);
+    this.runId = run.id;
+    this.attempt = run.run_attempt;
+  }
+}
+export class SourceVerificationDeferred extends Error {}
 
 function assertSha(sha) {
   if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
@@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) {
 
 // Consume one versioned job, independent of image/migrator availability. A
 // failed image build must not invalidate source checks that already passed.
-export async function readSourceVerification(sha, api) {
+export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) {
   assertSha(sha);
   const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
   if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
@@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) {
   }
   const matches = jobs.filter((job) => job.name === sourceVerificationJob);
   if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
+  await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt));
+  if (failFast) {
+    const failure = jobs.find((job) => job.status === "completed" &&
+      ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) &&
+      job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt);
+    if (failure) {
+      const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+      if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
+      throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run);
+    }
+  }
   const job = matches[0];
   if (job?.status === "completed" && job.conclusion === "success" &&
       job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
@@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) {
     return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
   }
   if (job?.status === "completed" || run.status === "completed") {
-    throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
+    const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+    if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
+    if (job?.conclusion === "skipped" && current.status !== "completed") return undefined;
+    if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission.");
+    throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run);
   }
   return undefined;
 }
 
 // Statuses that say "ask again", not "the answer is no". A release must not be
-// blocked because GitHub returned a gateway error during a 45-minute poll.
+// blocked because GitHub returned a gateway error during a 60-minute poll.
 const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
 
 // A rate-limited read also says "ask again", and GitHub reports both primary
@@ -137,14 +161,20 @@ export function createActionsReader({
 
 export async function waitForSourceVerification(sha, {
   api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
-  timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
+  timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun,
 } = {}) {
   assertSha(sha);
   const deadline = now() + timeoutMs;
+  const requested = new Set();
   log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
   while (now() < deadline) {
-    const result = await readSourceVerification(sha, api);
-    if (result) return result;
+    try {
+      const result = await readSourceVerification(sha, api);
+      if (result) return result;
+    } catch (error) {
+      if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error;
+      await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
+    }
     const remaining = deadline - now();
     if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
   }
@@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
     if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
     // One deadline for both layers: the reader stops retrying when the poll it
     // serves is out of time, instead of extending the wait past its timeout.
-    const timeoutMs = 45 * 60_000;
+    const timeoutMs = 60 * 60_000;
     const deadline = Date.now() + timeoutMs;
     const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });
-    const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });
+    const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined;
+    const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun });
     const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
     console.log(message);
     if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);

scripts/cloud-source-verification.test.mjs

@@ -1,6 +1,10 @@
 import assert from "node:assert/strict";
-import test from "node:test";
-import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
+import test, { describe } from "node:test";
+import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
+
+import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
+
+import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs";
 
 const sha = "a".repeat(40);
 const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
@@ -62,7 +66,8 @@ test("job identities and terminal failures fail closed", async () => {
     { head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 },
     { conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" },
   ]) {
-    const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] });
+    const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }],
+      current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun });
     await assert.rejects(readSourceVerification(sha, api), /did not pass/);
   }
 });
@@ -231,3 +236,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy
   assert.equal(seen.length, 1);
   assert.deepEqual(waits, []);
 });
+
+
+describe("image build source monitor", () => {
+  const sha = "a".repeat(40);
+  const runId = 50;
+  const attempt = 1;
+  const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
+  const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1,
+    repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" },
+    head_sha: sha, head_branch: "master", event: "push", status: "in_progress" };
+  const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push",
+    repository: { id: 1170821064 }, head_repository: { id: 1170821064 },
+    path: ".github/workflows/docker.yml", status: "in_progress" };
+  const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1,
+    head_sha: sha, status: "completed", conclusion: "success" };
+
+  function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) {
+    const cancellations = [];
+    const messages = [];
+    const api = async path => {
+      if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow;
+      if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 };
+      if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs };
+      if (path.endsWith("/runs/99")) return current;
+      if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length };
+      if (path.endsWith(`/runs/${runId}`)) return image;
+      throw new Error(`Unexpected request: ${path}`);
+    };
+    return { sha, runId, attempt, api, cancel: async path => cancellations.push(path),
+      cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) };
+  }
+
+  test("successful exact-source proof permits publication without canceling image work", async () => {
+    const f = fixture();
+    assert.equal((await watchReleaseSource(f)).jobId, proof.id);
+    assert.deepEqual(f.cancellations, []);
+  });
+
+  test("a required check failing cancels the same image attempt before siblings finish", async () => {
+    const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] });
+    await assert.rejects(watchReleaseSource(f), /General tests/);
+    assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]);
+  });
+
+  test("a superseded readiness attempt cancels image work with no deployable future", async () => {
+    const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } });
+    await assert.rejects(watchReleaseSource(f), /did not pass/);
+    assert.equal(f.cancellations.length, 1);
+  });
+
+  test("a rerun racing a failed job is retried rather than canceling the image", async () => {
+    const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }],
+      current: { ...sourceRun, run_attempt: 2 } });
+    let time = 0;
+    const api = f.api;
+    f.api = async path => time > 0 ? fixture().api(path) : api(path);
+    const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } });
+    assert.equal(result.jobId, proof.id);
+    assert.deepEqual(f.cancellations, []);
+  });
+
+  test("a transport failure does not issue a destructive cancellation", async () => {
+    const f = fixture();
+    await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/);
+    assert.deepEqual(f.cancellations, []);
+  });
+
+  test("the observation window ends without cancelling healthy pending verification", async () => {
+    const f = fixture({ missing: true });
+    let time = 0;
+    const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 });
+    assert.equal(proof, undefined);
+    assert.equal(time, 20);
+    assert.deepEqual(f.cancellations, []);
+    assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification")));
+  });
+
+  test("final publication requires proof even after the early observation window ends", async () => {
+    const f = fixture({ missing: true });
+    let time = 0;
+    await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time,
+      sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/);
+    assert.deepEqual(f.cancellations, []);
+    assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id);
+  });
+
+  test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => {
+    for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" },
+      { path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 },
+      { repository: { id: 42 } }, { head_repository: { id: 42 } }]) {
+      const f = fixture({ image: { ...imageRun, ...change } });
+      await assert.rejects(cancelImageRun(f), /different image workflow identity/);
+      assert.deepEqual(f.cancellations, []);
+    }
+  });
+
+  test("queue measurement does not count running time or skipped dependency jobs", () => {
+    const created_at = "2020-01-01T00:00:00Z";
+    const started_at = "2020-01-01T00:00:20Z";
+    const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" };
+    const now = Date.parse(created_at) + 120_000;
+    assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]);
+    assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120);
+  });
+
+  test("a runner wait over 60 seconds produces an Actions warning and summary", async () => {
+    const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] });
+    await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") });
+    assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s")));
+    assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64")));
+  });
+});
+
+describe("release candidate admission", () => {
+  const repository = { id: 1170821064, full_name: "paperclipai/paperclip" };
+  const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 };
+  function run(file, change = {}) {
+    return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`,
+      repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push",
+      run_attempt: 1, status: "in_progress", conclusion: null, ...change };
+  }
+  function job(run, name, conclusion = "skipped") {
+    return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha,
+      name, status: "completed", conclusion };
+  }
+  function fixture(file = "cloud-readiness.yml", change = {}) {
+    const producer = run(file, { status: "completed", conclusion: "success", ...change });
+    const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob;
+    const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] },
+      current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission };
+    f.api = async path => {
+      f.calls.push(path);
+      const url = new URL(`https://api.github.com${path}`);
+      const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/);
+      if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` };
+      const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/);
+      if (listMatch) {
+        const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1]));
+        const runs = f.runs[name] ?? [];
+        return { workflow_runs: runs, total_count: f.total ?? runs.length };
+      }
+      const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/);
+      if (jobMatch) {
+        const page = Number(url.searchParams.get("page") ?? "1");
+        const jobs = f.jobs[Number(jobMatch[1])] ?? [];
+        return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) };
+      }
+      const currentMatch = path.match(/\/runs\/(\d+)$/);
+      if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1]));
+      throw new Error(`Unexpected admission API path: ${path}`);
+    };
+    f.rerun = async path => f.writes.push(path);
+    return f;
+  }
+
+  test("only a running image or npm release admits a normal push", async () => {
+    for (const file of ["docker.yml", "release.yml"]) {
+      const f = fixture();
+      assert.equal(await releaseAdmitted(sha, f), false);
+      f.runs[file] = [run(file)];
+      assert.equal(await releaseAdmitted(sha, f), true);
+      assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml");
+      for (const status of ["queued", "pending", "completed"]) {
+        f.runs[file] = [run(file, { status })];
+        assert.equal(await releaseAdmitted(sha, f), false, status);
+      }
+    }
+  });
+
+  test("manual and retry requests admit exact source without consumer discovery", async () => {
+    const api = () => assert.fail("explicit request needs no discovery");
+    assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true);
+    assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true);
+    assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false);
+    for (const attempt of [0, "2"]) {
+      await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/);
+    }
+  });
+
+  test("forks, stale identities, tags, and non-push consumers do not admit work", async () => {
+    for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } },
+      { repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" },
+      { workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" },
+      { id: 0 }, { run_attempt: 0 }]) {
+      const f = fixture();
+      f.runs["docker.yml"] = [run("docker.yml", change)];
+      assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change));
+    }
+    const f = fixture();
+    const active = run("docker.yml");
+    f.runs["docker.yml"] = [active];
+    f.current[active.id] = { ...active, run_attempt: 2 };
+    assert.equal(await releaseAdmitted(sha, f), false);
+  });
+
+  test("incomplete discovery and transport failure fail closed", async () => {
+    for (const total of [1, 101, -1]) {
+      const f = fixture();
+      f.total = total;
+      await assert.rejects(releaseAdmitted(sha, f), /Incomplete/);
+      assert.deepEqual(f.writes, []);
+    }
+    await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/);
+    await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/);
+  });
+
+  test("a selected consumer restarts only a strictly deferred producer, once", async () => {
+    for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) {
+      const f = fixture(file);
+      assert.equal(await requestDeferredProducer(f), "pending");
+      assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]);
+      await requestDeferredProducer(f);
+      assert.equal(f.writes.length, 1);
+    }
+  });
+
+  test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => {
+    for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; },
+      jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); },
+      jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; },
+      jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) {
+      const f = fixture();
+      change(f.jobs[1123]);
+      assert.equal(await requestDeferredProducer(f), "finished");
+      assert.deepEqual(f.writes, []);
+    }
+    const f = fixture();
+    for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) {
+      f.current[1123] = { ...f.runs[f.file][0], ...change };
+      assert.equal(await requestDeferredProducer(f), "pending");
+      assert.deepEqual(f.writes, []);
+    }
+  });
+
+  test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => {
+    for (const conclusion of ["cancelled", "failure", "timed_out"]) {
+      const f = fixture("cloud-readiness.yml", { conclusion });
+      assert.equal(await requestDeferredProducer(f), "finished");
+      assert.deepEqual(f.writes, []);
+    }
+  });
+
+  test("a truncated job page cannot hide a failed check and trigger a retry", async () => {
+    const f = fixture();
+    f.jobTotal = 3;
+    await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/);
+    assert.deepEqual(f.writes, []);
+    f.jobTotal = "2";
+    await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/);
+    assert.deepEqual(f.writes, []);
+  });
+
+  test("producer retries reject arbitrary workflow and admission contracts", async () => {
+    const f = fixture();
+    for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) {
+      await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/);
+    }
+    assert.deepEqual(f.calls, []);
+  });
+
+  test("a deferred completion racing the run listing does not become a source failure", async () => {
+    const f = fixture();
+    const completed = f.runs[f.file][0];
+    f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }];
+    f.current[completed.id] = completed;
+    await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
+    delete f.current[completed.id];
+    assert.equal(await readSourceVerification(sha, f.api), undefined);
+  });
+
+  test("a competing consumer's accepted retry does not fail the losing consumer", async () => {
+    const f = fixture();
+    f.rerun = async () => {
+      f.current[1123] = run(f.file, { run_attempt: 2 });
+      throw new Error("duplicate restart rejected");
+    };
+    assert.equal(await requestDeferredProducer(f), "pending");
+    assert.equal(f.requested.size, 1);
+    const rejected = fixture();
+    rejected.rerun = async () => { throw new Error("token denied"); };
+    await assert.rejects(requestDeferredProducer(rejected), /token denied/);
+    assert.equal(rejected.requested.size, 0);
+  });
+
+  test("a queued migrator is pending until its build actually starts", async () => {
+    const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null });
+    const producer = f.runs[f.file][0];
+    const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] };
+    f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build];
+    assert.equal(await requestDeferredProducer(f), "pending");
+    build.status = "in_progress";
+    assert.equal(await requestDeferredProducer(f), "admitted");
+    build.run_attempt = 2;
+    assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work");
+    assert.deepEqual(f.writes, []);
+  });
+
+  test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => {
+    for (const status of [409, 422, 403]) {
+      const f = fixture();
+      f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); };
+      if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/);
+      else assert.equal(await requestDeferredProducer(f), "pending");
+      assert.equal(f.requested.size, 0);
+    }
+  });
+
+  test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => {
+    const f = fixture();
+    await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
+    let time = 0;
+    const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {},
+      sleep: async ms => {
+        time += ms;
+        const selected = run(f.file, { run_attempt: 2 });
+        f.runs[f.file] = [selected];
+        f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")];
+      } });
+    assert.equal(proof.attempt, 2);
+    assert.equal(time, 10);
+    assert.equal(f.writes.length, 1);
+  });
+
+  test("the image monitor admits a late migrator even when source proof is ready", async () => {
+    const f = fixture("cloud-migrator-artifacts.yml");
+    const source = run("cloud-readiness.yml");
+    f.runs["cloud-readiness.yml"] = [source];
+    f.jobs[source.id] = [job(source, sourceVerificationJob, "success")];
+    const image = run("docker.yml");
+    f.runs["docker.yml"] = [image];
+    f.jobs[image.id] = [];
+    let time = 0;
+    const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"),
+      now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => {
+        time += ms;
+        const selected = run(f.file, { run_attempt: 2 });
+        f.runs[f.file] = [selected];
+        f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }];
+      } });
+    assert.equal(proof.sha, sha);
+    assert.equal(time, 10);
+    assert.equal(f.writes.length, 1);
+  });
+
+  test("the retry transport is a bounded POST to a canonical run", async () => {
+    const calls = [];
+    const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; });
+    await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/);
+    await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun");
+    assert.equal(calls.length, 1);
+    assert.equal(calls[0].options.method, "POST");
+    assert.equal(calls[0].options.redirect, "error");
+  });
+});

scripts/watch-release-source.mjs

@@ -0,0 +1,108 @@
+import { appendFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
+import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
+
+const repository = "paperclipai/paperclip";
+
+export function runnerWaits(jobs, now = Date.now()) {
+  return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) &&
+    job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({
+    job: job.name, id: job.id,
+    seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)),
+  })).filter(job => Number.isFinite(job.seconds));
+}
+
+// The write is limited to this canonical master Docker run. Never cancel
+// another workflow, a tag release, or a run selected from untrusted input.
+export async function cancelImageRun({ api, cancel, runId, attempt, sha }) {
+  const run = await api(`/repos/${repository}/actions/runs/${runId}`);
+  if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" ||
+      run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 ||
+      run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) {
+    throw new Error("Refusing to cancel a different image workflow identity.");
+  }
+  if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`);
+}
+
+async function cancelFailedVerification({ api, cancel, sha }, failure) {
+  const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`);
+  // A concurrent retry owns the new attempt; a late observer leaves it alone.
+  if (run.run_attempt !== failure.attempt) return;
+  if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" ||
+      run.repository?.full_name !== repository || run.head_repository?.full_name !== repository ||
+      run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed.");
+  if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`);
+}
+
+export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now,
+  sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log,
+  timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) {
+  if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 ||
+      !Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity.");
+  const deadline = now() + timeoutMs;
+  const warned = new Set();
+  const requested = new Set();
+  let migratorAdmitted = !rerun;
+  const reportWaits = async jobs => {
+    for (const wait of runnerWaits(jobs, now())) {
+      if (wait.seconds > 60 && !warned.has(wait.id)) {
+        warned.add(wait.id);
+        log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`);
+        await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`);
+      }
+    }
+  };
+  while (now() < deadline) {
+    if (!migratorAdmitted) {
+      migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending";
+    }
+    const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`);
+    if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing.");
+    await reportWaits(batch.jobs);
+    try {
+      const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits });
+      if (proof && migratorAdmitted) {
+        await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`);
+        return proof;
+      }
+    } catch (error) {
+      if (error instanceof SourceVerificationDeferred && rerun) {
+        await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
+      } else {
+        if (error instanceof SourceVerificationFailed) {
+          await cancelFailedVerification({ api, cancel, sha }, error);
+          await cancelImageRun({ api, cancel, runId, attempt, sha });
+        }
+        throw error;
+      }
+    }
+    await sleep(Math.min(intervalMs, Math.max(0, deadline - now())));
+  }
+  // This early failure observer is not the publication deadline. Healthy
+  // checks can outlive a Spot worker; a separate exact-source gate still
+  // waits for proof before publishing any image.
+  if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline.");
+  await summary("Source monitoring window ended; final publication still requires exact-source verification.\n");
+  return undefined;
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  const token = process.env.GITHUB_TOKEN;
+  if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required.");
+  const requireProof = process.argv.includes("--require-proof");
+  const checkProof = process.argv.includes("--check-proof");
+  const api = createActionsReader({ token });
+  const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT),
+    api,
+    rerun: createReleaseRerunner(token),
+    cancel: async path => {
+      const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error",
+        headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
+      if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`);
+    },
+    summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(),
+  });
+  if (checkProof && !proof) throw new Error("Exact-source proof changed before publication.");
+  if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`);
+}