perf(ci): admit release candidates and isolate native image capacity (#18869)

Admit full source verification only for exact-source Docker or npm release work, while superseded pending sources complete metadata-only admission. Restart only strictly deferred runs; failed or cancelled checks require an explicit request.

Route canonical master native images to dedicated 32-core Spot profiles and keep source checks on protected verification capacity. Use the four-slot GitHub-hosted publication pool for registry work and signing, with proof waits outside that pool and a fresh source-proof check before tagging. Trim historical checkout blobs and prepared-runner disk cleanup while preserving tag and fork publication behavior.

Validation: release/source and source/artifact regressions, 665 workflow contracts, actionlint, complete latest-head PR CI, and Greptile 5/5 with all comments resolved. Existing CLI and large-file repository timing tests also passed independently; their CI shards received one scoped retry.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

scripts/cloud-source-verification.mjs

@@ -1,9 +1,18 @@
 import { appendFile } from "node:fs/promises";
 import { pathToFileURL } from "node:url";
+import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
 
 const repository = "paperclipai/paperclip";
 const workflowPath = ".github/workflows/cloud-readiness.yml";
 export const sourceVerificationJob = "Cloud source verified v1";
+export class SourceVerificationFailed extends Error {
+  constructor(message, run) {
+    super(message);
+    this.runId = run.id;
+    this.attempt = run.run_attempt;
+  }
+}
+export class SourceVerificationDeferred extends Error {}
 
 function assertSha(sha) {
   if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
@@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) {
 
 // Consume one versioned job, independent of image/migrator availability. A
 // failed image build must not invalidate source checks that already passed.
-export async function readSourceVerification(sha, api) {
+export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) {
   assertSha(sha);
   const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
   if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
@@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) {
   }
   const matches = jobs.filter((job) => job.name === sourceVerificationJob);
   if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
+  await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt));
+  if (failFast) {
+    const failure = jobs.find((job) => job.status === "completed" &&
+      ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) &&
+      job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt);
+    if (failure) {
+      const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+      if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
+      throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run);
+    }
+  }
   const job = matches[0];
   if (job?.status === "completed" && job.conclusion === "success" &&
       job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
@@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) {
     return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
   }
   if (job?.status === "completed" || run.status === "completed") {
-    throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
+    const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+    if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
+    if (job?.conclusion === "skipped" && current.status !== "completed") return undefined;
+    if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission.");
+    throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run);
   }
   return undefined;
 }
 
 // Statuses that say "ask again", not "the answer is no". A release must not be
-// blocked because GitHub returned a gateway error during a 45-minute poll.
+// blocked because GitHub returned a gateway error during a 60-minute poll.
 const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
 
 // A rate-limited read also says "ask again", and GitHub reports both primary
@@ -137,14 +161,20 @@ export function createActionsReader({
 
 export async function waitForSourceVerification(sha, {
   api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
-  timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
+  timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun,
 } = {}) {
   assertSha(sha);
   const deadline = now() + timeoutMs;
+  const requested = new Set();
   log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
   while (now() < deadline) {
-    const result = await readSourceVerification(sha, api);
-    if (result) return result;
+    try {
+      const result = await readSourceVerification(sha, api);
+      if (result) return result;
+    } catch (error) {
+      if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error;
+      await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
+    }
     const remaining = deadline - now();
     if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
   }
@@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
     if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
     // One deadline for both layers: the reader stops retrying when the poll it
     // serves is out of time, instead of extending the wait past its timeout.
-    const timeoutMs = 45 * 60_000;
+    const timeoutMs = 60 * 60_000;
     const deadline = Date.now() + timeoutMs;
     const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });
-    const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });
+    const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined;
+    const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun });
     const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
     console.log(message);
     if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);