fix(ci): reuse cloud source verification for npm canaries (#13233)

Reuse the exact master source-verification result before npm canary publication, removing a duplicate verification matrix while preserving fail-closed release checks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/cloud-readiness.yml

@@ -48,6 +48,33 @@ jobs:
           SOURCE_SHA: ${{ github.sha }}
         run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
 
+  source_verified:
+    # npm canary publication reuses this exact-source verification proof.
+    # Keep it independent of image/migrator availability, and fail closed when
+    # any source check fails, is cancelled, or is skipped.
+    name: Cloud source verified v1
+    needs: [verify]
+    if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: 24
+      - name: Check the source verification consumer
+        run: node --test scripts/cloud-source-verification.test.mjs
+      - name: Record source verification
+        env:
+          SOURCE_SHA: ${{ github.sha }}
+        run: |
+          echo "Cloud source verified v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
+
   ready:
     # Versioned consumer contract. Never add always() or continue-on-error:
     # failed, cancelled, or skipped prerequisites must not report readiness.

.github/workflows/pr-trusted.yml

@@ -335,7 +335,7 @@ jobs:
         run: node --test ./scripts/__tests__/e2e-shard.test.mjs
 
       - name: Test release verify workflow wiring
-        run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs
+        run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/cloud-source-verification.test.mjs
 
       - name: Test standalone package build concurrency
         run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs

.github/workflows/release.yml

@@ -296,10 +296,26 @@ jobs:
           retention-days: 30
 
   verify_canary:
-    if: github.event_name == 'push'
+    name: Reuse exact-source verification
-    uses: ./.github/workflows/release-verify.yml
+    if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
-    with:
+    runs-on: ubuntu-latest
-      ref: ${{ github.sha }}
+    timeout-minutes: 50
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.sha }}
+          persist-credentials: false
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: 24
+      - name: Require successful source checks for this exact master push
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          SOURCE_SHA: ${{ github.sha }}
+        run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA"
 
   publish_canary:
     if: github.event_name == 'push'

doc/cloud-build-readiness.md

@@ -26,11 +26,22 @@ build with the new identity must rebuild layers that depend on the base image;
 later builds can reuse those layers.
 
 Verification and image building run concurrently, outside the full npm release's
-concurrency group. Different commits have independent groups. Source verification
+concurrency group. Different commits have independent groups. The npm canary
-is initially duplicated with the normal npm release: this spends existing hosted
+release reuses `Cloud source verified v1` for the exact master push instead of
-runner capacity to avoid waiting behind an older release. No verification gate is
+starting a second copy of `Release Verify`. This source-only job depends on every
-removed from npm publication. Watch organization-wide runner queues when measuring
+source check but does not wait for Docker or migrator publication. npm canary
-the result.
+publication remains possible when source verification passes and an image build
+fails. Stable releases and candidate-branch betas still run full verification.
+
+The canary consumer requires the expected workflow ID and path, upstream source
+repository, master push event, full SHA, and a successful job in the latest run
+attempt. It checks the run again after reading the jobs to reject a concurrent
+rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled,
+ambiguous, or mismatched proof cannot authorize publication. API failures fail
+closed. If a source check fails, fix it and rerun Cloud readiness before retrying
+the release. Use **Re-run all jobs** when a later attempt did not rerun the source
+proof; an earlier attempt's successful job is not accepted. This avoids duplicate
+test jobs on standard runners. Measure queue time to assess the timing gain.
 
 Release verification spreads the general server suites across ten standard hosted
 runners, with the long chat suite split separately across three jobs. Each server
@@ -67,8 +78,9 @@ successfully verified a real master commit.
 ## Timing and rollout
 
 The reusable Runner chaos workflow scopes concurrency to the caller workflow
-and source ref. Cloud readiness and the npm release can verify the same commit
+and source ref. Cloud readiness, stable verification, and standalone evals can
-at the same time. They must not cancel each other's required test job.
+verify the same commit at the same time. They must not cancel each other's
+required test job.
 
 Measure the complete path from a master merge to a healthy target running that
 exact commit. Keep readiness and deployment as separate milestones:

scripts/__tests__/release-verify-workflow.test.mjs

@@ -35,13 +35,16 @@ test("chaos verification isolates callers that verify the same source commit", (
   assert.match(chaosWorkflow, /cancel-in-progress: true/);
 });
 
-test("release workflow delegates stable and canary verification to the reusable workflow", () => {
+test("canary reuses exact-source proof while stable keeps full verification", () => {
   const releaseWorkflow = readWorkflow("release.yml");
-
+  const canary = releaseWorkflow.split("  verify_canary:\n")[1].split("\n  publish_canary:")[0];
-  assert.match(
+  assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
-    releaseWorkflow,
+  assert.match(canary, /actions: read/);
-    /verify_canary:\n\s+if: github\.event_name == 'push'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/,
+  assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
-  );
+  assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
+  assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);
+  assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/);
+  assert.match(releaseWorkflow, /publish_canary:\n\s+if: github\.event_name == 'push'\n\s+needs: verify_canary/);
   // The stable lane is gated on the stable channel since the nightly lane
   // was added; a `needs:` line (for example a preflight job) may sit between
   // the gate and the delegation.
@@ -57,6 +60,17 @@ test("release workflow delegates stable and canary verification to the reusable
   );
 });
 
+test("source proof requires every source check and does not wait on image publication", () => {
+  const readiness = readWorkflow("cloud-readiness.yml");
+  const proof = readiness.split("  source_verified:\n")[1].split("\n  ready:")[0];
+  assert.match(proof, /name: Cloud source verified v1/);
+  assert.match(proof, /needs: \[verify\]/);
+  assert.match(proof, /node --test scripts\/cloud-source-verification.test.mjs/);
+  assert.match(proof, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
+  assert.doesNotMatch(proof, /always\(\)|continue-on-error|needs:.*(?:image|artifacts)/);
+  assert.match(readiness.split("  ready:\n")[1], /needs: \[verify, image, artifacts\]/);
+});
+
 test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => {
   const dockerfile = readFileSync(
     path.join(repoRoot, "docker/Dockerfile.onboard-smoke"),

scripts/cloud-source-verification.mjs

@@ -0,0 +1,99 @@
+import { appendFile } from "node:fs/promises";
+import { pathToFileURL } from "node:url";
+
+const repository = "paperclipai/paperclip";
+const workflowPath = ".github/workflows/cloud-readiness.yml";
+export const sourceVerificationJob = "Cloud source verified v1";
+
+function assertSha(sha) {
+  if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
+}
+
+function trustedRun(run, sha, workflowId) {
+  return run.workflow_id === workflowId && run.path === workflowPath &&
+    run.repository?.full_name === repository && run.head_repository?.full_name === repository &&
+    run.head_sha === sha && run.head_branch === "master" && run.event === "push" &&
+    Number.isSafeInteger(run.id) && run.id > 0 &&
+    Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0;
+}
+
+// Consume one versioned job, independent of image/migrator availability. A
+// failed image build must not invalidate source checks that already passed.
+export async function readSourceVerification(sha, api) {
+  assertSha(sha);
+  const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
+  if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
+    throw new Error("Cloud readiness workflow identity does not match.");
+  }
+  const listing = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?head_sha=${sha}&event=push&branch=master&per_page=100`);
+  if (!Array.isArray(listing.workflow_runs) || !Number.isSafeInteger(listing.total_count) ||
+      listing.total_count < 0 || listing.total_count > 100 || listing.workflow_runs.length !== listing.total_count) {
+    throw new Error("Cloud readiness run listing is incomplete.");
+  }
+  const run = listing.workflow_runs.filter((candidate) => trustedRun(candidate, sha, workflow.id))
+    .sort((a, b) => b.id - a.id)[0];
+  if (!run) return undefined;
+
+  // Attempt-specific jobs prevent an earlier successful attempt from blessing
+  // a later rerun. Keep pagination even though today's matrix fits one page.
+  const jobs = [];
+  for (let page = 1; page <= 10; page += 1) {
+    const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
+    if (!Array.isArray(batch.jobs)) throw new Error("Cloud readiness job listing is malformed.");
+    jobs.push(...batch.jobs);
+    if (batch.jobs.length < 100) break;
+    if (page === 10) throw new Error("Cloud readiness job listing is incomplete.");
+  }
+  const matches = jobs.filter((job) => job.name === sourceVerificationJob);
+  if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
+  const job = matches[0];
+  if (job?.status === "completed" && job.conclusion === "success" &&
+      job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
+    // Re-read the run after its jobs: a rerun that started during polling must
+    // not let the previous attempt through. Changes are retried next poll.
+    const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
+    if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
+    return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
+  }
+  if (job?.status === "completed" || run.status === "completed") {
+    throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
+  }
+  return undefined;
+}
+
+export async function waitForSourceVerification(sha, {
+  api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
+} = {}) {
+  assertSha(sha);
+  const deadline = now() + timeoutMs;
+  log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
+  while (now() < deadline) {
+    const result = await readSourceVerification(sha, api);
+    if (result) return result;
+    const remaining = deadline - now();
+    if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
+  }
+  throw new Error(`Cloud source verification timed out for ${sha}. Rerun Cloud readiness before retrying the release.`);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+  try {
+    if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
+    const api = async (path) => {
+      const response = await fetch(`https://api.github.com${path}`, {
+        headers: { Authorization: `Bearer ${process.env.GITHUB_TOKEN}`, Accept: "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28" },
+        signal: AbortSignal.timeout(30_000), redirect: "error",
+      });
+      if (!response.ok) throw new Error(`GitHub Actions read failed (HTTP ${response.status}).`);
+      return response.json();
+    };
+    const proof = await waitForSourceVerification(process.argv[2], { api });
+    const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
+    console.log(message);
+    if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);
+  } catch (error) {
+    console.error(error.message);
+    process.exitCode = 1;
+  }
+}

scripts/cloud-source-verification.test.mjs

@@ -0,0 +1,125 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
+
+const sha = "a".repeat(40);
+const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
+const baseRun = {
+  id: 456, workflow_id: workflow.id, path: workflow.path, run_attempt: 2,
+  repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" },
+  head_sha: sha, head_branch: "master", event: "push", status: "in_progress", conclusion: null,
+};
+const baseJob = { id: 789, name: sourceVerificationJob, run_id: 456, run_attempt: 2, head_sha: sha, status: "completed", conclusion: "success" };
+
+function fixture({ runs = [baseRun], jobs = [baseJob], current = baseRun, total, workflowRecord = workflow } = {}) {
+  const calls = [];
+  const api = async (path) => {
+    calls.push(path);
+    if (path.endsWith("/workflows/cloud-readiness.yml")) return workflowRecord;
+    if (path.includes("/workflows/123/runs?")) return { total_count: total ?? runs.length, workflow_runs: runs };
+    if (path.includes("/attempts/")) {
+      assert.ok(path.includes(`/attempts/${runs.at(-1).run_attempt}/jobs?`));
+      const page = Number(new URL("https://api.github.com" + path).searchParams.get("page"));
+      return { jobs: jobs.slice((page - 1) * 100, page * 100) };
+    }
+    if (path.endsWith("/runs/456")) return current;
+    throw new Error(`Unexpected API path: ${path}`);
+  };
+  return { api, calls };
+}
+
+test("source proof passes while image work is still running, or has failed", async () => {
+  for (const overrides of [{}, { status: "completed", conclusion: "failure" }]) {
+    const run = { ...baseRun, ...overrides };
+    const { api, calls } = fixture({ runs: [run], current: run });
+    assert.deepEqual(await readSourceVerification(sha, api), { sha, runId: 456, attempt: 2, jobId: 789 });
+    assert.ok(calls.some((path) => path.includes(`head_sha=${sha}&event=push&branch=master`)));
+    assert.ok(calls.some((path) => path.includes("/attempts/2/jobs?")));
+  }
+});
+
+test("only the expected workflow, repository, master push, and full SHA can supply proof", async () => {
+  for (const overrides of [
+    { workflow_id: 999 }, { path: ".github/workflows/pr.yml" },
+    { repository: { full_name: "other/paperclip" } }, { head_repository: { full_name: "fork/paperclip" } },
+    { head_sha: "b".repeat(40) }, { head_branch: "feature" }, { event: "workflow_dispatch" },
+    { run_attempt: undefined }, { run_attempt: 0 },
+  ]) {
+    const { api, calls } = fixture({ runs: [{ ...baseRun, ...overrides }] });
+    assert.equal(await readSourceVerification(sha, api), undefined, JSON.stringify(overrides));
+    assert.equal(calls.length, 2);
+  }
+});
+
+test("a newer pending run cannot fall back to an older successful run", async () => {
+  const newer = { ...baseRun, id: 457, run_attempt: 1 };
+  const { api } = fixture({ runs: [baseRun, newer], jobs: [] });
+  assert.equal(await readSourceVerification(sha, api), undefined);
+});
+
+test("job identities and terminal failures fail closed", async () => {
+  for (const overrides of [
+    { head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 },
+    { conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" },
+  ]) {
+    const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] });
+    await assert.rejects(readSourceVerification(sha, api), /did not pass/);
+  }
+});
+
+test("a rerun racing the jobs read cannot reuse the previous attempt", async () => {
+  const { api } = fixture({ current: { ...baseRun, run_attempt: 3 } });
+  assert.equal(await readSourceVerification(sha, api), undefined);
+});
+
+test("the versioned proof must exist and be unique", async () => {
+  for (const jobs of [[], [{ ...baseJob, name: "Cloud deployable v1" }]]) {
+    await assert.rejects(readSourceVerification(sha, fixture({ runs: [{ ...baseRun, status: "completed" }], jobs }).api), /did not pass/);
+  }
+  await assert.rejects(readSourceVerification(sha, fixture({ jobs: [baseJob, baseJob] }).api), /ambiguous/);
+});
+
+test("proof may appear after the first page of jobs", async () => {
+  const jobs = [...Array.from({ length: 100 }, (_, index) => ({ ...baseJob, name: `test ${index}` })), baseJob];
+  const { api, calls } = fixture({ jobs });
+  assert.equal((await readSourceVerification(sha, api)).jobId, 789);
+  assert.ok(calls.some((path) => path.endsWith("page=2")));
+});
+
+test("incomplete discovery and API failures cannot bless a release", async () => {
+  for (const total of [2, 101, -1]) {
+    await assert.rejects(readSourceVerification(sha, fixture({ total }).api), /incomplete/);
+  }
+  await assert.rejects(readSourceVerification(sha, fixture({ workflowRecord: { ...workflow, path: ".github/workflows/pr.yml" } }).api), /identity/);
+  await assert.rejects(readSourceVerification(sha, async () => { throw new Error("API unavailable"); }), /API unavailable/);
+});
+
+test("a missing or pending source proof waits and has a bounded deadline", async () => {
+  let time = 0;
+  let polls = 0;
+  const { api } = fixture({ runs: [] });
+  await assert.rejects(waitForSourceVerification(sha, {
+    api: async (path) => { if (path.endsWith(".yml")) polls += 1; return api(path); },
+    now: () => time, sleep: async (ms) => { time += ms; }, timeoutMs: 50, intervalMs: 30, log: () => {},
+  }), /timed out/);
+  assert.equal(time, 50);
+  assert.equal(polls, 2);
+});
+
+test("pending verification succeeds when its exact job completes", async () => {
+  let time = 0;
+  const pending = fixture({ jobs: [{ ...baseJob, status: "in_progress", conclusion: null }] });
+  const passed = fixture();
+  const proof = await waitForSourceVerification(sha, {
+    api: (path) => (time ? passed.api : pending.api)(path), now: () => time,
+    sleep: async (ms) => { time += ms; }, timeoutMs: 100, intervalMs: 30, log: () => {},
+  });
+  assert.equal(proof.sha, sha);
+  assert.equal(time, 30);
+});
+
+test("malformed source refs are rejected before any request", async () => {
+  for (const ref of ["master", "a".repeat(7), "A".repeat(40), undefined]) {
+    await assert.rejects(readSourceVerification(ref, () => assert.fail("must not request")), /full lowercase/);
+  }
+});