diff --git a/.github/scripts/tests/cloud-readiness.test.mjs b/.github/scripts/tests/cloud-readiness.test.mjs index c8137f7dd6d..26c3acb435d 100644 --- a/.github/scripts/tests/cloud-readiness.test.mjs +++ b/.github/scripts/tests/cloud-readiness.test.mjs @@ -18,6 +18,36 @@ test("retirement preserves exact-source verification without issuing legacy depl assert.equal(existsSync(new URL("../../workflows/docker-cloud.yml", import.meta.url)), false); }); +// Owner decision (2026-11-12): the Runner chaos evals run on every admitted +// master commit and report, but are not a prerequisite of the source proof +// that image publication and Cloud discovery wait for. +test("Runner chaos evals still run but stay out of the publication gate", async () => { + const { runnerChaosEvalsJob } = await import("../../../scripts/cloud-source-verification.mjs"); + const jobs = Object.fromEntries([...workflow.split(/^jobs:\n/m)[1].matchAll(/^ ([a-z_]+):\n([\s\S]*?)(?=^ [a-z_]+:\n|(?![\s\S]))/gm)] + .map(([, name, body]) => [name, body])); + assert.deepEqual(Object.keys(jobs), ["admission", "verify", "runner_chaos_evals", "source_verified"]); + + const chaos = jobs.runner_chaos_evals; + assert.match(chaos, new RegExp(`^ name: ${runnerChaosEvalsJob.replace(/[()]/g, "\\$&")}$`, "m")); + assert.match(chaos, /^ needs: admission$/m); + assert.match(chaos, /needs\.admission\.outputs\.admitted == 'true'/); + assert.match(chaos, /uses: \.\/\.github\/workflows\/runner-chaos-evals.yml\s+with:\s+ref: \$\{\{ github.sha \}\}/); + assert.doesNotMatch(chaos, /continue-on-error/); + + // The proof needs exactly the source checks; re-adding chaos must fail here. + assert.deepEqual(jobs.source_verified.match(/^ needs: (.+)$/m)?.[1], "[verify]"); + assert.match(jobs.verify, /runner_chaos_evals: false/); + + // Release callers keep the evals as a gate through the default. + const releaseVerify = readFileSync(new URL("../../workflows/release-verify.yml", import.meta.url), "utf8"); + assert.match(releaseVerify, /runner_chaos_evals:\n\s+description: [^\n]+\n\s+required: false\n\s+default: true\n\s+type: boolean/); + assert.match(releaseVerify, /^ runner_chaos_evals:\n(?: #[^\n]*\n)* if: \$\{\{ !inputs.runner_binaries_only && inputs.runner_chaos_evals \}\}$/m); + for (const file of ["release.yml", "release-smoke.yml"]) { + const caller = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8"); + assert.doesNotMatch(caller, /runner_chaos_evals:\s*false/, file); + } +}); + test("standard image provenance and independent exact-source migrators remain available", () => { const docker = readFileSync(new URL("../../workflows/docker.yml", import.meta.url), "utf8"); assert.match(docker, /target: production/); diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml index b9ee017ddea..1e9e0a7dcc5 100644 --- a/.github/workflows/cloud-readiness.yml +++ b/.github/workflows/cloud-readiness.yml @@ -45,6 +45,24 @@ jobs: uses: ./.github/workflows/release-verify.yml with: ref: ${{ github.sha }} + runner_chaos_evals: false + + # Owner decision (2026-11-12): the Runner chaos evals are NOT a publication + # prerequisite. They still run on every admitted master commit and still + # fail their own check (and this run) when they fail, but source_verified + # must not need this job, and scripts/cloud-source-verification.mjs must not + # treat its failure as a failed source verification. They took 10-12 min and + # set the image gate. Do not add this job to source_verified's needs. + # .github/scripts/tests/cloud-readiness.test.mjs fails if it is re-added. + runner_chaos_evals: + name: Runner chaos evals (not a publication gate) + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' + permissions: + contents: read + uses: ./.github/workflows/runner-chaos-evals.yml + with: + ref: ${{ github.sha }} source_verified: # npm canary publication reuses this exact-source verification proof. diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 99c551d0122..e69c8c9b86f 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -12,6 +12,11 @@ on: required: false default: false type: boolean + runner_chaos_evals: + description: Run the Runner chaos evals as part of this verification. Cloud readiness passes false and runs them as a separate, non-gating job. + required: false + default: true + type: boolean outputs: runner_artifact_prefix: value: ${{ jobs.runner_binaries.outputs.artifact_prefix }} @@ -24,8 +29,10 @@ on: # the caller runs on canonical master and verifies that event's exact SHA. # The organization group also restricts these workflow files to master. jobs: + # Release callers keep the default and stay gated on these evals. Cloud + # readiness turns them off here and runs them beside its source proof. runner_chaos_evals: - if: ${{ !inputs.runner_binaries_only }} + if: ${{ !inputs.runner_binaries_only && inputs.runner_chaos_evals }} name: Pre-release Runner chaos evals uses: ./.github/workflows/runner-chaos-evals.yml with: diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 4f9c70c647b..396a9b2667a 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -923,9 +923,12 @@ test("release daemon production uses exact-source native hosts and no publisher assert.match(runner.scripts['build:binary'], /cargo build --release --manifest-path runner\/Cargo.toml --locked/); assert.match(runner.scripts['build:binary'], /&& node scripts\/stage-runner-binary.mjs$/); assert.match(producer, /stage-release-runner-binaries.mjs record/); - for (const job of ["runner_chaos_evals", "typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"]) { + for (const job of ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"]) { assert.ok(workflow.includes(`${job}:\n if: \${{ !inputs.runner_binaries_only }}`)); } + // Binary-only calls still skip the chaos evals; Cloud readiness also turns + // them off here and runs them as its own non-gating job (2026-11-12). + assert.ok(workflow.includes("runner_chaos_evals:\n if: ${{ !inputs.runner_binaries_only && inputs.runner_chaos_evals }}")); const release = readWorkflow("release.yml"); const source = release.split(" release_runner_source:\n")[1].split(/\n [a-z_]+:\n/)[0]; const binaries = release.split(" release_runner_binaries:\n")[1].split(/\n [a-z_]+:\n/)[0]; diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index 09c4687fed1..f2e71314781 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -6,6 +6,12 @@ import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createRe const repository = "paperclipai/paperclip"; const workflowPath = ".github/workflows/cloud-readiness.yml"; export const sourceVerificationJob = "Cloud source verified v1"; +// Owner decision (2026-11-12): the Runner chaos evals run in the same Cloud +// readiness run but are not a publication prerequisite. Their reusable-workflow +// jobs are named " / ". Their failure still fails their own +// check and the run; it must not fail or cancel the source proof or the image. +export const runnerChaosEvalsJob = "Runner chaos evals (not a publication gate)"; +const nonGating = job => typeof job.name === "string" && job.name.startsWith(`${runnerChaosEvalsJob} / `); export class SourceVerificationFailed extends Error { constructor(message, run) { super(message); @@ -58,7 +64,7 @@ export async function readSourceVerification(sha, api, { failFast = false, onJob if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous."); await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt)); if (failFast) { - const failure = jobs.find((job) => job.status === "completed" && + const failure = jobs.find((job) => !nonGating(job) && job.status === "completed" && ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) && job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt); if (failure) { diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 35c07a1ed0d..6e1cf174714 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -5,7 +5,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; -import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; +import { createActionsReader, readSourceVerification, runnerChaosEvalsJob, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; import { releaseAdmitted, deferredProducer, requestDeferredProducer, readMigratorPublication, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; @@ -378,6 +378,27 @@ describe("image build source monitor", () => { assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]); }); + // Owner decision 2026-11-12: Runner chaos evals run and report, but do not + // gate image publication or the exact-source proof. + const chaosJob = { ...proof, id: 112, name: `${runnerChaosEvalsJob} / Restart, replay, trace, and recovery faults` }; + + test("a failed Runner chaos eval neither cancels image work nor blocks the source proof", async () => { + const f = fixture({ jobs: [{ ...chaosJob, conclusion: "failure" }, proof] }); + assert.equal((await watchReleaseSource(f)).jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a failed Runner chaos eval while the proof is pending keeps waiting instead of failing", async () => { + const f = fixture({ jobs: [{ ...chaosJob, conclusion: "failure" }, { ...proof, status: "in_progress", conclusion: null }] }); + assert.equal(await readSourceVerification(sha, f.api, { failFast: true }), undefined); + assert.deepEqual(f.cancellations, []); + }); + + test("a required check that only resembles the chaos job name still fails fast", async () => { + const f = fixture({ jobs: [{ ...proof, name: "Runner chaos evals / Restart", conclusion: "failure" }] }); + await assert.rejects(watchReleaseSource(f), /Runner chaos evals \/ Restart/); + }); + test("a superseded readiness attempt cancels image work with no deployable future", async () => { const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } }); await assert.rejects(watchReleaseSource(f), /did not pass/);