import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; import { canaryStartup } from "./canary-startup-fixture.mjs"; const repoRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), "..", "..", ); function readWorkflow(name) { return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8"); } test("Docker Hub mirrors apply to both image builders without widening publication access", () => { const dockerWorkflow = readWorkflow("docker.yml"); const releaseWorkflow = readWorkflow("release.yml"); const docker = dockerWorkflow.split(" build-and-push:\n")[1].split("\n merge-and-push:")[0]; const preview = releaseWorkflow.split(" image_preview:\n")[1].split("\n publish_image_preview:")[0]; for (const job of [docker, preview]) { const steps = job.split(/\n(?= - )/); const builder = steps.find((step) => /uses: docker\/setup-buildx-action@/.test(step)); const build = steps.find((step) => /uses: docker\/build-push-action@/.test(step)); assert.ok(builder && build, "the mirror must configure the builder that runs the image build"); assert.ok(steps.indexOf(builder) < steps.indexOf(build)); const config = builder.match(/buildkitd-config-inline: \|\n((?: {12,}[^\n]*(?:\n|$))+)/)?.[1]; assert.ok(config, "configure the BuildKit container, not only the host Docker daemon"); assert.deepEqual([...config.matchAll(/^\s*\[registry\."([^"]+)"\]/gm)].map((match) => match[1]), ["docker.io"], "GHCR publishing and cache traffic must not use the pull mirror"); const mirrors = config.match(/^\s*mirrors = (\[[^\n]+\])$/m)?.[1]; assert.ok(mirrors); assert.deepEqual(JSON.parse(mirrors), ["mirror.gcr.io"]); assert.doesNotMatch(config, /^\s*(?:http|insecure|ca|keypair)\s*=/m, "keep verified HTTPS and anonymous public-image pulls"); assert.doesNotMatch(builder, /driver-opts:|buildkitd-flags:|secrets\./, "do not change bootstrap images, entitlements, or credentials"); } assert.match(docker, /platform: linux\/amd64/); assert.match(docker, /platform: linux\/arm64/); assert.match(docker, /permissions:\n\s+contents: read\n\s+packages: write/); assert.match(docker, /outputs: type=image,name=ghcr\.io\/\$\{\{ github\.repository \}\},push-by-digest=true,name-canonical=true,push=true/); assert.match(docker, /cache-to: type=registry,ref=ghcr\.io\/\$\{\{ github\.repository \}\}:buildcache-\$\{\{ matrix\.arch \}\},mode=max/); assert.match(preview, /permissions:\n\s+contents: read\n\s+steps:/); assert.match(preview, /platforms: linux\/amd64\n\s+push: false/); assert.match(preview, /outputs: type=docker,dest=\$\{\{ runner\.temp \}\}\/preview-image\.tar/); assert.doesNotMatch(preview, /docker\/login-action|secrets\.|packages: write|id-token: write/); assert.match(releaseWorkflow, /publish_image_preview:\n[\s\S]*?needs: \[plan_preview, image_preview\]/); assert.doesNotMatch(dockerWorkflow.split(" merge-and-push:\n")[1], /buildkitd-config-inline/, "the registry-only manifest merge does not need a Docker Hub mirror"); }); test("chaos verification isolates callers that verify the same source commit", () => { const chaosWorkflow = readWorkflow("runner-chaos-evals.yml"); const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1]; assert.ok(group, "chaos verification must define its concurrency group"); // GitHub supplies the top-level caller's workflow name to reusable calls. const resolveGroup = (caller, ref) => group .replaceAll("${{ github.workflow }}", readWorkflow(caller).match(/^name: (.+)$/m)[1]) .replaceAll("${{ inputs.ref || github.ref }}", ref) .toLowerCase(); const sha = "a".repeat(40); const callers = ["cloud-readiness.yml", "release.yml", "runner-chaos-evals.yml"]; const groups = callers.map((caller) => resolveGroup(caller, sha)); assert.equal(new Set(groups).size, callers.length, "Cloud readiness, Release, and standalone evals must not cancel each other"); assert.ok(groups.every((value) => !value.includes("${{")), "resolve every group input"); assert.notEqual(resolveGroup("cloud-readiness.yml", sha), resolveGroup("cloud-readiness.yml", "b".repeat(40)), "different sources remain independent"); assert.match(chaosWorkflow, /cancel-in-progress: true/); }); test("canary reuses exact-source proof while stable keeps full verification", () => { const releaseWorkflow = readWorkflow("release.yml"); const canary = releaseWorkflow.split(" verify_canary:\n")[1].split("\n publish_canary:")[0]; assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); assert.match(canary, /actions: read/); assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/); assert.match(releaseWorkflow, /publish_canary:\n\s+if: github\.event_name == 'push'\n\s+needs: verify_canary/); // The stable lane is gated on the stable channel since the nightly lane // was added; a `needs:` line (for example a preflight job) may sit between // the gate and the delegation. // The stable preflight resolves source_ref to an immutable SHA exactly // once; verification must consume that pin, not re-resolve the ref. assert.match( releaseWorkflow, /verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch' && inputs\.channel == 'stable'\n(?:\s+needs: [^\n]+\n)?\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ needs\.preflight_stable\.outputs\.sha \}\}/, ); assert.doesNotMatch( releaseWorkflow, /verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/, ); }); test("source proof requires every source check and does not wait on image publication", () => { const readiness = readWorkflow("cloud-readiness.yml"); const proof = readiness.split(" source_verified:\n")[1]; assert.match(proof, /name: Cloud source verified v1/); assert.match(proof, /needs: \[verify\]/); assert.match(proof, /node --test scripts\/cloud-source-verification.test.mjs/); assert.match(proof, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.doesNotMatch(proof, /always\(\)|continue-on-error|needs:.*(?:image|artifacts)/); assert.doesNotMatch(readiness, /^ (?:image|artifacts|ready):/m); }); test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => { const dockerfile = readFileSync( path.join(repoRoot, "docker/Dockerfile.onboard-smoke"), "utf8", ); // `onboard --yes` without an explicit --bind prefers trusted-local // defaults and writes a loopback bind, which Docker port mapping cannot // reach. The smoke container must pin a non-loopback preset. assert.match(dockerfile, /onboard --yes --bind lan/); }); test("promotion selection guards against sources that predate their channel tooling", () => { const releaseWorkflow = readWorkflow("release.yml"); // Promotions run the source commit's release.sh, so selection must reject // sources whose tooling does not know the target channel yet. assert.match( releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly'/, ); assert.match( releaseWorkflow, /git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly\|beta\|stable\)'/, ); }); test("candidate-branch betas are validated and fully verified before publish", () => { const releaseWorkflow = readWorkflow("release.yml"); // Candidate heads are new commits: selection must pin the naming // convention and publication must be gated on full verification. assert.match(releaseWorkflow, /candidate\/beta-\*\)/); assert.match( releaseWorkflow, /verify_beta_candidate:\n\s+needs: select_beta\n\s+if: needs\.select_beta\.outputs\.mode == 'candidate'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml/, ); assert.match( releaseWorkflow, /needs\.verify_beta_candidate\.result == 'success'/, ); }); test("post-publish beta smoke survives the skipped candidate-verification ancestor", () => { const releaseWorkflow = readWorkflow("release.yml"); // publish_beta's needs chain contains verify_beta_candidate, which is // skipped on promote-mode betas. An `if:` without a status-check function // gets an implicit success() that evaluates that chain transitively and // silently skips the smoke. The condition must stay explicit. assert.match( releaseWorkflow, /smoke_beta:\n\s+needs: publish_beta\n\s+if: \$\{\{ !cancelled\(\) && needs\.publish_beta\.result == 'success' && !inputs\.dry_run \}\}/, ); }); test("published canaries are gated by the exact-version onboarding browser smoke", () => { const releaseWorkflow = readWorkflow("release.yml"); assert.match( releaseWorkflow, /publish_canary:[\s\S]*?outputs:\n\s+canary_version: \$\{\{ steps\.canary_tag\.outputs\.version \}\}/, ); assert.match( releaseWorkflow, /smoke_canary_onboarding:\n\s+needs: publish_canary\n\s+if: needs\.publish_canary\.result == 'success'/, ); assert.match( releaseWorkflow, /PAPERCLIPAI_VERSION: \$\{\{ needs\.publish_canary\.outputs\.canary_version \}\}/, ); assert.match(releaseWorkflow, /test:canary-onboarding-smoke/); assert.match( releaseWorkflow, /smoke_canary_onboarding:[\s\S]*?uses: actions\/checkout@[0-9a-f]{40} # v7[\s\S]*?uses: pnpm\/action-setup@[0-9a-f]{40} # v6[\s\S]*?uses: actions\/setup-node@[0-9a-f]{40} # v7/, ); assert.match( releaseWorkflow, /smoke_canary_onboarding:[\s\S]*?Install test dependencies\n\s+run: pnpm install --frozen-lockfile/, ); assert.doesNotMatch( releaseWorkflow.match( /smoke_canary_onboarding:[\s\S]*?(?=\n # ----- Nightly lane)/, )?.[0] ?? "", /cache: pnpm/, ); assert.match( releaseWorkflow, /name: Smoke exact published canary through onboarding\n\s+env:\n\s+PAPERCLIP_CANARY_SMOKE_SERVER_LOG: \$\{\{ runner\.temp \}\}\/canary-onboarding-server\.log/, ); assert.match( releaseWorkflow, /smoke_canary_onboarding:[\s\S]*?uses: actions\/upload-artifact@[0-9a-f]{40} # v7/, ); assert.match(releaseWorkflow, /canary-onboarding-server\.log/); assert.match(releaseWorkflow, /tests\/canary-onboarding\/playwright-report/); }); test("canary smoke consumes its publisher's source-bound lockfile before frozen installation", () => { const workflow = readWorkflow("release.yml"); const publish = workflow.split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0]; const smoke = workflow.split(" smoke_canary_onboarding:\n")[1].split(" # ----- Nightly lane")[0]; for (const job of [publish, smoke]) { assert.match(job, /name: Checkout repository\n\s+uses: actions\/checkout@[^\n]+\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/); assert.match(job, /name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}/); assert.match(job, /version: 9\.15\.4/); } assert.match(publish, /name: Save canary smoke lockfile\n\s+uses: actions\/upload-artifact@[0-9a-f]{40} # v4\n\s+with:\n\s+name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}\n\s+path: pnpm-lock\.yaml\n\s+if-no-files-found: error\n\s+overwrite: true\n\s+retention-days: 14/); assert.ok(publish.indexOf("name: Install dependencies") < publish.indexOf("name: Save canary smoke lockfile")); assert.ok(publish.indexOf("name: Save canary smoke lockfile") < publish.indexOf("name: Restore tracked install-time changes")); assert.ok(publish.indexOf("name: Restore tracked install-time changes") < publish.indexOf("name: Publish canary")); assert.match(smoke, /name: Restore canary smoke lockfile\n\s+uses: actions\/download-artifact@[0-9a-f]{40} # v4\n\s+with:\n\s+name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}/); assert.ok(smoke.indexOf("name: Restore canary smoke lockfile") < smoke.indexOf("name: Install test dependencies")); assert.doesNotMatch(smoke, /run-id:|github-token:|repository:|continue-on-error|--no-frozen-lockfile/); }); for (const drift of ["patch", "manifest"]) { test(`a publisher lockfile lets a fresh smoke install retain validation after ${drift} drift`, (t) => { const root = mkdtempSync(path.join(os.tmpdir(), "canary-smoke-lockfile-test-")); t.after(() => rmSync(root, { recursive: true, force: true })); const source = path.join(root, "package"); mkdirSync(source); writeFileSync(path.join(source, "package.json"), JSON.stringify({ name: "smoke-fixture-dependency", version: "1.0.0" })); writeFileSync(path.join(source, "index.js"), 'module.exports = "original";\n'); const pack = spawnSync("npm", ["pack", "--ignore-scripts", "--pack-destination", root], { cwd: source, encoding: "utf8" }); assert.equal(pack.status, 0, pack.stderr); const manifest = { name: "canary-smoke-fixture", private: true, packageManager: "pnpm@9.15.4", dependencies: { "smoke-fixture-dependency": "file:smoke-fixture-dependency-1.0.0.tgz" }, pnpm: { patchedDependencies: { "smoke-fixture-dependency@1.0.0": "fixture.patch" } }, }; writeFileSync(path.join(root, "package.json"), JSON.stringify(manifest)); writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n"); const patch = (value) => writeFileSync(path.join(root, "fixture.patch"), [ "diff --git a/index.js b/index.js", "index 1111111..2222222 100644", "--- a/index.js", "+++ b/index.js", "@@ -1 +1 @@", '-module.exports = "original";', `+module.exports = "${value}";`, "", ].join("\n")); const workflow = readWorkflow("release.yml"); const publish = workflow.split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0]; const smoke = workflow.split(" smoke_canary_onboarding:\n")[1].split(" # ----- Nightly lane")[0]; const installArgs = (name) => { const job = name === "Install dependencies" ? publish : smoke; const command = job.match(new RegExp(`name: ${name}\\n\\s+run: (pnpm install[^\\n]+)`))?.[1]; assert.ok(command, `missing ${name}`); return [...command.split(" ").slice(1), "--offline", "--ignore-scripts"]; }; const install = (args) => spawnSync("pnpm", args, { cwd: root, encoding: "utf8", env: { ...process.env, CI: "true" }, }); const success = (result) => assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`); patch(drift === "patch" ? "old-patch" : "published-patch"); success(install(installArgs("Install dependencies"))); const staleLock = readFileSync(path.join(root, "pnpm-lock.yaml")); if (drift === "patch") { patch("published-patch"); } else { copyFileSync(path.join(root, "smoke-fixture-dependency-1.0.0.tgz"), path.join(root, "updated-dependency.tgz")); manifest.dependencies["smoke-fixture-dependency"] = "file:updated-dependency.tgz"; writeFileSync(path.join(root, "package.json"), JSON.stringify(manifest)); } const before = install(installArgs("Install test dependencies")); assert.notEqual(before.status, 0); assert.match(before.stdout + before.stderr, drift === "patch" ? /ERR_PNPM_LOCKFILE_CONFIG_MISMATCH/ : /ERR_PNPM_OUTDATED_LOCKFILE/); // Capture the publisher's resolution, then reproduce its tracked-file restore // and a fresh smoke runner with no installed workspace dependencies. success(install(installArgs("Install dependencies"))); const artifact = path.join(root, "publisher-lock.yaml"); if (publish.includes("name: Save canary smoke lockfile")) { copyFileSync(path.join(root, "pnpm-lock.yaml"), artifact); } writeFileSync(path.join(root, "pnpm-lock.yaml"), staleLock); rmSync(path.join(root, "node_modules"), { recursive: true, force: true }); if (smoke.includes("name: Restore canary smoke lockfile")) { copyFileSync(artifact, path.join(root, "pnpm-lock.yaml")); } success(install(installArgs("Install test dependencies"))); assert.equal(readFileSync(path.join(root, "node_modules/smoke-fixture-dependency/index.js"), "utf8"), 'module.exports = "published-patch";\n'); assert.deepEqual(readFileSync(path.join(root, "pnpm-lock.yaml")), readFileSync(artifact)); // A lockfile from the wrong source still fails closed; restoring an artifact // must not disable hash validation or resolve another version in the smoke job. patch("different-source"); const wrongSource = install(installArgs("Install test dependencies")); assert.notEqual(wrongSource.status, 0); assert.match(wrongSource.stdout + wrongSource.stderr, /ERR_PNPM_LOCKFILE_CONFIG_MISMATCH/); }); } test("every lane's tag push degrades to recovery instructions when rejected", () => { const releaseWorkflow = readWorkflow("release.yml"); // GITHUB_TOKEN may not create refs pointing at workflow-modifying commits // from dispatch or scheduled runs; a rejected tag push after a successful // npm publish must surface runbook recovery commands, not a bare error. const occurrences = releaseWorkflow.match(/## Tag push rejected/g) ?? []; assert.equal( occurrences.length, 3, "nightly, beta, and stable each carry the recovery summary", ); }); test("release smoke workflow extends the container readiness budget for CI", () => { const smokeWorkflow = readWorkflow("release-smoke.yml"); const harness = readFileSync( path.join(repoRoot, "scripts/docker-onboard-smoke.sh"), "utf8", ); // CI containers cold-install paperclipai and embedded postgres, so the // workflow must extend the harness's local-default readiness budget. assert.match(smokeWorkflow, /SMOKE_READY_TIMEOUT_SECONDS=\d+/); const ciBudget = Number( smokeWorkflow.match(/SMOKE_READY_TIMEOUT_SECONDS=(\d+)/)[1], ); assert.ok( ciBudget >= 300, `CI readiness budget ${ciBudget}s should be at least 300s`, ); assert.match( harness, /SMOKE_READY_TIMEOUT_SECONDS="\$\{SMOKE_READY_TIMEOUT_SECONDS:-\d+\}"/, ); assert.match( harness, /wait_for_http "\$PAPERCLIP_PUBLIC_URL\/api\/health" "\$SMOKE_READY_TIMEOUT_SECONDS" 1/, ); }); test("release verify workflow covers the same split test surface as stable PR verification", () => { const verifyWorkflow = readWorkflow("release-verify.yml"); assert.match(verifyWorkflow, /workflow_call:/); assert.match( verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/, ); assert.match(verifyWorkflow, /pnpm -r typecheck/); assert.match(verifyWorkflow, /pnpm build/); const runnerScripts = JSON.parse(readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8")).scripts; const runnerChecks = [...verifyWorkflow.matchAll(/^ checks: (.+)$/gm)] .flatMap(([, checks]) => checks.split(" ")); assert.deepEqual(runnerChecks, runnerScripts["check:all"].split(" && ") .map((command) => command.replace(/^pnpm run /, ""))); assert.match(verifyWorkflow, /pnpm --filter @paperclipai\/paperclip-runner "\$check"/); assert.match(verifyWorkflow, /runner_workflow_evals:/); assert.match(verifyWorkflow, /runner_chaos_evals:/); assert.match( verifyWorkflow, /uses: \.\/\.github\/workflows\/runner-chaos-evals\.yml/, ); assert.match( verifyWorkflow, /runner_workflow_evals:[\s\S]*?Install dependencies\n\s+run: pnpm install --no-frozen-lockfile[\s\S]*?Run deterministic Runner workflow scorer tests/, ); assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/); const buildJob = verifyWorkflow.match(/ build:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? ""; assert.match(buildJob, /persist-credentials: false/); assert.doesNotMatch(buildJob, /cache: pnpm/); for (const group of ["general-server-without-chat-or-native-runner", "general-chat", "general-workspaces-a", "general-workspaces-b"]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); } for (const [group, count] of [["general-server-without-chat-or-native-runner", 10], ["general-chat", 3]]) { const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))]; assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]), Array.from({ length: count }, (_, index) => [index, count])); } for (const shardIndex of [0, 1, 2, 3, 4]) { assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`)); } // workspaces-a splits with Vitest native --shard in pr.yml; release // verification must keep the same two-shard coverage. for (const shardIndex of [0, 1]) { assert.match( verifyWorkflow, new RegExp( `group: general-workspaces-a[\\s\\S]*?shard_index: ${shardIndex}\\n\\s+shard_count: 2`, ), ); } assert.match(verifyWorkflow, /pnpm test:run:general -- --group/); assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/); }); test("release verification builds native test binaries in a required Rust-cached lane", () => { const workflow = readWorkflow("release-verify.yml"); const runnerJob = workflow.match(/ verify_paperclip_runner:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? ""; assert.equal((runnerJob.match(/- lane: server-integration/g) ?? []).length, 1); assert.doesNotMatch(runnerJob, /continue-on-error/); assert.match(runnerJob, /timeout-minutes: 20/); assert.match(runnerJob, /shared-key: release-runner-v2/); assert.match(runnerJob, /cache-workspace-crates: false/); assert.match(runnerJob, /cache-bin: false/); assert.match(runnerJob, /save-if: \$\{\{ matrix\.lane == 'rust'/); assert.match(runnerJob, /Build native server test binaries\n\s+if: \$\{\{ matrix\.lane == 'server-integration' \}\}\n\s+timeout-minutes: 10/); assert.match(runnerJob, /pnpm build:rust\n\s+cargo build --release --manifest-path runner\/Cargo\.toml --locked -p paperclip-runner-core --bin paperclip-runnerd --bin fake-codex-app-server/); assert.match(runnerJob, /Run native server integration suites\n\s+if: \$\{\{ matrix\.lane == 'server-integration' \}\}\n\s+run: pnpm test:run:general -- --group general-server-native-runner/); assert.ok(runnerJob.indexOf("Cache Runner Rust dependencies") < runnerJob.indexOf("Build native server test binaries")); assert.ok(runnerJob.indexOf("Build native server test binaries") < runnerJob.indexOf("Run native server integration suites")); // The reusable workflow result includes every matrix child. Its caller must // await that result without an override that can certify a failed native lane. const cloud = readWorkflow("cloud-readiness.yml"); assert.match(cloud, /verify:[\s\S]*?uses: \.\/\.github\/workflows\/release-verify\.yml/); assert.match(cloud, /source_verified:[\s\S]*?needs: \[verify\]/); assert.doesNotMatch(cloud, /continue-on-error|always\(\)/); }); test("Runner eval workflows pin actions and gate paid live execution", () => { const actionPinWorkflows = [ readWorkflow("release-verify.yml"), readWorkflow("runner-live-evals.yml"), readWorkflow("runner-chaos-evals.yml"), readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), ]; for (const workflow of actionPinWorkflows) { const remoteUses = workflow .split("\n") .filter( (line) => /^\s*(?:-\s*)?uses: /.test(line) && !line.includes("uses: ./"), ); assert.ok( remoteUses.length > 0, "expected at least one remote action reference", ); for (const line of remoteUses) { assert.match(line, /uses: [^@\s]+@[0-9a-f]{40}(?:\s+# .+)?$/); } } const liveWorkflow = actionPinWorkflows[1]; assert.match(liveWorkflow, /RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'/); assert.match(liveWorkflow, /REF: \$\{\{ github\.ref \}\}/); assert.match(liveWorkflow, /refs\/heads\/\$DEFAULT_BRANCH/); assert.match( liveWorkflow, /DEFAULT_BRANCH: \$\{\{ github\.event\.repository\.default_branch \}\}/, ); assert.match(liveWorkflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/); assert.match(liveWorkflow, /needs: authorize/); assert.match(liveWorkflow, /environment:\n\s+name: runner-e2e-paid/); assert.match( liveWorkflow, /OPENAI_API_KEY: \$\{\{ secrets\.OPENAI_API_KEY \}\}/, ); const paidWorkflowNames = [ "e2e.yml", "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); const providerSecretReference = /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)['"]\])/g; for (const name of readdirSync(path.join(repoRoot, ".github/workflows"))) { if (!/\.ya?ml$/.test(name)) continue; const workflow = readWorkflow(name); if ([...workflow.matchAll(providerSecretReference)].length > 0) { assert.ok( paidWorkflowNameSet.has(name), `${name} must not receive provider credentials`, ); } } for (const name of paidWorkflowNames) { const workflow = readWorkflow(name); const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n")); assert.doesNotMatch( triggerHeader, /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m, ); assert.match(triggerHeader, /^\s{2}workflow_dispatch:/m); assert.match(workflow, /^ authorize:/m); const jobBlocks = workflow .slice(workflow.indexOf("\njobs:\n") + "\njobs:\n".length) .split(/\n(?= [A-Za-z0-9_-]+:\n)/); const providerJobs = jobBlocks.filter( (block) => [...block.matchAll(providerSecretReference)].length > 0, ); assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`); for (const block of providerJobs) { assert.match(block, /\n environment:\n name: runner-e2e-paid\n/); assert.match( block, /\n steps:(?: &[A-Za-z0-9_-]+)?\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/, `${name} must reauthorize as the first provider-job step`, ); const reauthorize = block.indexOf(" - name: Reauthorize"); assert.ok(reauthorize > 0); assert.ok(block.indexOf("actions/checkout@") > reauthorize); assert.ok(block.search(providerSecretReference) > reauthorize); assert.match(block, /github\.actor_id/); assert.match(block, /github\.triggering_actor/); assert.match(block, /RUNNER_E2E_ALLOWED_ACTOR_IDS/); assert.match(block, /refs\/heads\/\$DEFAULT_BRANCH/); assert.doesNotMatch(block, /^\s+cache: pnpm$/m); } } const fullStackWorkflow = readWorkflow("runner-full-stack-e2e.yml"); for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'", DAYTONA_API_KEY: "matrix.environmentId == 'daytona'", })) { assert.ok( fullStackWorkflow.includes( `${secret}: \${{ ${condition} && secrets.${secret} || '' }}`, ), `${secret} must be scoped to only the matrix cells that require it`, ); } const historyPublisher = fullStackWorkflow.slice( fullStackWorkflow.indexOf(" publish_history:"), fullStackWorkflow.indexOf(" pages:"), ); assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m); for (const name of [ "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", ]) { const workflow = readWorkflow(name); const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map( (match) => match[1], ); assert.equal(crons.length, 1, `${name} must have one schedule`); assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/); } const chaosWorkflow = actionPinWorkflows[2]; const runnerBlock = chaosWorkflow.match( /- name: Run Runner fault and replay suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Build server test dependencies)/, )?.[1]; const serverBlock = chaosWorkflow.match( /- name: Run server finalization and recovery suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Upload chaos eval bundle)/, )?.[1]; assert.ok(runnerBlock, "expected Runner chaos test command"); assert.ok(serverBlock, "expected server chaos test command"); for (const [base, block] of [ [path.join(repoRoot, "packages/paperclip-runner"), runnerBlock], [path.join(repoRoot, "server"), serverBlock], ]) { const listedTestPaths = block.match(/src\/[A-Za-z0-9_./-]+\.test\.ts/g) ?? []; assert.ok(listedTestPaths.length > 0, "expected chaos workflow test paths"); assert.equal( new Set(listedTestPaths).size, listedTestPaths.length, "chaos workflow test paths must be unique", ); for (const testPath of listedTestPaths) { assert.ok( existsSync(path.join(base, testPath)), `chaos workflow test path does not exist: ${testPath}`, ); } } }); test("direct Grok qualification installs the pinned binary and scopes the selected credential", () => { const workflow = readWorkflow("runner-protocol-live-evals.yml"); assert.ok(workflow.includes("XAI_API_KEY: ${{ matrix.credentialName == 'XAI_API_KEY' && secrets.XAI_API_KEY || '' }}")); assert.ok(workflow.includes("if [ -f packages/paperclip-runner/scripts/provision-grok.mjs ]; then")); assert.ok(workflow.indexOf("sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok") < workflow.indexOf("pnpm --filter @paperclipai/paperclip-runner deploy --prod")); assert.ok(workflow.includes("PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET: ${{ matrix.credentialName == 'PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET' && secrets.GROK_AUTH_JSON || '' }}")); assert.equal((workflow.match(/secrets\.GROK_AUTH_JSON/gu) ?? []).length, 1); }); test("direct protocol concurrency override only lowers the configured ceiling", () => { const workflow = readWorkflow("runner-protocol-live-evals.yml"); const start = workflow.indexOf(' if [ -n "${REQUESTED_MAX_PARALLEL:-}" ]; then'); const end = workflow.indexOf(" node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog", start); assert.ok(start > 0 && end > start); const script = workflow.slice(start, end) + '\nprintf "%s" "$MAX_PARALLEL"\n'; for (const [requested, expected] of [["", "8"], ["2", "2"], ["8", "8"], ["1", null], ["9", null], ["0", null], ["-1", null], ["2.5", null], ["garbage", null], ["9999999999999999999999", null]]) { const result = spawnSync("bash", ["-eu", "-c", script], { env: { ...process.env, MAX_PARALLEL: "8", REQUESTED_MAX_PARALLEL: requested }, encoding: "utf8", }); assert.equal(result.status, expected === null ? 1 : 0, requested); if (expected !== null) assert.equal(result.stdout, expected); } }); test("canary startup refreshes an incomplete dependency publication, then onboards once", async () => { const result = await canaryStartup({ mode: "recover" }); assert.equal(result.code, 0, result.output); const installs = result.calls.filter(call => call.kind === "npm"); const onboarding = result.calls.filter(call => call.kind === "onboard"); assert.equal(installs.length, 2); assert.equal(onboarding.length, 1); for (const call of installs) { assert.equal(call.args[0], "install"); assert.ok(call.args.includes("paperclipai@2026.1009.0-canary.1")); assert.ok(call.args.includes("--no-package-lock")); assert.ok(call.args.includes("--no-save")); } assert.ok(!installs[0].args.includes("--prefer-online")); assert.ok(installs[1].args.includes("--prefer-online")); assert.notEqual(installs[0].args[2], installs[1].args[2]); assert.equal(installs[0].cache, installs[1].cache); assert.deepEqual(onboarding[0].args.slice(0, 3), ["onboard", "--yes", "--data-dir"]); }); test("permanent missing versions stop after three attempts without onboarding", async () => { const result = await canaryStartup({ mode: "permanent" }); assert.equal(result.code, 1); assert.equal(result.calls.filter(call => call.kind === "npm").length, 3); assert.equal(result.calls.filter(call => call.kind === "onboard").length, 0); assert.match(result.output, /installation failed.*onboarding was not started/); }); test("npm failures other than ETARGET fail immediately without onboarding", async () => { const result = await canaryStartup({ mode: "auth" }); assert.equal(result.code, 1); assert.equal(result.calls.length, 1); assert.match(result.output, /E401/); }); test("onboarding failures are never retried, even when they mention ETARGET", async () => { const result = await canaryStartup({ mode: "success", onboard: "fail" }); assert.equal(result.code, 1); assert.equal(result.calls.filter(call => call.kind === "npm").length, 1); assert.equal(result.calls.filter(call => call.kind === "onboard").length, 1); assert.match(result.output, /onboarding failed \(exit 17\)/); }); test("a hanging npm install is stopped within the shared acquisition budget", async () => { const result = await canaryStartup({ mode: "hang", budget: 1000 }); assert.equal(result.code, 1); assert.equal(result.calls.filter(call => call.kind === "npm").length, 1); assert.equal(result.calls.filter(call => call.kind === "onboard").length, 0); assert.ok(result.elapsed < 4000, `install exceeded bounded cancellation: ${result.elapsed}ms`); assert.match(result.output, /exceeded its startup budget/); }); test("backoff consumes the shared budget and does not begin another install after expiry", async () => { const result = await canaryStartup({ mode: "permanent", budget: 1000, delay: 5000 }); assert.equal(result.code, 1); assert.equal(result.calls.length, 1); assert.match(result.output, /exceeded its startup budget/); }); test("Playwright cancellation stops an onboarding process without replay", async () => { const result = await canaryStartup({ mode: "success", onboard: "hang", cancelAtKind: "onboard" }); assert.equal(result.code, 1); assert.equal(result.calls.filter(call => call.kind === "npm").length, 1); const onboarding = result.calls.filter(call => call.kind === "onboard"); assert.equal(onboarding.length, 1); assert.throws(() => process.kill(onboarding[0].pid, 0), {code:"ESRCH"}); assert.match(result.output, /stopped by SIGTERM/); assert.ok(result.elapsed < 2500, `onboarding cancellation took ${result.elapsed}ms`); }); test("canary install retries retain the existing overall Playwright startup deadline", async () => { const { INSTALL_BUDGET_MS } = await import("../../tests/canary-onboarding/start-published-canary.mjs"); const config = readFileSync(path.join(repoRoot, "tests/canary-onboarding/playwright.config.ts"), "utf8"); assert.equal(INSTALL_BUDGET_MS, 120_000); assert.match(config, /timeout: 300_000/); assert.match(config, /start-published-canary\.mjs/); assert.match(config, /reuseExistingServer: false/); }); test("nested ETARGET output does not retry a final authentication or network failure", async () => { for (const mode of ["mixed", "network"]) { const result = await canaryStartup({ mode }); assert.equal(result.code, 1); assert.equal(result.calls.length, 1); } });