mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-14 17:40:37 +02:00
DottaandPaperclip
36d9eb0c3c
Build the npm Linux daemon as static musl and validate its ELF load contract. Keep the Docker daemon and provider-pack image authority separate. Preserve asset assembly when unrelated workflow ancestors are skipped, and reuse the release verifier for a bounded Linux-only check. Co-Authored-By: Paperclip <noreply@paperclip.ing>
615 lines
32 KiB
JavaScript
615 lines
32 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import test from "node:test";
|
|
|
|
const repoRoot = path.resolve(
|
|
path.dirname(fileURLToPath(import.meta.url)),
|
|
"..",
|
|
"..",
|
|
);
|
|
|
|
function readWorkflow(name) {
|
|
return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");
|
|
}
|
|
|
|
function job(workflow, name) {
|
|
return workflow.split(`\n ${name}:\n`)[1]?.split(/\n [a-z_]+:\n/)[0] ?? '';
|
|
}
|
|
function condition(block, needs, github = {}, inputs = {}, isCancelled = false) {
|
|
const expression = block.match(/ if: >-\n((?: .+\n)+)/)?.[1]?.trim();
|
|
assert.ok(expression, 'Routing must use an explicit condition, including skipped ancestors');
|
|
assert.match(expression, /!cancelled\(\)/, 'Suppress implicit success() from unrelated skipped ancestors');
|
|
return Boolean(Function('needs', 'github', 'inputs', 'cancelled', `return (${expression})`)(needs, github, inputs, () => isCancelled));
|
|
}
|
|
|
|
test('release producer mode validates immutable data inputs and adds only the missing native target jobs', () => {
|
|
const workflow = readWorkflow('release-verify.yml');
|
|
assert.match(workflow, /runner_assets_only:[\s\S]*?default: false/);
|
|
const dispatch = workflow.split(' workflow_dispatch:\n')[1].split(' workflow_call:\n')[0];
|
|
const call = workflow.split(' workflow_call:\n')[1].split('\n# Caller-provided')[0];
|
|
assert.match(dispatch, /portable_linux_only:[\s\S]*?default: true/);
|
|
assert.match(dispatch, /runner_assets_only:[\s\S]*?default: true/);
|
|
assert.match(call, /portable_linux_only:[\s\S]*?default: false/);
|
|
for (const name of ['runner_chaos_evals', 'typecheck', 'general_tests', 'serialized_tests', 'runner_workflow_evals', 'verify_paperclip_runner', 'build']) {
|
|
// The default source gate keeps all its prior checks. Producer-only mode
|
|
// must not create a second broad suite or repeat paid eval authorization.
|
|
assert.match(job(workflow, name), /if: \$\{\{ !inputs\.runner_assets_only && !inputs\.portable_linux_only \}\}/, name);
|
|
}
|
|
const request = job(workflow, 'runner_release_request');
|
|
assert.doesNotMatch(request, /uses: actions\/checkout|secrets\./);
|
|
assert.equal(condition(request, {}, {}, { runner_assets_only: true }), true);
|
|
assert.equal(condition(request, {}, {}, { runner_assets_only: false }), false);
|
|
assert.equal(condition(request, {}, {}, { runner_assets_only: true }, true), false);
|
|
const validation = request.split(' run: |\n')[1];
|
|
assert.ok(validation);
|
|
const source = 'a'.repeat(40);
|
|
for (const [SOURCE_SHA, RUNNER_ASSETS_ONLY, PORTABLE_LINUX_ONLY, expected] of [
|
|
[source, 'true', 'true', 0], [source, 'true', 'false', 0], [source, 'false', 'true', 1],
|
|
['master', 'true', 'true', 1], ['', 'true', 'true', 1],
|
|
]) {
|
|
const result = spawnSync('bash', ['-c', validation], { env: { ...process.env, SOURCE_SHA, RUNNER_ASSETS_ONLY, PORTABLE_LINUX_ONLY }, encoding: 'utf8' });
|
|
assert.equal(result.status, expected, result.stderr);
|
|
}
|
|
for (const name of ['runner_release_binaries', 'runner_release_pack']) {
|
|
const producer = job(workflow, name);
|
|
assert.match(producer, /needs: runner_release_request/);
|
|
assert.match(producer, /contents: read/);
|
|
assert.doesNotMatch(producer, /secrets\.|contents: write|id-token: write|packages: write|--push|--password/);
|
|
assert.match(producer, /test "\$\(git rev-parse HEAD\)" = "\$SOURCE_SHA"/);
|
|
const ready = { runner_release_request: { result: 'success' }, unrelated_lane: { result: 'skipped' } };
|
|
assert.equal(condition(producer, ready, {}, { runner_assets_only: true }), true, name);
|
|
assert.equal(condition(producer, ready, {}, { runner_assets_only: false }), false, name);
|
|
assert.equal(condition(producer, ready, {}, { runner_assets_only: true }, true), false, name);
|
|
assert.equal(condition(producer, ready, { event_name: 'workflow_dispatch' }, { runner_assets_only: true, portable_linux_only: false }), true, `${name}/manual release caller`);
|
|
assert.equal(condition(producer, ready, {}, { runner_assets_only: true, portable_linux_only: true }), name === 'runner_release_binaries', `${name}/portable only`);
|
|
for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(condition(producer, { ...ready, runner_release_request: { result } }, {}, { runner_assets_only: true }), false, `${name}/${result}`);
|
|
}
|
|
}
|
|
const assemble = job(workflow, 'runner_release_assemble');
|
|
const produced = { runner_release_binaries: { result: 'success' }, runner_release_pack: { result: 'success' }, unrelated_lane: { result: 'skipped' } };
|
|
assert.equal(condition(assemble, produced, {}, { runner_assets_only: true }), true);
|
|
assert.equal(condition(assemble, produced, {}, { runner_assets_only: false }), false);
|
|
assert.equal(condition(assemble, produced, {}, { runner_assets_only: true }, true), false);
|
|
assert.equal(condition(assemble, produced, {}, { runner_assets_only: true, portable_linux_only: true }), false, 'Portable-only check has no assembler or publisher');
|
|
for (const name of ['runner_release_binaries', 'runner_release_pack']) for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(condition(assemble, { ...produced, [name]: { result } }, {}, { runner_assets_only: true }), false, `${name}/${result}`);
|
|
}
|
|
const binaries = job(workflow, 'runner_release_binaries');
|
|
const matrix = binaries.match(/include: \$\{\{ fromJSON\(inputs\.portable_linux_only && '([^']+)' \|\| '([^']+)'\) \}\}/);
|
|
assert.ok(matrix, 'Use the same producer with explicit portable-only target selection');
|
|
assert.deepEqual(JSON.parse(matrix[1]), [{ target: 'linux-x64', runner: 'ubuntu-latest' }]);
|
|
assert.deepEqual(JSON.parse(matrix[2]), [{ target: 'darwin-arm64', runner: 'macos-15' }, { target: 'darwin-x64', runner: 'macos-15-intel' }, { target: 'linux-x64', runner: 'ubuntu-latest' }]);
|
|
assert.match(binaries, /cargo build --release.*--locked/);
|
|
assert.match(binaries, /rustup target add x86_64-unknown-linux-musl/);
|
|
assert.match(binaries, /musl-tools cmake/);
|
|
assert.match(binaries, /CC_x86_64_unknown_linux_musl=musl-gcc/);
|
|
assert.match(binaries, /RUSTFLAGS='-C target-feature=\+crt-static'/);
|
|
assert.match(binaries, /--target x86_64-unknown-linux-musl/);
|
|
assert.match(binaries, /target\/x86_64-unknown-linux-musl\/release\/paperclip-runnerd/);
|
|
assert.match(binaries, /record-binary/);
|
|
const pack = job(workflow, 'runner_release_pack');
|
|
assert.doesNotMatch(pack, /docker pull|IMAGE_DIGEST/);
|
|
assert.match(pack, /--target runner-provider-pack/);
|
|
assert.match(pack, /docker cp "\$owner:\$pack_path"/);
|
|
assert.match(pack, /daemon_path=\/app\/server\/dist\/vendor\/paperclip-runner\/bin\/paperclip-runnerd/);
|
|
assert.match(pack, /--entrypoint "\$daemon_path" "\$image" --build-metadata/);
|
|
assert.match(pack, /docker cp "\$owner:\$daemon_path"/);
|
|
assert.match(pack, /timeout 30s docker start --attach "\$owner"/);
|
|
assert.match(pack, /record-image-binary/);
|
|
assert.match(pack, /"\$RUNNER_TEMP\/runner-release-input\/image-linux-x64"/);
|
|
assert.doesNotMatch(pack, /"\$RUNNER_TEMP\/runner-release\/linux-x64"/, 'Image GNU daemon is diagnostics, never the npm Linux payload');
|
|
assert.match(pack, /trap 'docker rm -f "\$owner"/);
|
|
const helper = readFileSync(path.join(repoRoot, 'scripts/release-runner-artifacts.mjs'), 'utf8');
|
|
assert.match(helper, /stage-release-runner-binaries\.mjs/);
|
|
assert.equal((helper.match(/execFileSync\(process\.execPath/g) ?? []).length, 1, 'Only the existing assembler owns staging');
|
|
});
|
|
|
|
test('channel assembly routing preserves skipped promotions and blocks incomplete data before publishers', () => {
|
|
const workflow = readWorkflow('release.yml');
|
|
const skipped = () => ({ result: 'skipped', outputs: {} });
|
|
const base = () => Object.fromEntries(['verify_canary', 'select_nightly', 'select_beta', 'verify_beta_candidate', 'preflight_stable', 'verify_stable'].map(name => [name, skipped()]));
|
|
const pin = job(workflow, 'pin_runner_release_source');
|
|
assert.equal(condition(pin, base(), { event_name: 'workflow_dispatch' }), false);
|
|
for (const [name, extra, event] of [
|
|
['verify_canary', {}, 'push'], ['select_nightly', { proceed: 'true' }, 'schedule'],
|
|
['select_beta', { mode: 'promote' }, 'workflow_dispatch'],
|
|
]) {
|
|
const needs = base(); needs[name] = { result: 'success', outputs: extra };
|
|
assert.equal(condition(pin, needs, { event_name: event }), true, `${name} with unrelated skipped gates`);
|
|
assert.equal(condition(pin, needs, { event_name: event }, {}, true), false);
|
|
}
|
|
const stable = base(); stable.preflight_stable.result = 'success'; stable.verify_stable.result = 'success';
|
|
assert.equal(condition(pin, stable, { event_name: 'workflow_dispatch' }), true);
|
|
stable.verify_stable.result = 'failure';
|
|
assert.equal(condition(pin, stable, { event_name: 'workflow_dispatch' }), false);
|
|
assert.match(pin, /git show "\$source_sha:scripts\/release\.sh" \| grep -F 'PAPERCLIP_RELEASE_RUNNER_ASSETS'/);
|
|
assert.match(pin, /git cat-file -e "\$source_sha:scripts\/release-runner-artifacts\.mjs"/);
|
|
const assets = job(workflow, 'release_runner_assets');
|
|
const pinned = { ...base(), pin_runner_release_source: { result: 'success', outputs: { required: 'true' } } };
|
|
assert.equal(condition(assets, pinned), true, 'A verified pin must produce assets despite unrelated skipped lanes');
|
|
assert.equal(condition(assets, pinned, {}, {}, true), false);
|
|
for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(condition(assets, { ...pinned, pin_runner_release_source: { result, outputs: { required: 'true' } } }), false, result);
|
|
}
|
|
assert.equal(condition(assets, { ...pinned, pin_runner_release_source: { result: 'success', outputs: { required: 'false' } } }), false);
|
|
|
|
const ready = { verify_canary: { result: 'success' },
|
|
select_nightly: { result: 'success', outputs: { proceed: 'true' } }, smoke_nightly: { result: 'success' },
|
|
select_beta: { result: 'success', outputs: { mode: 'promote' } }, verify_beta_candidate: skipped(),
|
|
verify_stable: { result: 'success' },
|
|
pin_runner_release_source: { result: 'success', outputs: { required: 'true' } }, release_runner_assets: { result: 'success' } };
|
|
for (const [name, github, inputs] of [
|
|
['publish_canary', { event_name: 'push' }, {}], ['publish_nightly', { event_name: 'schedule' }, {}],
|
|
['publish_beta', { event_name: 'workflow_dispatch' }, {}],
|
|
['preview_stable', { event_name: 'workflow_dispatch' }, { channel: 'stable', dry_run: true }],
|
|
['publish_stable', { event_name: 'workflow_dispatch' }, { channel: 'stable', dry_run: false }],
|
|
]) {
|
|
const block = job(workflow, name);
|
|
assert.equal(condition(block, ready, github, inputs), true, name);
|
|
for (const result of ['failure', 'cancelled', 'skipped']) {
|
|
assert.equal(condition(block, { ...ready, release_runner_assets: { result } }, github, inputs), false, `${name}/${result}`);
|
|
}
|
|
const older = { ...ready, pin_runner_release_source: { result: 'success', outputs: { required: 'false' } }, release_runner_assets: skipped() };
|
|
assert.equal(condition(block, older, github, inputs), true, `${name} older promoted contract`);
|
|
assert.match(block, /working-directory: source/);
|
|
const validation = block.match(/name: Validate runner data against[\s\S]*?(?=\n - name:)/)?.[0];
|
|
assert.ok(validation);
|
|
assert.match(validation, /node "\$GITHUB_WORKSPACE\/trusted\/scripts\/release-runner-artifacts\.mjs" unpack/);
|
|
assert.match(validation, /"\$RUNNER_TEMP\/runner-release-assets" "\$GITHUB_WORKSPACE\/source"/);
|
|
assert.doesNotMatch(validation, /node scripts\//, 'Credentialed publishers may consume only trusted transfer tooling');
|
|
}
|
|
const release = readFileSync(path.join(repoRoot, 'scripts/release.sh'), 'utf8');
|
|
assert.ok(release.indexOf('pnpm build\n# Production publication') < release.indexOf('PAPERCLIP_RELEASE_RUNNER_ASSETS/bin'));
|
|
assert.ok(release.indexOf('PAPERCLIP_RELEASE_RUNNER_ASSETS/bin') < release.indexOf('node "$REPO_ROOT/scripts/build-standalone-public-packages.mjs"'));
|
|
assert.match(release, /elif \[ "\$dry_run" = false \]; then\n\s+release_fail/);
|
|
});
|
|
|
|
test("chaos verification isolates callers that verify the same source commit", () => {
|
|
const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
|
|
const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1];
|
|
assert.ok(group, "chaos verification must define its concurrency group");
|
|
|
|
// GitHub supplies the top-level caller's workflow name to reusable calls.
|
|
const resolveGroup = (caller, ref) => group
|
|
.replaceAll("${{ github.workflow }}", readWorkflow(caller).match(/^name: (.+)$/m)[1])
|
|
.replaceAll("${{ inputs.ref || github.ref }}", ref)
|
|
.toLowerCase();
|
|
const sha = "a".repeat(40);
|
|
const callers = ["cloud-readiness.yml", "release.yml", "runner-chaos-evals.yml"];
|
|
const groups = callers.map((caller) => resolveGroup(caller, sha));
|
|
assert.equal(new Set(groups).size, callers.length,
|
|
"Cloud readiness, Release, and standalone evals must not cancel each other");
|
|
assert.ok(groups.every((value) => !value.includes("${{")), "resolve every group input");
|
|
assert.notEqual(resolveGroup("cloud-readiness.yml", sha),
|
|
resolveGroup("cloud-readiness.yml", "b".repeat(40)), "different sources remain independent");
|
|
assert.match(chaosWorkflow, /cancel-in-progress: true/);
|
|
});
|
|
|
|
test("canary reuses exact-source proof while stable keeps full verification", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
const canary = releaseWorkflow.split(" verify_canary:\n")[1].split("\n publish_canary:")[0];
|
|
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
|
|
assert.match(canary, /actions: read/);
|
|
assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
|
|
assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
|
assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);
|
|
assert.doesNotMatch(canary, /release-verify\.yml|continue-on-error|always\(\)/);
|
|
const publishCanary = job(releaseWorkflow, 'publish_canary');
|
|
assert.match(publishCanary, /needs: \[verify_canary, pin_runner_release_source, release_runner_assets\]/);
|
|
assert.match(publishCanary, /needs\.verify_canary\.result == 'success'/);
|
|
assert.match(publishCanary, /needs\.release_runner_assets\.result == 'success'/);
|
|
// The stable lane is gated on the stable channel since the nightly lane
|
|
// was added; a `needs:` line (for example a preflight job) may sit between
|
|
// the gate and the delegation.
|
|
// The stable preflight resolves source_ref to an immutable SHA exactly
|
|
// once; verification must consume that pin, not re-resolve the ref.
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/verify_stable:\n\s+if: github\.event_name == 'workflow_dispatch' && inputs\.channel == 'stable'\n(?:\s+needs: [^\n]+\n)?\s+uses: \.\/\.github\/workflows\/release-verify\.yml\n\s+with:\n\s+ref: \$\{\{ needs\.preflight_stable\.outputs\.sha \}\}/,
|
|
);
|
|
assert.doesNotMatch(
|
|
releaseWorkflow,
|
|
/verify_(?:canary|stable):[\s\S]*?pnpm test:run(?:\n|$)/,
|
|
);
|
|
});
|
|
|
|
test("source proof requires every source check and does not wait on image publication", () => {
|
|
const readiness = readWorkflow("cloud-readiness.yml");
|
|
const proof = readiness.split(" source_verified:\n")[1];
|
|
assert.match(proof, /name: Cloud source verified v1/);
|
|
assert.match(proof, /needs: \[verify\]/);
|
|
assert.match(proof, /node --test scripts\/cloud-source-verification.test.mjs/);
|
|
assert.match(proof, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
|
assert.doesNotMatch(proof, /always\(\)|continue-on-error|needs:.*(?:image|artifacts)/);
|
|
assert.doesNotMatch(readiness, /^ (?:image|artifacts|ready):/m);
|
|
});
|
|
|
|
test("onboard smoke container binds beyond loopback so the mapped port is reachable", () => {
|
|
const dockerfile = readFileSync(
|
|
path.join(repoRoot, "docker/Dockerfile.onboard-smoke"),
|
|
"utf8",
|
|
);
|
|
|
|
// `onboard --yes` without an explicit --bind prefers trusted-local
|
|
// defaults and writes a loopback bind, which Docker port mapping cannot
|
|
// reach. The smoke container must pin a non-loopback preset.
|
|
assert.match(dockerfile, /onboard --yes --bind lan/);
|
|
});
|
|
|
|
test("promotion selection guards against sources that predate their channel tooling", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
|
|
// Promotions run the source commit's release.sh, so selection must reject
|
|
// sources whose tooling does not know the target channel yet.
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly'/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/git show "\$\{sha\}:scripts\/release\.sh" \| grep -qF 'canary\|nightly\|beta\|stable\)'/,
|
|
);
|
|
});
|
|
|
|
test("candidate-branch betas are validated and fully verified before publish", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
|
|
// Candidate heads are new commits: selection must pin the naming
|
|
// convention and publication must be gated on full verification.
|
|
assert.match(releaseWorkflow, /candidate\/beta-\*\)/);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/verify_beta_candidate:\n\s+needs: select_beta\n\s+if: needs\.select_beta\.outputs\.mode == 'candidate'\n\s+uses: \.\/\.github\/workflows\/release-verify\.yml/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/needs\.verify_beta_candidate\.result == 'success'/,
|
|
);
|
|
});
|
|
|
|
test("post-publish beta smoke survives the skipped candidate-verification ancestor", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
|
|
// publish_beta's needs chain contains verify_beta_candidate, which is
|
|
// skipped on promote-mode betas. An `if:` without a status-check function
|
|
// gets an implicit success() that evaluates that chain transitively and
|
|
// silently skips the smoke. The condition must stay explicit.
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/smoke_beta:\n\s+needs: publish_beta\n\s+if: \$\{\{ !cancelled\(\) && needs\.publish_beta\.result == 'success' && !inputs\.dry_run \}\}/,
|
|
);
|
|
});
|
|
|
|
test("published canaries are gated by the exact-version onboarding browser smoke", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/publish_canary:[\s\S]*?outputs:\n\s+canary_version: \$\{\{ steps\.canary_tag\.outputs\.version \}\}/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/smoke_canary_onboarding:\n\s+needs: publish_canary\n\s+if: needs\.publish_canary\.result == 'success'/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/PAPERCLIPAI_VERSION: \$\{\{ needs\.publish_canary\.outputs\.canary_version \}\}/,
|
|
);
|
|
assert.match(releaseWorkflow, /test:canary-onboarding-smoke/);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/smoke_canary_onboarding:[\s\S]*?uses: actions\/checkout@[0-9a-f]{40} # v7[\s\S]*?uses: pnpm\/action-setup@[0-9a-f]{40} # v6[\s\S]*?uses: actions\/setup-node@[0-9a-f]{40} # v7/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/smoke_canary_onboarding:[\s\S]*?Install test dependencies\n\s+run: pnpm install --frozen-lockfile/,
|
|
);
|
|
assert.doesNotMatch(
|
|
releaseWorkflow.match(
|
|
/smoke_canary_onboarding:[\s\S]*?(?=\n # ----- Nightly lane)/,
|
|
)?.[0] ?? "",
|
|
/cache: pnpm/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/name: Smoke exact published canary through onboarding\n\s+env:\n\s+PAPERCLIP_CANARY_SMOKE_SERVER_LOG: \$\{\{ runner\.temp \}\}\/canary-onboarding-server\.log/,
|
|
);
|
|
assert.match(
|
|
releaseWorkflow,
|
|
/smoke_canary_onboarding:[\s\S]*?uses: actions\/upload-artifact@[0-9a-f]{40} # v7/,
|
|
);
|
|
assert.match(releaseWorkflow, /canary-onboarding-server\.log/);
|
|
assert.match(releaseWorkflow, /tests\/canary-onboarding\/playwright-report/);
|
|
});
|
|
|
|
test("every lane's tag push degrades to recovery instructions when rejected", () => {
|
|
const releaseWorkflow = readWorkflow("release.yml");
|
|
|
|
// GITHUB_TOKEN may not create refs pointing at workflow-modifying commits
|
|
// from dispatch or scheduled runs; a rejected tag push after a successful
|
|
// npm publish must surface runbook recovery commands, not a bare error.
|
|
const occurrences = releaseWorkflow.match(/## Tag push rejected/g) ?? [];
|
|
assert.equal(
|
|
occurrences.length,
|
|
3,
|
|
"nightly, beta, and stable each carry the recovery summary",
|
|
);
|
|
});
|
|
|
|
test("release smoke workflow extends the container readiness budget for CI", () => {
|
|
const smokeWorkflow = readWorkflow("release-smoke.yml");
|
|
const harness = readFileSync(
|
|
path.join(repoRoot, "scripts/docker-onboard-smoke.sh"),
|
|
"utf8",
|
|
);
|
|
|
|
// CI containers cold-install paperclipai and embedded postgres, so the
|
|
// workflow must extend the harness's local-default readiness budget.
|
|
assert.match(smokeWorkflow, /SMOKE_READY_TIMEOUT_SECONDS=\d+/);
|
|
const ciBudget = Number(
|
|
smokeWorkflow.match(/SMOKE_READY_TIMEOUT_SECONDS=(\d+)/)[1],
|
|
);
|
|
assert.ok(
|
|
ciBudget >= 300,
|
|
`CI readiness budget ${ciBudget}s should be at least 300s`,
|
|
);
|
|
|
|
assert.match(
|
|
harness,
|
|
/SMOKE_READY_TIMEOUT_SECONDS="\$\{SMOKE_READY_TIMEOUT_SECONDS:-\d+\}"/,
|
|
);
|
|
assert.match(
|
|
harness,
|
|
/wait_for_http "\$PAPERCLIP_PUBLIC_URL\/api\/health" "\$SMOKE_READY_TIMEOUT_SECONDS" 1/,
|
|
);
|
|
});
|
|
|
|
test("release verify workflow covers the same split test surface as stable PR verification", () => {
|
|
const verifyWorkflow = readWorkflow("release-verify.yml");
|
|
|
|
assert.match(verifyWorkflow, /workflow_call:/);
|
|
assert.match(
|
|
verifyWorkflow,
|
|
/node \.\/scripts\/release-package-map\.mjs check/,
|
|
);
|
|
assert.match(verifyWorkflow, /pnpm -r typecheck/);
|
|
assert.match(verifyWorkflow, /pnpm build/);
|
|
const runnerScripts = JSON.parse(readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8")).scripts;
|
|
const runnerChecks = [...verifyWorkflow.matchAll(/^ checks: (.+)$/gm)]
|
|
.flatMap(([, checks]) => checks.split(" "));
|
|
assert.deepEqual(runnerChecks, runnerScripts["check:all"].split(" && ")
|
|
.map((command) => command.replace(/^pnpm run /, "")));
|
|
assert.match(verifyWorkflow, /pnpm --filter @paperclipai\/paperclip-runner "\$check"/);
|
|
assert.match(verifyWorkflow, /runner_workflow_evals:/);
|
|
assert.match(verifyWorkflow, /runner_chaos_evals:/);
|
|
assert.match(
|
|
verifyWorkflow,
|
|
/uses: \.\/\.github\/workflows\/runner-chaos-evals\.yml/,
|
|
);
|
|
assert.match(
|
|
verifyWorkflow,
|
|
/runner_workflow_evals:[\s\S]*?Install dependencies\n\s+run: pnpm install --no-frozen-lockfile[\s\S]*?Run deterministic Runner workflow scorer tests/,
|
|
);
|
|
assert.match(verifyWorkflow, /pnpm test:runner-workflow-evals/);
|
|
|
|
const buildJob = verifyWorkflow.match(/ build:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? "";
|
|
assert.match(buildJob, /persist-credentials: false/);
|
|
assert.doesNotMatch(buildJob, /cache: pnpm/);
|
|
|
|
for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) {
|
|
assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
|
|
}
|
|
for (const [group, count] of [["general-server-without-chat", 10], ["general-chat", 3]]) {
|
|
const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))];
|
|
assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]),
|
|
Array.from({ length: count }, (_, index) => [index, count]));
|
|
}
|
|
for (const shardIndex of [0, 1, 2, 3, 4]) {
|
|
assert.match(verifyWorkflow, new RegExp(`shard_index: ${shardIndex}[\\s\\S]*?shard_count: 5`));
|
|
}
|
|
|
|
// workspaces-a splits with Vitest native --shard in pr.yml; release
|
|
// verification must keep the same two-shard coverage.
|
|
for (const shardIndex of [0, 1]) {
|
|
assert.match(
|
|
verifyWorkflow,
|
|
new RegExp(
|
|
`group: general-workspaces-a[\\s\\S]*?shard_index: ${shardIndex}\\n\\s+shard_count: 2`,
|
|
),
|
|
);
|
|
}
|
|
|
|
assert.match(verifyWorkflow, /pnpm test:run:general -- --group/);
|
|
assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/);
|
|
});
|
|
|
|
test("Runner eval workflows pin actions and gate paid live execution", () => {
|
|
const actionPinWorkflows = [
|
|
readWorkflow("release-verify.yml"),
|
|
readWorkflow("runner-live-evals.yml"),
|
|
readWorkflow("runner-chaos-evals.yml"),
|
|
readWorkflow("runner-full-stack-e2e.yml"),
|
|
readWorkflow("e2e.yml"),
|
|
readWorkflow("runner-protocol-live-evals.yml"),
|
|
];
|
|
|
|
for (const workflow of actionPinWorkflows) {
|
|
const remoteUses = workflow
|
|
.split("\n")
|
|
.filter(
|
|
(line) =>
|
|
/^\s*(?:-\s*)?uses: /.test(line) && !line.includes("uses: ./"),
|
|
);
|
|
assert.ok(
|
|
remoteUses.length > 0,
|
|
"expected at least one remote action reference",
|
|
);
|
|
for (const line of remoteUses) {
|
|
assert.match(line, /uses: [^@\s]+@[0-9a-f]{40}(?:\s+# .+)?$/);
|
|
}
|
|
}
|
|
|
|
const liveWorkflow = actionPinWorkflows[1];
|
|
assert.match(liveWorkflow, /RUNNER_LIVE_EVALS_NIGHTLY_ENABLED == 'true'/);
|
|
assert.match(liveWorkflow, /REF: \$\{\{ github\.ref \}\}/);
|
|
assert.match(liveWorkflow, /refs\/heads\/\$DEFAULT_BRANCH/);
|
|
assert.match(
|
|
liveWorkflow,
|
|
/DEFAULT_BRANCH: \$\{\{ github\.event\.repository\.default_branch \}\}/,
|
|
);
|
|
assert.match(liveWorkflow, /RUNNER_E2E_ALLOWED_ACTOR_IDS/);
|
|
assert.match(liveWorkflow, /needs: authorize/);
|
|
assert.match(liveWorkflow, /environment:\n\s+name: runner-e2e-paid/);
|
|
assert.match(
|
|
liveWorkflow,
|
|
/OPENAI_API_KEY: \$\{\{ secrets\.OPENAI_API_KEY \}\}/,
|
|
);
|
|
|
|
const paidWorkflowNames = [
|
|
"e2e.yml",
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
"runner-protocol-live-evals.yml",
|
|
];
|
|
const paidWorkflowNameSet = new Set(paidWorkflowNames);
|
|
const providerSecretReference =
|
|
/secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)['"]\])/g;
|
|
for (const name of readdirSync(path.join(repoRoot, ".github/workflows"))) {
|
|
if (!/\.ya?ml$/.test(name)) continue;
|
|
const workflow = readWorkflow(name);
|
|
if ([...workflow.matchAll(providerSecretReference)].length > 0) {
|
|
assert.ok(
|
|
paidWorkflowNameSet.has(name),
|
|
`${name} must not receive provider credentials`,
|
|
);
|
|
}
|
|
}
|
|
|
|
for (const name of paidWorkflowNames) {
|
|
const workflow = readWorkflow(name);
|
|
const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n"));
|
|
assert.doesNotMatch(
|
|
triggerHeader,
|
|
/^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
|
|
);
|
|
assert.match(triggerHeader, /^\s{2}workflow_dispatch:/m);
|
|
assert.match(workflow, /^ authorize:/m);
|
|
|
|
const jobBlocks = workflow
|
|
.slice(workflow.indexOf("\njobs:\n") + "\njobs:\n".length)
|
|
.split(/\n(?= [A-Za-z0-9_-]+:\n)/);
|
|
const providerJobs = jobBlocks.filter(
|
|
(block) => [...block.matchAll(providerSecretReference)].length > 0,
|
|
);
|
|
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
|
|
for (const block of providerJobs) {
|
|
assert.match(block, /\n environment:\n name: runner-e2e-paid\n/);
|
|
assert.match(
|
|
block,
|
|
/\n steps:(?: &[A-Za-z0-9_-]+)?\n(?:\s*\n)* - name: Reauthorize[^\n]*\n/,
|
|
`${name} must reauthorize as the first provider-job step`,
|
|
);
|
|
const reauthorize = block.indexOf(" - name: Reauthorize");
|
|
assert.ok(reauthorize > 0);
|
|
assert.ok(block.indexOf("actions/checkout@") > reauthorize);
|
|
assert.ok(block.search(providerSecretReference) > reauthorize);
|
|
assert.match(block, /github\.actor_id/);
|
|
assert.match(block, /github\.triggering_actor/);
|
|
assert.match(block, /RUNNER_E2E_ALLOWED_ACTOR_IDS/);
|
|
assert.match(block, /refs\/heads\/\$DEFAULT_BRANCH/);
|
|
assert.doesNotMatch(block, /^\s+cache: pnpm$/m);
|
|
}
|
|
}
|
|
|
|
const fullStackWorkflow = readWorkflow("runner-full-stack-e2e.yml");
|
|
for (const [secret, condition] of Object.entries({
|
|
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
|
|
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
|
|
OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'",
|
|
DAYTONA_API_KEY: "matrix.environmentId == 'daytona'",
|
|
})) {
|
|
assert.ok(
|
|
fullStackWorkflow.includes(
|
|
`${secret}: \${{ ${condition} && secrets.${secret} || '' }}`,
|
|
),
|
|
`${secret} must be scoped to only the matrix cells that require it`,
|
|
);
|
|
}
|
|
const historyPublisher = fullStackWorkflow.slice(
|
|
fullStackWorkflow.indexOf(" publish_history:"),
|
|
fullStackWorkflow.indexOf(" pages:"),
|
|
);
|
|
assert.doesNotMatch(historyPublisher, /^\s+cache: pnpm$/m);
|
|
|
|
for (const name of [
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
"runner-protocol-live-evals.yml",
|
|
]) {
|
|
const workflow = readWorkflow(name);
|
|
const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
|
|
(match) => match[1],
|
|
);
|
|
assert.equal(crons.length, 1, `${name} must have one schedule`);
|
|
assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/);
|
|
}
|
|
|
|
const chaosWorkflow = actionPinWorkflows[2];
|
|
const runnerBlock = chaosWorkflow.match(
|
|
/- name: Run Runner fault and replay suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Build server test dependencies)/,
|
|
)?.[1];
|
|
const serverBlock = chaosWorkflow.match(
|
|
/- name: Run server finalization and recovery suites[\s\S]*?run: \|([\s\S]*?)(?=\n\s+- name: Upload chaos eval bundle)/,
|
|
)?.[1];
|
|
assert.ok(runnerBlock, "expected Runner chaos test command");
|
|
assert.ok(serverBlock, "expected server chaos test command");
|
|
for (const [base, block] of [
|
|
[path.join(repoRoot, "packages/paperclip-runner"), runnerBlock],
|
|
[path.join(repoRoot, "server"), serverBlock],
|
|
]) {
|
|
const listedTestPaths =
|
|
block.match(/src\/[A-Za-z0-9_./-]+\.test\.ts/g) ?? [];
|
|
assert.ok(listedTestPaths.length > 0, "expected chaos workflow test paths");
|
|
assert.equal(
|
|
new Set(listedTestPaths).size,
|
|
listedTestPaths.length,
|
|
"chaos workflow test paths must be unique",
|
|
);
|
|
for (const testPath of listedTestPaths) {
|
|
assert.ok(
|
|
existsSync(path.join(base, testPath)),
|
|
`chaos workflow test path does not exist: ${testPath}`,
|
|
);
|
|
}
|
|
}
|
|
});
|
|
|
|
|
|
test("direct Grok qualification installs the pinned binary and scopes the selected credential", () => {
|
|
const workflow = readWorkflow("runner-protocol-live-evals.yml");
|
|
assert.ok(workflow.includes("XAI_API_KEY: ${{ matrix.credentialName == 'XAI_API_KEY' && secrets.XAI_API_KEY || '' }}"));
|
|
assert.ok(workflow.includes("if [ -f packages/paperclip-runner/scripts/provision-grok.mjs ]; then"));
|
|
assert.ok(workflow.indexOf("sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok") < workflow.indexOf("pnpm --filter @paperclipai/paperclip-runner deploy --prod"));
|
|
assert.ok(workflow.includes("PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET: ${{ matrix.credentialName == 'PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET' && secrets.GROK_AUTH_JSON || '' }}"));
|
|
assert.equal((workflow.match(/secrets\.GROK_AUTH_JSON/gu) ?? []).length, 1);
|
|
});
|
|
|
|
test("direct protocol concurrency override only lowers the configured ceiling", () => {
|
|
const workflow = readWorkflow("runner-protocol-live-evals.yml");
|
|
const start = workflow.indexOf(' if [ -n "${REQUESTED_MAX_PARALLEL:-}" ]; then');
|
|
const end = workflow.indexOf(" node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog", start);
|
|
assert.ok(start > 0 && end > start);
|
|
const script = workflow.slice(start, end) + '\nprintf "%s" "$MAX_PARALLEL"\n';
|
|
for (const [requested, expected] of [["", "8"], ["2", "2"], ["8", "8"], ["1", null], ["9", null], ["0", null], ["-1", null], ["2.5", null], ["garbage", null], ["9999999999999999999999", null]]) {
|
|
const result = spawnSync("bash", ["-eu", "-c", script], {
|
|
env: { ...process.env, MAX_PARALLEL: "8", REQUESTED_MAX_PARALLEL: requested }, encoding: "utf8",
|
|
});
|
|
assert.equal(result.status, expected === null ? 1 : 0, requested);
|
|
if (expected !== null) assert.equal(result.stdout, expected);
|
|
}
|
|
});
|