mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-11 14:10:50 +02:00
DottaandPaperclip
a60b6badb8
Preserve file-level provisioning evidence before cleanup without weakening reviewed closure admission. Record the original cloud failure and the passing current-source native fixtures on macOS 26.5.2. Co-Authored-By: Paperclip <noreply@paperclip.ing>
938 lines
43 KiB
TypeScript
938 lines
43 KiB
TypeScript
import { readdir, readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
const repositoryRoot = path.resolve(import.meta.dirname, "../..");
|
|
// PR #13470 uses the code-owner-reviewed default branch for this first-party workflow.
|
|
const ordinaryPrTrustedWorkflowRevision = "master";
|
|
const fullStackTestNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
|
|
const buildRunnerNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u;
|
|
const buildRemoteProviderPackNeeds =
|
|
/needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u;
|
|
const everydayOracleImage =
|
|
"python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285";
|
|
|
|
describe("public repository paid workflow security", () => {
|
|
it("runs native Hermes fixtures for shared sources, patches, and build inputs", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-hermes-native.yml"), "utf8");
|
|
const filter = workflow.slice(workflow.indexOf(" paths:"), workflow.indexOf(" workflow_dispatch:"));
|
|
const patterns = [...filter.matchAll(/^ - (.+)$/gmu)].map(match => match[1]!);
|
|
const triggers = (file: string) => patterns.some(pattern => path.matchesGlob(file, pattern));
|
|
for (const file of [
|
|
"patches/acpx@0.13.1.patch",
|
|
"packages/paperclip-runner/src/contracts/user-attachments.ts",
|
|
"packages/paperclip-runner/src/backends/codex-native-backend.ts",
|
|
"packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts",
|
|
"packages/paperclip-runner/runner/crates/runner-core/src/durable/runner.rs",
|
|
"packages/paperclip-runner/runner/Cargo.lock",
|
|
"packages/paperclip-runner/package.json",
|
|
"packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs",
|
|
"packages/paperclip-eval-kernel/src/index.ts",
|
|
"packages/adapter-utils/src/paperclip-runner-permissions.ts",
|
|
"packages/shared/src/ai-connections.ts",
|
|
"package.json", "pnpm-lock.yaml", "pnpm-workspace.yaml", "tsconfig.json",
|
|
"tests/runner-e2e/provision-hermes-linux.sh",
|
|
]) expect(triggers(file), `Missing native fixture trigger: ${file}`).toBe(true);
|
|
expect(triggers("README.md")).toBe(false);
|
|
});
|
|
it("keeps native Hermes PR fixtures outside paid authority and strips their child environment", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-hermes-native.yml"), "utf8");
|
|
expect(workflow).toContain("pull_request:");
|
|
expect(workflow).not.toContain("pull_request_target:");
|
|
expect(workflow).not.toMatch(/\bsecrets\.|\benvironment:|\bwrite\b/);
|
|
expect(workflow).toContain("contents: read");
|
|
expect(workflow).toContain("persist-credentials: false");
|
|
const fixtures = workflow.slice(workflow.indexOf(" - name: Run native fixtures"), workflow.indexOf(" - name: Record source"));
|
|
expect(fixtures).toContain('env -i PATH="$PATH" HOME="$RUNNER_TEMP/hermes-fixture-home"');
|
|
expect(fixtures).toContain("PAPERCLIP_HERMES_QUALIFY=1");
|
|
expect(fixtures).not.toMatch(/(?:API_KEY|TOKEN|AUTH_JSON|KEEP)/);
|
|
expect(workflow.indexOf("Provision the pinned Python closure")).toBeLessThan(workflow.indexOf("Run native fixtures"));
|
|
expect(workflow.indexOf("Qualify the job's provider Node interpreter")).toBeLessThan(workflow.indexOf("Run native fixtures"));
|
|
expect(workflow).toContain("fs.chmodSync(process.execPath, mode & ~0o022)");
|
|
expect(workflow).toContain("no paid model, browser or Daytona proof");
|
|
const upload = workflow.slice(workflow.indexOf(" - name: Upload fixture evidence"));
|
|
expect(upload).toContain("hermes-native-evidence/");
|
|
expect(upload).not.toMatch(/hermes-fixture-home|provider-assets|runtimeDirectory/);
|
|
});
|
|
it("qualifies both native targets in cloud and binds the exported binary to the tested artifact", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-hermes-native.yml"), "utf8");
|
|
const targets = workflow.slice(workflow.indexOf(" strategy:"), workflow.indexOf(" timeout-minutes:"));
|
|
expect(targets).toContain("fail-fast: false");
|
|
expect(targets.match(/- name:/gu)).toHaveLength(2);
|
|
expect(targets).toMatch(/name: Linux amd64\s+runner: ubuntu-22\.04\s+platform: linux\s+architecture: x64/u);
|
|
expect(targets).toMatch(/name: Mac arm64\s+runner: macos-15\s+platform: darwin\s+architecture: arm64/u);
|
|
expect(targets).toContain("runs-on: ${{ matrix.runner }}");
|
|
expect(workflow).toContain("process.platform !== process.env.EXPECTED_PLATFORM || process.arch !== process.env.EXPECTED_ARCHITECTURE");
|
|
const provision = workflow.slice(workflow.indexOf(" - name: Provision the pinned Python closure"), workflow.indexOf(" - name: Run native fixtures"));
|
|
expect(provision).toContain('test "$(uname -s)" = Darwin');
|
|
expect(provision).toContain('test "$(uname -m)" = arm64');
|
|
expect(provision).toContain("uv==0.12.17");
|
|
expect(provision).toContain("provider-assets/hermes/darwin-arm64");
|
|
expect(provision).toContain("materializePinnedHermesDistribution");
|
|
expect(provision).toContain("runpy.run_path(");
|
|
expect(provision).toContain("scripts/materialize-hermes.py");
|
|
expect(provision).toContain("shutil.copyfile(Path(sys.argv[2]) / 'manifest.json'");
|
|
expect(provision).toContain("candidate-manifest.json");
|
|
expect(provision).not.toContain("HERMES_CLOSURES[");
|
|
const fixtures = workflow.slice(workflow.indexOf(" - name: Run native fixtures"), workflow.indexOf(" - name: Record source"));
|
|
expect(fixtures).toContain('PAPERCLIP_RUNNER_BINARY="$PWD/dist/bin/paperclip-runnerd"');
|
|
expect(fixtures).toContain('TMPDIR="$RUNNER_TEMP"');
|
|
const evidence = workflow.slice(workflow.indexOf(" - name: Record source"));
|
|
expect(evidence).toContain("prHeadSha: process.env.HERMES_PR_HEAD_SHA");
|
|
expect(evidence).toContain("['rev-parse', 'HEAD^{tree}']");
|
|
expect(evidence).toContain("const binaryPath = join(root, 'dist/bin/paperclip-runnerd')");
|
|
expect(evidence).toContain("createHash('sha256').update(binary).digest('hex')");
|
|
expect(evidence).toContain("runnerBinaryTarget(binary) !== platform");
|
|
expect(evidence).toContain("binarySha256, binaryBytes: binary?.byteLength ?? null");
|
|
expect(evidence).toContain("paperclip-runnerd.tar.gz");
|
|
expect(evidence).toContain("SHA256SUMS");
|
|
expect(evidence).toContain("hermes-native-${{ matrix.platform }}-${{ matrix.architecture }}");
|
|
});
|
|
it("provisions selected Hermes assets before credentials and uses the same selection for image identity and packs", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
|
|
const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:"));
|
|
expect(catalog).toContain('if any(.include[]; .qualificationCandidate == "hermes") then "hermes" else "" end');
|
|
expect(catalog.indexOf("Validate selectors and emit matrix")).toBeLessThan(catalog.indexOf("Compute Daytona image content ID"));
|
|
expect(catalog).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"');
|
|
const image = workflow.slice(workflow.indexOf(" daytona_image:"), workflow.indexOf(" build_runner_artifacts:"));
|
|
expect(image).toContain('--build-arg "PAPERCLIP_RUNNER_CANDIDATE_PROVIDERS=${CANDIDATE_PROVIDERS}"');
|
|
const pack = workflow.slice(workflow.indexOf(" build_remote_provider_pack:"), workflow.indexOf(" test:"));
|
|
expect(pack).toContain("needs.catalog.outputs.candidate_providers == 'hermes'");
|
|
expect(pack.indexOf("Provision pinned Hermes assets")).toBeLessThan(pack.indexOf("Assemble native remote provider pack"));
|
|
expect(pack).toContain('"--candidate-providers=$CANDIDATE_PROVIDERS"');
|
|
expect(pack).not.toContain("secrets.");
|
|
const paid = workflow.slice(workflow.indexOf(" test:"), workflow.indexOf(" aggregate:"));
|
|
expect(paid).toContain("matrix.environmentId == 'local' && matrix.qualificationCandidate == 'hermes'");
|
|
const setup = paid.indexOf("Provision pinned Hermes before the paid local test");
|
|
expect(setup).toBeGreaterThan(0);
|
|
expect(setup).toBeLessThan(paid.indexOf("secrets.OPENROUTER_API_KEY"));
|
|
expect(paid.slice(setup, paid.indexOf("Install checksum-verified Grok"))).not.toContain("secrets.");
|
|
const provision = await readFile(path.join(repositoryRoot, "tests/runner-e2e/provision-hermes-linux.sh"), "utf8");
|
|
expect(provision).toContain("uv==0.12.17");
|
|
expect(provision).toContain("scripts/provision-hermes.mjs");
|
|
});
|
|
it("downloads and verifies qualification archives outside controller source", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
|
|
expect(workflow.match(/path: \$\{\{ runner.temp \}\}\/runner-e2e-build/gu)).toHaveLength(2);
|
|
expect(workflow.match(/path: \$\{\{ runner.temp \}\}\/runner-e2e-provider-pack/gu)).toHaveLength(1);
|
|
expect(workflow.match(/cd "\$RUNNER_TEMP\/runner-e2e-build"/gu)).toHaveLength(2);
|
|
expect(workflow).toContain('cd "$RUNNER_TEMP/runner-e2e-provider-pack"');
|
|
expect(workflow.match(/--file "\$RUNNER_TEMP\/runner-e2e-build\/runner-e2e-build-bundle.tar.gz"/gu)).toHaveLength(2);
|
|
expect(workflow).toContain('--file "$RUNNER_TEMP/runner-e2e-provider-pack/runner-e2e-provider-pack.tar.gz"');
|
|
expect(workflow).not.toMatch(/path: runner-e2e-(?:build|provider-pack)\n/u);
|
|
expect(workflow).not.toMatch(/--file runner-e2e-(?:build|provider-pack)\//u);
|
|
});
|
|
it("keeps the manual EC2 image build credential-free and pins the authorized target", async () => {
|
|
const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/docker-runner-check.yml"), "utf8");
|
|
const manual = workflow.slice(workflow.indexOf(" authorize_manual:"));
|
|
expect(manual.match(/AWS_CI_TRUSTED_USER_IDS/gu)).toHaveLength(2);
|
|
expect(manual.match(/test "\$REPOSITORY_ID" = 1170821064/gu)).toHaveLength(2);
|
|
expect(manual).toContain('runs-on: runs-on/fleet=paperclip-public-pr-x64/env=public-ci');
|
|
expect(manual).toContain('repos/$REPOSITORY/git/ref/heads/$TARGET_BRANCH');
|
|
expect(manual).toContain('ref: ${{ needs.authorize_manual.outputs.target_sha }}');
|
|
expect(manual).toContain('SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}');
|
|
expect([...manual.matchAll(/secrets\.([A-Z_]+)/gu)].map(match => match[1])).toEqual(["GITHUB_TOKEN"]);
|
|
expect(manual).toContain('pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile');
|
|
expect(manual).toContain('PAPERCLIP_RUNNER_LOCK_SHA256=$lock_sha');
|
|
expect(manual).toContain('docker logout ghcr.io');
|
|
});
|
|
|
|
it("uses the reviewed master branch for the first-party trusted PR workflow", async () => {
|
|
const ordinaryPrWorkflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/pr.yml"),
|
|
"utf8",
|
|
);
|
|
const trustedWorkflowCalls = [
|
|
...ordinaryPrWorkflow.matchAll(
|
|
/^\s+uses:\s+(paperclipai\/paperclip\/\.github\/workflows\/pr-trusted\.yml)@([^\s#]+)$/gmu,
|
|
),
|
|
];
|
|
|
|
expect(trustedWorkflowCalls).toHaveLength(1);
|
|
expect(trustedWorkflowCalls[0]?.[1]).toBe(
|
|
"paperclipai/paperclip/.github/workflows/pr-trusted.yml",
|
|
);
|
|
expect(trustedWorkflowCalls[0]?.[2]).toBe(
|
|
ordinaryPrTrustedWorkflowRevision,
|
|
);
|
|
});
|
|
|
|
it("keeps pnpm bootstrap registry telemetry out of trusted workflow setup", async () => {
|
|
for (const workflowName of [
|
|
"runner-full-stack-e2e.yml",
|
|
"pr-trusted.yml",
|
|
]) {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows", workflowName),
|
|
"utf8",
|
|
);
|
|
const pnpmSetupSteps = workflow
|
|
.split(/\n(?= {6}- )/u)
|
|
.filter((step) => step.includes("uses: pnpm/action-setup@"));
|
|
|
|
expect(pnpmSetupSteps, workflowName).toHaveLength(
|
|
workflowName === "pr-trusted.yml" ? 8 : 7,
|
|
);
|
|
for (const step of pnpmSetupSteps) {
|
|
expect(step, workflowName).toContain('NPM_CONFIG_AUDIT: "false"');
|
|
expect(step, workflowName).toContain('NPM_CONFIG_FUND: "false"');
|
|
expect(step, workflowName).toContain(
|
|
'NPM_CONFIG_UPDATE_NOTIFIER: "false"',
|
|
);
|
|
}
|
|
for (const variable of [
|
|
"NPM_CONFIG_AUDIT",
|
|
"NPM_CONFIG_FUND",
|
|
"NPM_CONFIG_UPDATE_NOTIFIER",
|
|
]) {
|
|
expect(
|
|
workflow.match(new RegExp(`${variable}:`, "gu")),
|
|
workflowName,
|
|
).toHaveLength(pnpmSetupSteps.length);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("installs a modern Node runtime before every trusted pnpm bootstrap", async () => {
|
|
const workflows = [
|
|
{
|
|
name: "runner-full-stack-e2e.yml",
|
|
expectedCachedSetupNodeSteps: 4,
|
|
},
|
|
{
|
|
name: "pr-trusted.yml",
|
|
// PR #13300 restores shared stores directly without setup-node cache writes.
|
|
expectedCachedSetupNodeSteps: 0,
|
|
},
|
|
];
|
|
|
|
for (const { name, expectedCachedSetupNodeSteps } of workflows) {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows", name),
|
|
"utf8",
|
|
);
|
|
const steps = workflow.split(/\n(?= {6}- )/u);
|
|
const pnpmSetupStepIndexes = steps.flatMap((step, index) =>
|
|
step.includes("uses: pnpm/action-setup@") ? [index] : [],
|
|
);
|
|
|
|
expect(pnpmSetupStepIndexes, name).toHaveLength(
|
|
name === "pr-trusted.yml" ? 8 : 7,
|
|
);
|
|
for (const pnpmSetupStepIndex of pnpmSetupStepIndexes) {
|
|
const pnpmSetupStep = steps[pnpmSetupStepIndex]!;
|
|
const nodeBootstrapStep = steps[pnpmSetupStepIndex - 1]!;
|
|
expect(nodeBootstrapStep, name).toContain("uses: actions/setup-node@");
|
|
expect(nodeBootstrapStep, name).not.toContain("cache: pnpm");
|
|
|
|
const nodeVersionMatch = nodeBootstrapStep.match(
|
|
/^\s*node-version:\s*["']?(\d+)(?:\.(\d+))?/mu,
|
|
);
|
|
expect(nodeVersionMatch, name).not.toBeNull();
|
|
const nodeMajor = Number(nodeVersionMatch![1]);
|
|
const nodeMinor = Number(nodeVersionMatch![2] ?? 0);
|
|
expect(
|
|
nodeMajor > 22 || (nodeMajor === 22 && nodeMinor >= 13),
|
|
`${name} must install Node >=22.13 before pnpm/action-setup`,
|
|
).toBe(true);
|
|
|
|
const conditionPattern = /^ {6}- if:\s*(.+)$/mu;
|
|
expect(nodeBootstrapStep.match(conditionPattern)?.[1] ?? null).toBe(
|
|
pnpmSetupStep.match(conditionPattern)?.[1] ?? null,
|
|
);
|
|
}
|
|
|
|
expect(workflow.match(/^\s+cache: pnpm$/gmu) ?? [], name).toHaveLength(
|
|
expectedCachedSetupNodeSteps,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("gates every provider-secret job with stable actor IDs", async () => {
|
|
const workflows = await Promise.all(
|
|
[
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
"runner-protocol-live-evals.yml",
|
|
"e2e.yml",
|
|
].map(async (name) => ({
|
|
name,
|
|
contents: await readFile(
|
|
path.join(repositoryRoot, ".github/workflows", name),
|
|
"utf8",
|
|
),
|
|
})),
|
|
);
|
|
|
|
for (const { name, contents } of workflows) {
|
|
const authorize = contents.indexOf(" authorize:");
|
|
const reauthorize = contents.indexOf("Reauthorize");
|
|
const paidCheckout = contents.indexOf("actions/checkout@", reauthorize);
|
|
const providerAccess = contents.search(
|
|
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./,
|
|
);
|
|
expect(
|
|
authorize,
|
|
`${name} must have an authorization job`,
|
|
).toBeGreaterThan(0);
|
|
expect(
|
|
reauthorize,
|
|
`${name} must reauthorize partial job reruns`,
|
|
).toBeGreaterThan(authorize);
|
|
expect(
|
|
paidCheckout,
|
|
`${name} must authorize before checkout`,
|
|
).toBeGreaterThan(reauthorize);
|
|
expect(
|
|
providerAccess,
|
|
`${name} must authorize before provider access`,
|
|
).toBeGreaterThan(reauthorize);
|
|
expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS");
|
|
expect(contents).toContain("github.actor_id");
|
|
expect(contents).toContain("github.triggering_actor");
|
|
expect(contents).toContain("refs/heads/$DEFAULT_BRANCH");
|
|
expect(contents).toContain("needs: authorize");
|
|
expect(contents).toContain("name: runner-e2e-paid");
|
|
expect(contents).not.toMatch(
|
|
/^\s*(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
|
|
);
|
|
const actionReferences = [
|
|
...contents.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gm),
|
|
].map((match) => match[1]!);
|
|
expect(actionReferences.length).toBeGreaterThan(0);
|
|
for (const reference of actionReferences) {
|
|
expect(reference).toMatch(/^[^@]+@[0-9a-f]{40}$/);
|
|
}
|
|
}
|
|
|
|
const fullStack = workflows[0]!.contents;
|
|
const paidJob = fullStack.slice(
|
|
fullStack.indexOf(" test:"),
|
|
fullStack.indexOf(" report:"),
|
|
);
|
|
const authorizeJob = fullStack.slice(
|
|
fullStack.indexOf(" authorize:"),
|
|
fullStack.indexOf(" target_lock:"),
|
|
);
|
|
const targetLockJob = fullStack.slice(
|
|
fullStack.indexOf(" target_lock:"),
|
|
fullStack.indexOf(" catalog:"),
|
|
);
|
|
const daytonaImageJob = fullStack.slice(
|
|
fullStack.indexOf(" daytona_image:"),
|
|
fullStack.indexOf(" build_runner_artifacts:"),
|
|
);
|
|
expect(daytonaImageJob).toMatch(buildRunnerNeeds);
|
|
expect(daytonaImageJob).toContain(
|
|
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
|
|
);
|
|
expect(daytonaImageJob).not.toContain("name: runner-e2e-paid");
|
|
expect(daytonaImageJob).not.toMatch(
|
|
/(?:(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI)_API_KEY|GROK_AUTH_JSON)/,
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
|
|
);
|
|
expect(authorizeJob).toContain("github_runner='ubuntu-latest'");
|
|
expect(authorizeJob).toContain(
|
|
"AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"playwright_channel: ${{ steps.runner.outputs.playwright_channel }}",
|
|
);
|
|
expect(authorizeJob).toContain('echo "playwright_channel=chrome"');
|
|
expect(authorizeJob).toContain('echo "playwright_channel="');
|
|
expect(authorizeJob).toContain(
|
|
"Resolve requested repository branch to an immutable commit",
|
|
);
|
|
expect(authorizeJob).toContain(
|
|
"repos/$REPOSITORY/branches/$encoded_branch",
|
|
);
|
|
expect(authorizeJob).toContain('echo "sha=$target_sha"');
|
|
expect(authorizeJob).toContain(
|
|
"target_ref: ${{ steps.target.outputs.ref }}",
|
|
);
|
|
expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"');
|
|
expect(authorizeJob).not.toContain("actions/checkout@");
|
|
expect(authorizeJob).not.toContain("pnpm install");
|
|
expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
|
|
expect(targetLockJob).toContain("needs: authorize");
|
|
expect(targetLockJob).toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(targetLockJob).toContain("persist-credentials: false");
|
|
expect(targetLockJob).toContain(
|
|
"pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only",
|
|
);
|
|
expect(targetLockJob).toContain(
|
|
"artifact_id: ${{ steps.upload.outputs.artifact-id }}",
|
|
);
|
|
expect(targetLockJob).toContain("lock_sha256:");
|
|
expect(targetLockJob).toContain(
|
|
"runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}",
|
|
);
|
|
expect(targetLockJob).not.toContain("name: runner-e2e-paid");
|
|
expect(targetLockJob).not.toMatch(
|
|
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
|
);
|
|
expect(paidJob).toContain(
|
|
"runs-on: ${{ needs.authorize.outputs.test_runner }}",
|
|
);
|
|
expect(paidJob).toMatch(fullStackTestNeeds);
|
|
expect(paidJob).toContain("name: runner-e2e-paid");
|
|
expect(paidJob).toMatch(
|
|
/Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
|
|
);
|
|
expect(paidJob).toContain(
|
|
"pnpm exec playwright install --with-deps --only-shell chromium",
|
|
);
|
|
expect(paidJob).toContain("name: Qualify preinstalled Chrome");
|
|
expect(paidJob).toContain(
|
|
"if: needs.authorize.outputs.playwright_channel == 'chrome'",
|
|
);
|
|
expect(paidJob).toContain("google-chrome --version");
|
|
expect(paidJob).toContain(
|
|
"if: needs.authorize.outputs.playwright_channel != 'chrome'",
|
|
);
|
|
expect(paidJob).toContain(
|
|
"PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}",
|
|
);
|
|
expect(paidJob).not.toContain(
|
|
"pnpm exec playwright install --with-deps chromium",
|
|
);
|
|
const paidInstall = paidJob.indexOf(
|
|
"pnpm install --frozen-lockfile --ignore-scripts",
|
|
);
|
|
const daytonaPluginPreparation = paidJob.indexOf(
|
|
"Prepare bundled Daytona plugin without dependency lifecycle scripts",
|
|
);
|
|
const awsFfmpegInstall = paidJob.indexOf(
|
|
"- name: Install Playwright FFmpeg on AWS runner",
|
|
);
|
|
const hostedChromiumInstall = paidJob.indexOf(
|
|
"- name: Install Chromium headless shell on GitHub-hosted fallback",
|
|
);
|
|
const everydayOraclePreparation = paidJob.indexOf(
|
|
"- name: Prepare pinned Python artifact oracle image",
|
|
);
|
|
const paidExecution = paidJob.indexOf("- name: Run paid cell");
|
|
expect(paidInstall).toBeGreaterThan(0);
|
|
expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);
|
|
expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation);
|
|
expect(hostedChromiumInstall).toBeGreaterThan(awsFfmpegInstall);
|
|
expect(everydayOraclePreparation).toBeGreaterThan(hostedChromiumInstall);
|
|
expect(paidExecution).toBeGreaterThan(awsFfmpegInstall);
|
|
expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation);
|
|
expect(paidExecution).toBeGreaterThan(everydayOraclePreparation);
|
|
const grokPreparation = paidJob.indexOf("- name: Install checksum-verified Grok executable");
|
|
expect(grokPreparation).toBeGreaterThan(paidInstall);
|
|
expect(paidExecution).toBeGreaterThan(grokPreparation);
|
|
expect(paidJob).toContain("if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-acpx-grok' || matrix.profileId == 'runner-acpx-grok-subscription')");
|
|
expect(paidJob).toContain("run: sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok");
|
|
|
|
const everydayOracleStep = paidJob.slice(
|
|
everydayOraclePreparation,
|
|
paidExecution,
|
|
);
|
|
expect(everydayOracleStep).toContain(
|
|
"if: (matrix.suiteId == 'everyday-workflows' || matrix.suiteId == 'grok-qualification' || matrix.suiteId == 'grok-subscription-qualification') && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')",
|
|
);
|
|
expect(everydayOracleStep).toContain(
|
|
`oracle_image='${everydayOracleImage}'`,
|
|
);
|
|
expect(everydayOracleStep).toContain(
|
|
"timeout 30s docker version --format '{{.Server.Version}}'",
|
|
);
|
|
expect(everydayOracleStep).toContain(
|
|
'timeout 120s docker pull "$oracle_image"',
|
|
);
|
|
expect(everydayOracleStep).toContain(
|
|
"timeout 30s docker image inspect \"$oracle_image\" --format '{{.Id}}'",
|
|
);
|
|
const artifactSource = await readFile(
|
|
path.join(repositoryRoot, "tests/runner-e2e/everyday-artifact.py"),
|
|
"utf8",
|
|
);
|
|
const artifactImage = artifactSource.match(
|
|
/^SANDBOX_IMAGE = '([^']+)'$/mu,
|
|
)?.[1];
|
|
expect(artifactImage).toBe(everydayOracleImage);
|
|
expect(everydayOracleStep).not.toMatch(/secrets\./u);
|
|
const awsFfmpegStep = paidJob.slice(
|
|
awsFfmpegInstall,
|
|
hostedChromiumInstall,
|
|
);
|
|
expect(awsFfmpegStep).toContain(
|
|
"if: needs.authorize.outputs.playwright_channel == 'chrome'",
|
|
);
|
|
expect(awsFfmpegStep).toContain("pnpm exec playwright install ffmpeg");
|
|
expect(awsFfmpegStep).not.toMatch(
|
|
/(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
|
);
|
|
const preparedBeforeProviderAccess = paidJob.slice(
|
|
daytonaPluginPreparation,
|
|
paidExecution,
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"if: matrix.environmentId == 'daytona'",
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'test -f "$daytona_root/pnpm-lock.yaml"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"pnpm install --ignore-workspace --frozen-lockfile --ignore-scripts",
|
|
);
|
|
expect(preparedBeforeProviderAccess).not.toContain("--no-lockfile");
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
"node scripts/link-plugin-dev-sdk.mjs",
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'"@paperclipai/plugin-daytona"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain('"@paperclipai/plugin-sdk"');
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'realpath "$daytona_root/node_modules/@paperclipai/plugin-sdk"',
|
|
);
|
|
expect(preparedBeforeProviderAccess).toContain(
|
|
'pnpm --dir "$daytona_root" build',
|
|
);
|
|
expect(preparedBeforeProviderAccess).not.toContain("secrets.");
|
|
expect(preparedBeforeProviderAccess).not.toContain("pnpm rebuild");
|
|
expect(paidJob.slice(0, paidExecution)).not.toMatch(
|
|
/secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
|
|
);
|
|
expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
|
|
expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
|
|
expect(fullStack).toContain("REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}");
|
|
expect(fullStack).toContain('[ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]');
|
|
expect(fullStack).toContain('[[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]]');
|
|
|
|
expect(fullStack).toContain(
|
|
'[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]',
|
|
);
|
|
expect(fullStack).toContain(
|
|
"group: runner-full-stack-e2e-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}",
|
|
);
|
|
expect(fullStack).toContain(
|
|
"cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}",
|
|
);
|
|
expect(daytonaImageJob).toMatch(
|
|
/- if: needs\.catalog\.outputs\.needs_daytona == 'true'\n\s+uses: actions\/checkout@[0-9a-f]{40}/u,
|
|
);
|
|
for (const stepName of [
|
|
"Download resolved target lockfile",
|
|
"Restore resolved target lockfile",
|
|
]) {
|
|
expect(daytonaImageJob).toMatch(
|
|
new RegExp(
|
|
`- name: ${stepName}\\n\\s+if: needs\\.catalog\\.outputs\\.needs_daytona == 'true'`,
|
|
"u",
|
|
),
|
|
);
|
|
}
|
|
expect(daytonaImageJob).toMatch(
|
|
/- name: No Daytona image needed\n\s+id: local_only\n\s+if: needs\.catalog\.outputs\.needs_daytona != 'true'/u,
|
|
);
|
|
expect(daytonaImageJob).toContain('echo "source_revision="');
|
|
expect(daytonaImageJob).toContain('echo "content_id="');
|
|
expect(daytonaImageJob).toContain(
|
|
"IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64",
|
|
);
|
|
expect(daytonaImageJob).toContain(
|
|
"TARGET_REF: ${{ needs.authorize.outputs.target_ref }}",
|
|
);
|
|
expect(daytonaImageJob).toContain(
|
|
"DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}",
|
|
);
|
|
const cacheRead = daytonaImageJob.indexOf(
|
|
'--cache-from "type=registry,ref=${IMAGE_CACHE}"',
|
|
);
|
|
const trustedTargetCheck = daytonaImageJob.indexOf(
|
|
'if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then',
|
|
);
|
|
const cacheWrite = daytonaImageJob.indexOf(
|
|
'--cache-to "type=registry,ref=${IMAGE_CACHE},mode=max"',
|
|
);
|
|
expect(cacheRead).toBeGreaterThan(0);
|
|
expect(trustedTargetCheck).toBeGreaterThan(cacheRead);
|
|
expect(cacheWrite).toBeGreaterThan(trustedTargetCheck);
|
|
expect(daytonaImageJob.slice(trustedTargetCheck, cacheWrite)).not.toContain(
|
|
"secrets.",
|
|
);
|
|
const targetCodeJobs = [
|
|
fullStack.slice(
|
|
fullStack.indexOf(" catalog:"),
|
|
fullStack.indexOf(" daytona_image:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" daytona_image:"),
|
|
fullStack.indexOf(" build_runner_artifacts:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" build_runner_artifacts:"),
|
|
fullStack.indexOf(" build_remote_provider_pack:"),
|
|
),
|
|
fullStack.slice(
|
|
fullStack.indexOf(" build_remote_provider_pack:"),
|
|
fullStack.indexOf(" test:"),
|
|
),
|
|
paidJob,
|
|
];
|
|
for (const targetCodeJob of targetCodeJobs) {
|
|
const checkout = targetCodeJob.indexOf("actions/checkout@");
|
|
const downloadLock = targetCodeJob.indexOf(
|
|
"Download resolved target lockfile",
|
|
);
|
|
const restoreLock = targetCodeJob.indexOf(
|
|
"Restore resolved target lockfile",
|
|
);
|
|
const setupNode = targetCodeJob.indexOf("actions/setup-node@");
|
|
const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile");
|
|
expect(checkout).toBeGreaterThan(0);
|
|
expect(downloadLock).toBeGreaterThan(checkout);
|
|
expect(restoreLock).toBeGreaterThan(downloadLock);
|
|
if (setupNode >= 0) {
|
|
expect(setupNode).toBeGreaterThan(restoreLock);
|
|
}
|
|
if (install >= 0) {
|
|
expect(install).toBeGreaterThan(restoreLock);
|
|
}
|
|
expect(targetCodeJob).toContain(
|
|
"artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}",
|
|
);
|
|
expect(targetCodeJob).toContain(
|
|
"EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}",
|
|
);
|
|
}
|
|
expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
|
|
5,
|
|
);
|
|
expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
|
|
5,
|
|
);
|
|
expect(
|
|
fullStack.match(
|
|
/ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
|
|
),
|
|
).toHaveLength(6);
|
|
expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
|
|
expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8);
|
|
expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
|
|
expect(fullStack).toContain(
|
|
"PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
|
|
);
|
|
const reportJob = fullStack.slice(
|
|
fullStack.indexOf(" report:"),
|
|
fullStack.indexOf(" publish_history:"),
|
|
);
|
|
const historyJob = fullStack.slice(fullStack.indexOf(" publish_history:"));
|
|
expect(reportJob).toContain("ref: ${{ github.sha }}");
|
|
expect(reportJob).not.toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(reportJob).not.toContain("Download resolved target lockfile");
|
|
expect(historyJob).toContain("ref: ${{ github.sha }}");
|
|
expect(historyJob).not.toContain(
|
|
"ref: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(historyJob).not.toContain("Download resolved target lockfile");
|
|
expect(fullStack).toContain(
|
|
"if: always() && !cancelled() && needs.catalog.result == 'success'",
|
|
);
|
|
for (const targetProvenanceJob of [paidJob, reportJob]) {
|
|
expect(targetProvenanceJob).toContain(
|
|
"PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}",
|
|
);
|
|
expect(targetProvenanceJob).toContain(
|
|
"PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}",
|
|
);
|
|
}
|
|
for (const [secret, condition] of Object.entries({
|
|
OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
|
|
ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
|
|
OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'",
|
|
XAI_API_KEY: "matrix.credentialName == 'XAI_API_KEY'",
|
|
GEMINI_API_KEY: "matrix.credentialName == 'GEMINI_API_KEY'",
|
|
GROK_AUTH_JSON: "matrix.credentialName == 'GROK_AUTH_JSON'",
|
|
DAYTONA_API_KEY: "matrix.environmentId == 'daytona'",
|
|
})) {
|
|
expect(fullStack).toContain(
|
|
`${secret}: \${{ ${condition} && secrets.${secret} || '' }}`,
|
|
);
|
|
}
|
|
});
|
|
|
|
it("keeps provider credentials inside explicitly gated paid workflows", async () => {
|
|
const workflowDirectory = path.join(repositoryRoot, ".github/workflows");
|
|
const allowedProviderWorkflows = new Set([
|
|
"e2e.yml",
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
"runner-protocol-live-evals.yml",
|
|
]);
|
|
const names = (await readdir(workflowDirectory)).filter((name) =>
|
|
/\.ya?ml$/.test(name),
|
|
);
|
|
|
|
for (const name of names) {
|
|
const contents = await readFile(
|
|
path.join(workflowDirectory, name),
|
|
"utf8",
|
|
);
|
|
const providerSecretReferences = [
|
|
...contents.matchAll(
|
|
/secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GEMINI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GEMINI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)['"]\])/g,
|
|
),
|
|
];
|
|
if (providerSecretReferences.length > 0) {
|
|
expect(
|
|
allowedProviderWorkflows.has(name),
|
|
`${name} must not receive provider credentials`,
|
|
).toBe(true);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("runs paid scheduled campaigns only on Sundays", async () => {
|
|
const workflows = await Promise.all(
|
|
[
|
|
"runner-full-stack-e2e.yml",
|
|
"runner-live-evals.yml",
|
|
"runner-protocol-live-evals.yml",
|
|
].map((name) =>
|
|
readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
|
|
),
|
|
);
|
|
for (const workflow of workflows) {
|
|
const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
|
|
(match) => match[1]!,
|
|
);
|
|
expect(crons).toHaveLength(1);
|
|
expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
|
|
expect(workflow).toContain("workflow_dispatch:");
|
|
}
|
|
});
|
|
|
|
it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
|
"utf8",
|
|
);
|
|
const buildJobStart = workflow.indexOf(" build_runner_artifacts:");
|
|
const testJobStart = workflow.indexOf(" test:", buildJobStart);
|
|
const reportJobStart = workflow.indexOf(" report:", testJobStart);
|
|
const buildJob = workflow.slice(buildJobStart, testJobStart);
|
|
const testJob = workflow.slice(testJobStart, reportJobStart);
|
|
|
|
expect(buildJobStart).toBeGreaterThan(0);
|
|
expect(testJobStart).toBeGreaterThan(buildJobStart);
|
|
expect(buildJob).toMatch(buildRunnerNeeds);
|
|
expect(buildJob).toMatch(buildRemoteProviderPackNeeds);
|
|
expect(buildJob).not.toContain("environment:");
|
|
expect(buildJob).not.toContain("secrets.");
|
|
expect(
|
|
buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g),
|
|
).toHaveLength(2);
|
|
expect(buildJob).toContain(
|
|
"pnpm --filter @paperclipai/paperclip-runner build:typescript",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"node packages/paperclip-runner/scripts/build-provider-pack.mjs",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
|
|
);
|
|
expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
|
|
expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
|
|
expect(buildJob).toContain(
|
|
"build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
|
|
);
|
|
expect(buildJob).toContain(
|
|
"runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
|
|
);
|
|
expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}");
|
|
expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}");
|
|
expect(workflow).toContain("needs_runner_typescript=");
|
|
expect(workflow).toContain("needs_native_binaries=");
|
|
expect(workflow).toContain("needs_remote_provider_pack=");
|
|
|
|
expect(testJob).toMatch(fullStackTestNeeds);
|
|
expect(testJob).toContain("Download immutable campaign outputs");
|
|
expect(
|
|
testJob.match(
|
|
/if: startsWith\(matrix\.profileId, 'runner-'\) \|\| matrix\.suiteId == 'openrouter-model-breadth'/gu,
|
|
),
|
|
).toHaveLength(2);
|
|
expect(testJob).toContain("Download immutable remote provider pack");
|
|
expect(testJob).toContain(
|
|
"needs.build_runner_artifacts.outputs.build_artifact_name",
|
|
);
|
|
expect(testJob).toContain(
|
|
"needs.build_remote_provider_pack.outputs.provider_pack_artifact_name",
|
|
);
|
|
expect(testJob).toContain("sha256sum --check");
|
|
expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
|
|
testJob.indexOf("tar --extract"),
|
|
);
|
|
expect(testJob).toContain(
|
|
"test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
|
|
);
|
|
expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
|
|
expect(workflow).toContain("Qualify local provider Node interpreter");
|
|
expect(testJob).toContain(
|
|
"Materialize verified pinned OpenCode executable",
|
|
);
|
|
expect(testJob).toContain(
|
|
"matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'",
|
|
);
|
|
expect(testJob).toContain(
|
|
"node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
|
|
);
|
|
expect(testJob).not.toContain("postinstall.mjs");
|
|
expect(testJob).not.toContain("pnpm rebuild");
|
|
expect(testJob).not.toContain("build:typescript");
|
|
expect(testJob).not.toContain("build:runner-binaries");
|
|
expect(testJob).not.toContain("build-provider-pack.mjs");
|
|
});
|
|
|
|
it("uses the reviewed AWS Chrome channel without weakening the local executable override", async () => {
|
|
const config = await readFile(
|
|
path.join(repositoryRoot, "tests/runner-e2e/playwright.config.ts"),
|
|
"utf8",
|
|
);
|
|
|
|
expect(config).toContain(
|
|
"process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL?.trim()",
|
|
);
|
|
expect(config).toContain("{ channel: playwrightChannel }");
|
|
expect(config).toContain(
|
|
"process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim()",
|
|
);
|
|
expect(config).toContain(
|
|
"PAPERCLIP_PLAYWRIGHT_CHANNEL and PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE are mutually exclusive",
|
|
);
|
|
});
|
|
|
|
it("resolves each trusted reporting lockfile before frozen install and AWS credentials", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8",
|
|
);
|
|
for (const jobName of ["report", "publish_history"]) {
|
|
const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!;
|
|
const resolve = job.indexOf("pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile");
|
|
const install = job.indexOf("pnpm install --frozen-lockfile");
|
|
expect(job).toContain("ref: ${{ github.sha }}");
|
|
expect(job).not.toContain("resolved-target-lockfile");
|
|
expect(resolve, jobName).toBeGreaterThan(-1);
|
|
expect(install, jobName).toBeGreaterThan(resolve);
|
|
const credentials = job.indexOf("aws-actions/configure-aws-credentials@");
|
|
if (credentials >= 0) expect(install).toBeLessThan(credentials);
|
|
}
|
|
});
|
|
|
|
it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
|
"utf8",
|
|
);
|
|
const reportStart = workflow.indexOf(" report:");
|
|
const publisherStart = workflow.indexOf(" publish_history:", reportStart);
|
|
const report = workflow.slice(reportStart, publisherStart);
|
|
const publisher = workflow.slice(publisherStart);
|
|
|
|
expect(report).toContain("actions: read");
|
|
expect(report).toContain(
|
|
'"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"',
|
|
);
|
|
expect(report).toContain("gh api --paginate --slurp");
|
|
expect(report).toContain(
|
|
'"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"',
|
|
);
|
|
expect(report).toContain("--jq '{run_attempt, run_started_at}'");
|
|
expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*");
|
|
expect(report).not.toContain(
|
|
"pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*",
|
|
);
|
|
expect(report.match(/merge-multiple: false/g)).toHaveLength(2);
|
|
expect(report).toContain("Select latest workflow attempt per cell");
|
|
expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts");
|
|
expect(report).toContain(
|
|
"PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",
|
|
);
|
|
expect(report).toContain(
|
|
"PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}",
|
|
);
|
|
expect(report).toContain(
|
|
"PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}",
|
|
);
|
|
expect(
|
|
report.indexOf("Select latest workflow attempt per cell"),
|
|
).toBeLessThan(report.indexOf("Collect blob reports"));
|
|
expect(publisher).toContain(
|
|
'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"',
|
|
);
|
|
expect(publisher).toContain(
|
|
"pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}",
|
|
);
|
|
expect(publisher).toContain(
|
|
"name: ${{ steps.pages_artifact_name.outputs.name }}",
|
|
);
|
|
expect(publisher).toContain(
|
|
"artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}",
|
|
);
|
|
});
|
|
|
|
it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
|
|
const workflow = await readFile(
|
|
path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
|
|
"utf8",
|
|
);
|
|
const publisher = workflow.slice(workflow.indexOf(" publish_history:"));
|
|
expect(publisher).toContain("id-token: write");
|
|
expect(publisher).toContain("name: runner-e2e-history");
|
|
expect(publisher).toContain("aws-actions/configure-aws-credentials@");
|
|
expect(publisher).toContain("RUNNER_E2E_HISTORY_AWS_ROLE_ARN");
|
|
expect(publisher).not.toContain("cache: pnpm");
|
|
expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/);
|
|
expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/);
|
|
expect(workflow).toContain("history_source_ready");
|
|
expect(workflow).toContain("Verify normalized history source report");
|
|
expect(workflow).not.toContain("sanitized_screenshot=");
|
|
expect(workflow).toContain(
|
|
"Publish S3 history and Pages bundle with declared screenshots",
|
|
);
|
|
expect(workflow).toContain(
|
|
"Publish trusted summary and declared screenshots to public bundles",
|
|
);
|
|
expect(publisher).toContain(
|
|
"pnpm exec playwright install --with-deps --only-shell chromium",
|
|
);
|
|
expect(workflow).toContain(
|
|
"Package pruned dashboard with declared screenshots for GitHub Pages",
|
|
);
|
|
expect(workflow).toContain("path: runner-e2e-merged-report/pages");
|
|
expect(workflow).toContain(
|
|
"Publish latest dashboard with declared screenshots",
|
|
);
|
|
expect(workflow).not.toContain("dashboard_ready");
|
|
expect(workflow).not.toContain("Publish latest screenshot dashboard");
|
|
expect(
|
|
workflow.indexOf("pnpm test:e2e:runner:history:publish"),
|
|
).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));
|
|
});
|
|
});
|