import { readdir, readFile } from "node:fs/promises"; import path from "node:path"; import { describe, expect, it } from "vitest"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); // PR #13470 uses the code-owner-reviewed default branch for this first-party workflow. const ordinaryPrTrustedWorkflowRevision = "master"; const fullStackTestNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u; const buildRunnerNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u; const buildRemoteProviderPackNeeds = /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u; const everydayOracleImage = "python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285"; describe("public repository paid workflow security", () => { it("keeps the manual hosted Linux image build credential-free and pins the authorized target", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/docker-runner-check.yml"), "utf8"); const manual = workflow.slice(workflow.indexOf(" authorize_manual:")); expect(manual.match(/AWS_CI_TRUSTED_USER_IDS/gu)).toHaveLength(2); expect(manual.match(/test "\$REPOSITORY_ID" = 1170821064/gu)).toHaveLength(2); expect(manual.slice(manual.indexOf(' manual_image:'))).toContain('runs-on: ubuntu-latest'); expect(manual).toContain('repos/$REPOSITORY/git/ref/heads/$TARGET_BRANCH'); expect(manual).toContain("ref: ${{ inputs.release_run_id != '' && github.sha || needs.authorize_manual.outputs.target_sha }}"); expect(manual).toContain('test "$PUBLISH_EVAL_IMAGE" = true'); expect(manual).toContain('test "$VERIFY_SOURCE" != true'); expect(manual).toContain('test "$VERIFY_PUBLIC_INSTALL" != true'); expect(manual).toContain('SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}'); expect([...manual.matchAll(/secrets\.([A-Z_]+)/gu)].map(match => match[1])).toEqual(["GITHUB_TOKEN"]); expect(manual).toContain('pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile'); expect(manual).toContain('PAPERCLIP_RUNNER_LOCK_SHA256=$lock_sha'); expect(manual).toContain('docker logout ghcr.io'); }); it("uses the reviewed master branch for the first-party trusted PR workflow", async () => { const ordinaryPrWorkflow = await readFile( path.join(repositoryRoot, ".github/workflows/pr.yml"), "utf8", ); const trustedWorkflowCalls = [ ...ordinaryPrWorkflow.matchAll( /^\s+uses:\s+(paperclipai\/paperclip\/\.github\/workflows\/pr-trusted\.yml)@([^\s#]+)$/gmu, ), ]; expect(trustedWorkflowCalls).toHaveLength(1); expect(trustedWorkflowCalls[0]?.[1]).toBe( "paperclipai/paperclip/.github/workflows/pr-trusted.yml", ); expect(trustedWorkflowCalls[0]?.[2]).toBe( ordinaryPrTrustedWorkflowRevision, ); }); it("keeps pnpm bootstrap registry telemetry out of trusted workflow setup", async () => { for (const workflowName of [ "runner-full-stack-e2e.yml", "pr-trusted.yml", ]) { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows", workflowName), "utf8", ); const pnpmSetupSteps = workflow .split(/\n(?= {6}- )/u) .filter((step) => step.includes("uses: pnpm/action-setup@")); expect(pnpmSetupSteps, workflowName).toHaveLength( workflowName === "pr-trusted.yml" ? 8 : 7, ); for (const step of pnpmSetupSteps) { expect(step, workflowName).toContain('NPM_CONFIG_AUDIT: "false"'); expect(step, workflowName).toContain('NPM_CONFIG_FUND: "false"'); expect(step, workflowName).toContain( 'NPM_CONFIG_UPDATE_NOTIFIER: "false"', ); } for (const variable of [ "NPM_CONFIG_AUDIT", "NPM_CONFIG_FUND", "NPM_CONFIG_UPDATE_NOTIFIER", ]) { expect( workflow.match(new RegExp(`${variable}:`, "gu")), workflowName, ).toHaveLength(pnpmSetupSteps.length); } } }); it("installs a modern Node runtime before every trusted pnpm bootstrap", async () => { const workflows = [ { name: "runner-full-stack-e2e.yml", expectedCachedSetupNodeSteps: 3, }, { name: "pr-trusted.yml", // PR #13300 restores shared stores directly without setup-node cache writes. expectedCachedSetupNodeSteps: 0, }, ]; for (const { name, expectedCachedSetupNodeSteps } of workflows) { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows", name), "utf8", ); const steps = workflow.split(/\n(?= {6}- )/u); const pnpmSetupStepIndexes = steps.flatMap((step, index) => step.includes("uses: pnpm/action-setup@") ? [index] : [], ); expect(pnpmSetupStepIndexes, name).toHaveLength( name === "pr-trusted.yml" ? 8 : 7, ); for (const pnpmSetupStepIndex of pnpmSetupStepIndexes) { const pnpmSetupStep = steps[pnpmSetupStepIndex]!; const nodeBootstrapStep = steps[pnpmSetupStepIndex - 1]!; expect(nodeBootstrapStep, name).toContain("uses: actions/setup-node@"); expect(nodeBootstrapStep, name).not.toContain("cache: pnpm"); const nodeVersionMatch = nodeBootstrapStep.match( /^\s*node-version:\s*["']?(\d+)(?:\.(\d+))?/mu, ); expect(nodeVersionMatch, name).not.toBeNull(); const nodeMajor = Number(nodeVersionMatch![1]); const nodeMinor = Number(nodeVersionMatch![2] ?? 0); expect( nodeMajor > 22 || (nodeMajor === 22 && nodeMinor >= 13), `${name} must install Node >=22.13 before pnpm/action-setup`, ).toBe(true); const conditionPattern = /^ {6}- if:\s*(.+)$/mu; expect(nodeBootstrapStep.match(conditionPattern)?.[1] ?? null).toBe( pnpmSetupStep.match(conditionPattern)?.[1] ?? null, ); } expect(workflow.match(/^\s+cache: pnpm$/gmu) ?? [], name).toHaveLength( expectedCachedSetupNodeSteps, ); } }); it("gates every provider-secret job with stable actor IDs", async () => { const workflows = await Promise.all( [ "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", "e2e.yml", ].map(async (name) => ({ name, contents: await readFile( path.join(repositoryRoot, ".github/workflows", name), "utf8", ), })), ); for (const { name, contents } of workflows) { const authorize = contents.indexOf(" authorize:"); const reauthorize = contents.indexOf("Reauthorize"); const paidCheckout = contents.indexOf("actions/checkout@", reauthorize); const providerAccess = contents.search( /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./, ); expect( authorize, `${name} must have an authorization job`, ).toBeGreaterThan(0); expect( reauthorize, `${name} must reauthorize partial job reruns`, ).toBeGreaterThan(authorize); expect( paidCheckout, `${name} must authorize before checkout`, ).toBeGreaterThan(reauthorize); expect( providerAccess, `${name} must authorize before provider access`, ).toBeGreaterThan(reauthorize); expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS"); expect(contents).toContain("github.actor_id"); expect(contents).toContain("github.triggering_actor"); expect(contents).toContain("refs/heads/$DEFAULT_BRANCH"); expect(contents).toContain("needs: authorize"); expect(contents).toContain("name: runner-e2e-paid"); expect(contents).not.toMatch( /^\s*(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m, ); const actionReferences = [ ...contents.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gm), ].map((match) => match[1]!); expect(actionReferences.length).toBeGreaterThan(0); for (const reference of actionReferences) { expect(reference).toMatch(/^[^@]+@[0-9a-f]{40}$/); } } const fullStack = workflows[0]!.contents; const paidJob = fullStack.slice( fullStack.indexOf(" test:"), fullStack.indexOf(" report:"), ); const authorizeJob = fullStack.slice( fullStack.indexOf(" authorize:"), fullStack.indexOf(" target_lock:"), ); const targetLockJob = fullStack.slice( fullStack.indexOf(" target_lock:"), fullStack.indexOf(" catalog:"), ); const daytonaImageJob = fullStack.slice( fullStack.indexOf(" daytona_image:"), fullStack.indexOf(" build_runner_artifacts:"), ); expect(daytonaImageJob).toMatch(buildRunnerNeeds); expect(daytonaImageJob).toContain( "runs-on: ${{ needs.authorize.outputs.test_runner }}", ); expect(daytonaImageJob).not.toContain("name: runner-e2e-paid"); expect(daytonaImageJob).not.toMatch( /(?:(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI)_API_KEY|GROK_AUTH_JSON)/, ); expect(authorizeJob).toContain( "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'", ); expect(authorizeJob).toContain("github_runner='ubuntu-latest'"); expect(authorizeJob).toContain( "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}", ); expect(authorizeJob).toContain( "playwright_channel: ${{ steps.runner.outputs.playwright_channel }}", ); expect(authorizeJob).toContain('echo "playwright_channel=chrome"'); expect(authorizeJob).toContain('echo "playwright_channel="'); expect(authorizeJob).toContain( "Resolve requested repository branch to an immutable commit", ); expect(authorizeJob).toContain( "repos/$REPOSITORY/branches/$encoded_branch", ); expect(authorizeJob).toContain('echo "sha=$target_sha"'); expect(authorizeJob).toContain( "target_ref: ${{ steps.target.outputs.ref }}", ); expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"'); expect(authorizeJob).not.toContain("actions/checkout@"); expect(authorizeJob).not.toContain("pnpm install"); expect(targetLockJob).toContain("name: Resolve target pnpm lockfile"); expect(targetLockJob).toContain("needs: authorize"); expect(targetLockJob).toContain( "ref: ${{ needs.authorize.outputs.target_sha }}", ); expect(targetLockJob).toContain("persist-credentials: false"); expect(targetLockJob).toContain( "pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only", ); expect(targetLockJob).toContain( "artifact_id: ${{ steps.upload.outputs.artifact-id }}", ); expect(targetLockJob).toContain("lock_sha256:"); expect(targetLockJob).toContain( "runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}", ); expect(targetLockJob).not.toContain("name: runner-e2e-paid"); expect(targetLockJob).not.toMatch( /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, ); expect(paidJob).toContain( "runs-on: ${{ needs.authorize.outputs.test_runner }}", ); expect(paidJob).toMatch(fullStackTestNeeds); expect(paidJob).toContain("name: runner-e2e-paid"); expect(paidJob).toMatch( /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/, ); expect(paidJob).toContain( "pnpm exec playwright install --with-deps --only-shell chromium", ); expect(paidJob).toContain("name: Qualify preinstalled Chrome"); expect(paidJob).toContain( "if: needs.authorize.outputs.playwright_channel == 'chrome'", ); expect(paidJob).toContain("google-chrome --version"); expect(paidJob).toContain( "if: needs.authorize.outputs.playwright_channel != 'chrome'", ); expect(paidJob).toContain( "PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}", ); expect(paidJob).not.toContain( "pnpm exec playwright install --with-deps chromium", ); const paidInstall = paidJob.indexOf( "pnpm install --frozen-lockfile --ignore-scripts", ); const daytonaPluginPreparation = paidJob.indexOf( "Prepare bundled Daytona plugin without dependency lifecycle scripts", ); const awsFfmpegInstall = paidJob.indexOf( "- name: Install Playwright FFmpeg on AWS runner", ); const hostedChromiumInstall = paidJob.indexOf( "- name: Install Chromium headless shell on GitHub-hosted fallback", ); const everydayOraclePreparation = paidJob.indexOf( "- name: Prepare pinned Python artifact oracle image", ); const paidExecution = paidJob.indexOf("- name: Run bounded campaign and retain feedback review"); expect(paidInstall).toBeGreaterThan(0); expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall); expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation); expect(hostedChromiumInstall).toBeGreaterThan(awsFfmpegInstall); expect(everydayOraclePreparation).toBeGreaterThan(hostedChromiumInstall); expect(paidExecution).toBeGreaterThan(awsFfmpegInstall); expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation); expect(paidExecution).toBeGreaterThan(everydayOraclePreparation); const grokPreparation = paidJob.indexOf("- name: Install checksum-verified Grok executable"); expect(grokPreparation).toBeGreaterThan(paidInstall); expect(paidExecution).toBeGreaterThan(grokPreparation); expect(paidJob).toContain("if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-acpx-grok' || matrix.profileId == 'runner-acpx-grok-subscription')"); expect(paidJob).toContain("run: sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok"); const everydayOracleStep = paidJob.slice( everydayOraclePreparation, paidExecution, ); expect(everydayOracleStep).toContain( "if: (matrix.suiteId == 'everyday-workflows' || matrix.suiteId == 'grok-qualification' || matrix.suiteId == 'grok-subscription-qualification') && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')", ); expect(everydayOracleStep).toContain( `oracle_image='${everydayOracleImage}'`, ); expect(everydayOracleStep).toContain( "timeout 30s docker version --format '{{.Server.Version}}'", ); expect(everydayOracleStep).toContain( 'timeout 120s docker pull "$oracle_image"', ); expect(everydayOracleStep).toContain( "timeout 30s docker image inspect \"$oracle_image\" --format '{{.Id}}'", ); const artifactSource = await readFile( path.join(repositoryRoot, "tests/runner-e2e/everyday-artifact.py"), "utf8", ); const artifactImage = artifactSource.match( /^SANDBOX_IMAGE = '([^']+)'$/mu, )?.[1]; expect(artifactImage).toBe(everydayOracleImage); expect(everydayOracleStep).not.toMatch(/secrets\./u); const awsFfmpegStep = paidJob.slice( awsFfmpegInstall, hostedChromiumInstall, ); expect(awsFfmpegStep).toContain( "if: needs.authorize.outputs.playwright_channel == 'chrome'", ); expect(awsFfmpegStep).toContain("pnpm exec playwright install ffmpeg"); expect(awsFfmpegStep).not.toMatch( /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, ); const preparedBeforeProviderAccess = paidJob.slice( daytonaPluginPreparation, paidExecution, ); expect(preparedBeforeProviderAccess).toContain( "if: matrix.environmentId == 'daytona'", ); expect(preparedBeforeProviderAccess).toContain( 'test -f "$daytona_root/pnpm-lock.yaml"', ); expect(preparedBeforeProviderAccess).toContain( "pnpm install --ignore-workspace --frozen-lockfile --ignore-scripts", ); expect(preparedBeforeProviderAccess).not.toContain("--no-lockfile"); expect(preparedBeforeProviderAccess).toContain( "node scripts/link-plugin-dev-sdk.mjs", ); expect(preparedBeforeProviderAccess).toContain( '"@paperclipai/plugin-daytona"', ); expect(preparedBeforeProviderAccess).toContain('"@paperclipai/plugin-sdk"'); expect(preparedBeforeProviderAccess).toContain( 'realpath "$daytona_root/node_modules/@paperclipai/plugin-sdk"', ); expect(preparedBeforeProviderAccess).toContain( 'pnpm --dir "$daytona_root" build', ); expect(preparedBeforeProviderAccess).not.toContain("secrets."); expect(preparedBeforeProviderAccess).not.toContain("pnpm rebuild"); expect(paidJob.slice(0, paidExecution)).not.toMatch( /secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/, ); expect(authorizeJob).toContain('echo "max_parallel_limit=100"'); expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]'); expect(fullStack).toContain("REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}"); expect(fullStack).toContain('[ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]'); expect(fullStack).toContain('[[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]]'); expect(fullStack).toContain( '[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]', ); expect(fullStack).toContain( "group: runner-full-stack-e2e-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}", ); expect(fullStack).toContain( "cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}", ); expect(daytonaImageJob).toMatch( /- if: needs\.catalog\.outputs\.needs_daytona == 'true'\n\s+uses: actions\/checkout@[0-9a-f]{40}/u, ); for (const stepName of [ "Download resolved target lockfile", "Restore resolved target lockfile", ]) { expect(daytonaImageJob).toMatch( new RegExp( `- name: ${stepName}\\n\\s+if: needs\\.catalog\\.outputs\\.needs_daytona == 'true'`, "u", ), ); } expect(daytonaImageJob).toMatch( /- name: No Daytona image needed\n\s+id: local_only\n\s+if: needs\.catalog\.outputs\.needs_daytona != 'true'/u, ); expect(daytonaImageJob).toContain('echo "source_revision="'); expect(daytonaImageJob).toContain('echo "content_id="'); expect(daytonaImageJob).toContain( "IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64", ); expect(daytonaImageJob).toContain( "TARGET_REF: ${{ needs.authorize.outputs.target_ref }}", ); expect(daytonaImageJob).toContain( "DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}", ); const cacheRead = daytonaImageJob.indexOf( '--cache-from "type=registry,ref=${IMAGE_CACHE}"', ); const trustedTargetCheck = daytonaImageJob.indexOf( 'if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then', ); const cacheWrite = daytonaImageJob.indexOf( '--cache-to "type=registry,ref=${IMAGE_CACHE},mode=max"', ); expect(cacheRead).toBeGreaterThan(0); expect(trustedTargetCheck).toBeGreaterThan(cacheRead); expect(cacheWrite).toBeGreaterThan(trustedTargetCheck); expect(daytonaImageJob.slice(trustedTargetCheck, cacheWrite)).not.toContain( "secrets.", ); const targetCodeJobs = [ fullStack.slice( fullStack.indexOf(" catalog:"), fullStack.indexOf(" daytona_image:"), ), fullStack.slice( fullStack.indexOf(" daytona_image:"), fullStack.indexOf(" build_runner_artifacts:"), ), fullStack.slice( fullStack.indexOf(" build_runner_artifacts:"), fullStack.indexOf(" build_remote_provider_pack:"), ), fullStack.slice( fullStack.indexOf(" build_remote_provider_pack:"), fullStack.indexOf(" test:"), ), paidJob, ]; for (const targetCodeJob of targetCodeJobs) { const checkout = targetCodeJob.indexOf("actions/checkout@"); const downloadLock = targetCodeJob.indexOf( "Download resolved target lockfile", ); const restoreLock = targetCodeJob.indexOf( "Restore resolved target lockfile", ); const setupNode = targetCodeJob.indexOf("actions/setup-node@"); const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile"); expect(checkout).toBeGreaterThan(0); expect(downloadLock).toBeGreaterThan(checkout); expect(restoreLock).toBeGreaterThan(downloadLock); if (setupNode >= 0) { expect(setupNode).toBeGreaterThan(restoreLock); } if (install >= 0) { expect(install).toBeGreaterThan(restoreLock); } expect(targetCodeJob).toContain( "artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}", ); expect(targetCodeJob).toContain( "EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}", ); } expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength( 5, ); expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength( 5, ); expect( fullStack.match( /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g, ), ).toHaveLength(6); expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2); expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8); expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}"); expect(fullStack).toContain( "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}", ); const reportJob = fullStack.slice( fullStack.indexOf(" report:"), fullStack.indexOf(" publish_history:"), ); const historyJob = fullStack.slice(fullStack.indexOf(" publish_history:")); expect(reportJob).toContain("ref: ${{ github.sha }}"); expect(reportJob).not.toContain( "ref: ${{ needs.authorize.outputs.target_sha }}", ); expect(reportJob).not.toContain("Download resolved target lockfile"); expect(historyJob).toContain("ref: ${{ github.sha }}"); expect(historyJob).not.toContain( "ref: ${{ needs.authorize.outputs.target_sha }}", ); expect(historyJob).not.toContain("Download resolved target lockfile"); expect(fullStack).toContain( "if: always() && !cancelled() && needs.catalog.result == 'success'", ); for (const targetProvenanceJob of [paidJob, reportJob]) { expect(targetProvenanceJob).toContain( "PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}", ); expect(targetProvenanceJob).toContain( "PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}", ); } for (const [secret, condition] of Object.entries({ OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'", ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'", OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'", XAI_API_KEY: "matrix.credentialName == 'XAI_API_KEY'", GROK_AUTH_JSON: "matrix.credentialName == 'GROK_AUTH_JSON'", DAYTONA_API_KEY: "matrix.environmentId == 'daytona'", })) { expect(fullStack).toContain( `${secret}: \${{ ${condition} && secrets.${secret} || '' }}`, ); } }); it("keeps provider credentials inside explicitly gated paid workflows", async () => { const workflowDirectory = path.join(repositoryRoot, ".github/workflows"); const allowedProviderWorkflows = new Set([ "e2e.yml", "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), ); for (const name of names) { const contents = await readFile( path.join(workflowDirectory, name), "utf8", ); const providerSecretReferences = [ ...contents.matchAll( /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY)['"]\])/g, ), ]; if (providerSecretReferences.length > 0) { expect( allowedProviderWorkflows.has(name), `${name} must not receive provider credentials`, ).toBe(true); } } }); it("uses bounded daily and weekly Product schedules while preserving Runner schedules", async () => { const workflows = await Promise.all( [ "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", ].map((name) => readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"), ), ); for (const [index, workflow] of workflows.entries()) { const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(match => match[1]!); if (index === 0) { expect(crons).toEqual(["47 8 * * *", "47 9 * * 0"]); expect(workflow).toContain("vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'"); expect(workflow).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"'); expect(workflow).toContain('echo "max_parallel=1"'); expect(workflow).toContain("--max-automatic-retries 0"); const paid = workflow.split("- name: Run bounded campaign and retain feedback review")[1]!.split("- name: Run paid cell")[0]!; expect([...paid.matchAll(/secrets\.([A-Z_]+)/g)].map(match => match[1])).toEqual(["BRAINTRUST_API_KEY"]); } else { expect(crons).toHaveLength(1); expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/); } expect(workflow).toContain("workflow_dispatch:"); } }); it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); const buildJobStart = workflow.indexOf(" build_runner_artifacts:"); const testJobStart = workflow.indexOf(" test:", buildJobStart); const reportJobStart = workflow.indexOf(" report:", testJobStart); const buildJob = workflow.slice(buildJobStart, testJobStart); const testJob = workflow.slice(testJobStart, reportJobStart); expect(buildJobStart).toBeGreaterThan(0); expect(testJobStart).toBeGreaterThan(buildJobStart); expect(buildJob).toMatch(buildRunnerNeeds); expect(buildJob).toMatch(buildRemoteProviderPackNeeds); expect(buildJob).not.toContain("environment:"); expect(buildJob).not.toContain("secrets."); expect( buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g), ).toHaveLength(2); expect(buildJob).toContain( "pnpm --filter @paperclipai/paperclip-runner build:typescript", ); expect(buildJob).toContain( "pnpm --filter @paperclipai/paperclip-runner build:runner-binaries", ); expect(buildJob).toContain( "node packages/paperclip-runner/scripts/build-provider-pack.mjs", ); expect(buildJob).toContain( "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs", ); expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256"); expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256"); expect(buildJob).toContain( "build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}", ); expect(buildJob).toContain( "needs.build_runner_artifacts.outputs.build_artifact_name", ); expect(buildJob).toContain( "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}", ); expect(buildJob).toContain( "runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", ); expect(buildJob).toContain( "runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}", ); expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}"); expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}"); expect(workflow).toContain("needs_runner_typescript="); expect(workflow).toContain("needs_native_binaries="); expect(workflow).toContain("needs_remote_provider_pack="); expect(testJob).toMatch(fullStackTestNeeds); expect(testJob).toContain("Download immutable campaign outputs"); expect( testJob.match( /if: startsWith\(matrix\.profileId, 'runner-'\) \|\| matrix\.suiteId == 'openrouter-model-breadth'/gu, ), ).toHaveLength(2); expect(testJob).toContain("Download immutable remote provider pack"); expect(testJob).toContain( "needs.build_runner_artifacts.outputs.build_artifact_name", ); expect(testJob).toContain( "needs.build_remote_provider_pack.outputs.provider_pack_artifact_name", ); expect(testJob).toContain("sha256sum --check"); expect(testJob.indexOf("sha256sum --check")).toBeLessThan( testJob.indexOf("tar --extract"), ); expect(testJob).toContain( "test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd", ); expect(testJob).toContain(".payload.runnerSourceRevision == $revision"); expect(workflow).toContain("Qualify local provider Node interpreter"); expect(testJob).toContain( "Materialize verified pinned OpenCode executable", ); expect(testJob).toContain( "matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'", ); expect(testJob).toContain( "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs", ); expect(testJob).not.toContain("postinstall.mjs"); expect(testJob).not.toContain("pnpm rebuild"); expect(testJob).not.toContain("build:typescript"); expect(testJob).not.toContain("build:runner-binaries"); expect(testJob).not.toContain("build-provider-pack.mjs"); }); it("uses the reviewed AWS Chrome channel without weakening the local executable override", async () => { const config = await readFile( path.join(repositoryRoot, "tests/runner-e2e/playwright.config.ts"), "utf8", ); expect(config).toContain( "process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL?.trim()", ); expect(config).toContain("{ channel: playwrightChannel }"); expect(config).toContain( "process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim()", ); expect(config).toContain( "PAPERCLIP_PLAYWRIGHT_CHANNEL and PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE are mutually exclusive", ); }); it("keeps reporting jobs on the committed lockfile without registry re-resolution", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); for (const jobName of ["report", "publish_history"]) { const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!; const install = job.indexOf("pnpm install --frozen-lockfile --ignore-scripts"); expect(job).toContain("ref: ${{ github.sha }}"); expect(job).not.toContain("resolved-target-lockfile"); expect(job, jobName).not.toContain("--no-frozen-lockfile"); expect(job, jobName).not.toContain("--lockfile-only"); expect(install, jobName).toBeGreaterThan(job.indexOf("ref: ${{ github.sha }}")); const credentials = job.indexOf("aws-actions/configure-aws-credentials@"); if (credentials >= 0) expect(install).toBeLessThan(credentials); } }); it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); const reportStart = workflow.indexOf(" report:"); const publisherStart = workflow.indexOf(" publish_history:", reportStart); const report = workflow.slice(reportStart, publisherStart); const publisher = workflow.slice(publisherStart); expect(report).toContain("actions: read"); expect(report).toContain( '"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"', ); expect(report).toContain("gh api --paginate --slurp"); expect(report).toContain( '"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"', ); expect(report).toContain("--jq '{run_attempt, run_started_at}'"); expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*"); expect(report).not.toContain( "pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*", ); expect(report.match(/merge-multiple: false/g)).toHaveLength(2); expect(report).toContain("Select latest workflow attempt per cell"); expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts"); expect(report).toContain( "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", ); expect(report).toContain( "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}", ); const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}"; expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'"); expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports")); expect(publisher).toContain( 'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"', ); expect(publisher).toContain( "pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}", ); expect(publisher).toContain( "name: ${{ steps.pages_artifact_name.outputs.name }}", ); expect(publisher).toContain( "artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}", ); }); it("keeps recurring reports private while preserving a content-free liveness receipt", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); const paid = workflow.split("\n test:")[1]!.split("\n report:")[0]!; const manual = paid.split("\n test_recurring:")[0]!; const recurring = paid.split("\n test_recurring:")[1]!; expect(manual).not.toContain("id-token: write"); expect(manual).toContain("if: needs.catalog.outputs.operations_cadence == ''"); expect(recurring).toContain("id-token: write"); expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); expect(recurring).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); expect(recurring).toContain("steps: *paid_test_steps"); expect(manual).toContain("steps: &paid_test_steps"); const report = workflow.split("\n report:")[1]!.split("\n publish_history:")[0]!; const step = (source: string, name: string) => source.split(` - name: ${name}\n`)[1]!.split(/\n - /u)[0]!; for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) { expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''"); } expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''"); expect(step(paid, "Run bounded campaign and retain feedback review")).not.toContain("AWS_"); for (const job of [paid, report]) { const credentials = step(job, "Obtain scoped private history credentials"); expect(credentials).toContain("output-env-credentials: false"); expect(credentials).toContain("disable-retry: true"); expect(credentials).toContain("RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN"); } expect(paid.indexOf("Verify private evidence retention before paid execution")).toBeLessThan(paid.indexOf("Run bounded campaign and retain feedback review")); expect(report).toContain("report > runner-e2e-merged-report/report.log 2>&1"); const marker = step(report, "Retain report availability for the schedule monitor"); expect(marker).toContain("steps.private_report.outcome == 'success'"); expect(marker).toContain("path: private-report-status.json"); expect(report).toContain('echo \'{"version":1,"privateReportRetained":true}\' > private-report-status.json'); expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/); }); it("isolates private archive trust from workflows that execute target-branch code", async () => { const workflows = path.join(repositoryRoot, ".github/workflows"); const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8"); const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8"); const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"]; const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]); expect(subjects).toEqual(environments); const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!; const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!; for (const [index, job] of [recurring, report].entries()) { expect(job).toContain(`name: ${environments[index]}`); expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); } expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || filename === "runner-full-stack-e2e.yml") continue; const other = await readFile(path.join(workflows, filename), "utf8"); for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); expect(protocol).toContain("name: runner-e2e-paid"); expect(protocol).toContain("needs.authorize.outputs.target_sha"); }); it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); const publisher = workflow.slice(workflow.indexOf(" publish_history:")); expect(publisher).toContain("id-token: write"); expect(publisher).toContain("name: runner-e2e-history"); expect(publisher).toContain("aws-actions/configure-aws-credentials@"); expect(publisher).toContain("RUNNER_E2E_HISTORY_AWS_ROLE_ARN"); expect(publisher).not.toContain("cache: pnpm"); expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/); expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/); expect(workflow).toContain("history_source_ready"); expect(workflow).toContain("Verify normalized history source report"); expect(workflow).not.toContain("sanitized_screenshot="); expect(workflow).toContain( "Publish S3 history and Pages bundle with declared screenshots", ); expect(workflow).toContain( "Publish trusted summary and declared screenshots to public bundles", ); expect(publisher).toContain( "pnpm exec playwright install --with-deps --only-shell chromium", ); expect(workflow).toContain( "Package pruned dashboard with declared screenshots for GitHub Pages", ); expect(workflow).toContain("path: runner-e2e-merged-report/pages"); expect(workflow).toContain( "Publish latest dashboard with declared screenshots", ); expect(workflow).not.toContain("dashboard_ready"); expect(workflow).not.toContain("Publish latest screenshot dashboard"); expect( workflow.indexOf("pnpm test:e2e:runner:history:publish"), ).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@")); }); it("does not restore dependency caches or run install scripts in OIDC-authorized jobs", async () => { const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); const jobs = workflow.split(/\n(?= [a-z_]+:)/u); const paidSteps = workflow.split("steps: &paid_test_steps")[1]!.split("\n test_recurring:")[0]!; const oidcJobs = jobs.filter(job => job.includes("id-token: write")); expect(oidcJobs.length).toBeGreaterThanOrEqual(3); for (const job of oidcJobs) { const steps = job + (job.includes("steps: *paid_test_steps") ? paidSteps : ""); expect(steps, job.split("\n")[0]).not.toMatch(/cache:\s*(?:pnpm|npm|yarn)\b/u); expect(steps).not.toMatch(/actions\/cache(?:\/restore)?@/u); for (const install of steps.matchAll(/\bpnpm install[^\n]*/gu)) { expect(install[0]).toContain("--ignore-scripts"); } } for (const name of ["report", "publish_history"]) { const job = jobs.find(value => value.startsWith(` ${name}:`))!; const installs = [...job.matchAll(/\bpnpm install[^\n]*/gu)]; expect(installs).toHaveLength(1); for (const install of installs) expect(install[0]).toContain('--store-dir "${{ runner.temp }}/eval-'); } }); });