PaperClipAI/tests/runner-e2e/workflow-security.test.ts

import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";

const repositoryRoot = path.resolve(import.meta.dirname, "../..");
// PR #13470 uses the code-owner-reviewed default branch for this first-party workflow.
const ordinaryPrTrustedWorkflowRevision = "master";
const fullStackTestNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
const buildRunnerNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u;
const buildRemoteProviderPackNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u;
const everydayOracleImage =
  "python@sha256:9d2e5553305c7c7b0097999bb17187c69b921ccd6bc9d40e4bb5ebe652c00285";

describe("public repository paid workflow security", () => {
  it("exposes Copilot only to selected cells in the protected, reauthorized paid jobs", async () => {
    for (const [filename, jobName, nextJobName, stepName] of [
      ["runner-full-stack-e2e.yml", "test", "report", "Run paid cell"],
      ["runner-protocol-live-evals.yml", "eval_shard_0", "eval_shard_1", "Run one immutable direct protocol cell"],
    ] as const) {
      const workflow = await readFile(path.join(repositoryRoot, ".github/workflows", filename), "utf8");
      const credentialSteps = workflow.split(/\n(?= {6}- )/u).filter(step => step.includes("secrets.COPILOT_GITHUB_TOKEN"));
      expect(credentialSteps, filename).toHaveLength(1);
      const credentialStep = credentialSteps[0]!;
      expect(credentialStep, filename).toContain(`name: ${stepName}`);
      expect(credentialStep, filename).toContain("COPILOT_GITHUB_TOKEN: ${{ matrix.credentialName == 'COPILOT_GITHUB_TOKEN' && secrets.COPILOT_GITHUB_TOKEN || '' }}");
      const jobStart = workflow.indexOf(`\n  ${jobName}:`);
      const jobEnd = workflow.indexOf(`\n  ${nextJobName}:`, jobStart + 1);
      expect(jobStart, filename).toBeGreaterThanOrEqual(0);
      expect(jobEnd, filename).toBeGreaterThan(jobStart);
      const paidJob = workflow.slice(jobStart, jobEnd);
      expect(paidJob, filename).toContain("name: runner-e2e-paid");
      expect(paidJob, filename).toContain(credentialStep);
      const reauthorizationIndex = paidJob.indexOf("Reauthorize paid execution before provider access");
      expect(reauthorizationIndex, filename).toBeGreaterThanOrEqual(0);
      expect(reauthorizationIndex, filename).toBeLessThan(paidJob.indexOf("secrets.COPILOT_GITHUB_TOKEN"));
    }
  });

  it("keeps the manual hosted Linux image build credential-free and pins the authorized target", async () => {
    const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/docker-runner-check.yml"), "utf8");
    const manual = workflow.slice(workflow.indexOf("  authorize_manual:"));
    expect(manual.match(/AWS_CI_TRUSTED_USER_IDS/gu)).toHaveLength(2);
    expect(manual.match(/test "\$REPOSITORY_ID" = 1170821064/gu)).toHaveLength(2);
    expect(manual.slice(manual.indexOf('  manual_image:'))).toContain('runs-on: ubuntu-latest');
    expect(manual).toContain('repos/$REPOSITORY/git/ref/heads/$TARGET_BRANCH');
    expect(manual).toContain("ref: ${{ inputs.release_run_id != '' && github.sha || needs.authorize_manual.outputs.target_sha }}");
    expect(manual).toContain('test "$PUBLISH_EVAL_IMAGE" = true');
    expect(manual).toContain('test "$VERIFY_SOURCE" != true');
    expect(manual).toContain('test "$VERIFY_PUBLIC_INSTALL" != true');
    expect(manual).toContain('SOURCE_SHA: ${{ needs.authorize_manual.outputs.target_sha }}');
    expect([...manual.matchAll(/secrets\.([A-Z_]+)/gu)].map(match => match[1])).toEqual(["GITHUB_TOKEN"]);
    expect(manual).toContain('pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile');
    expect(manual).toContain('PAPERCLIP_RUNNER_LOCK_SHA256=$lock_sha');
    expect(manual).toContain('docker logout ghcr.io');
  });

  it("uses the reviewed master branch for the first-party trusted PR workflow", async () => {
    const ordinaryPrWorkflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/pr.yml"),
      "utf8",
    );
    const trustedWorkflowCalls = [
      ...ordinaryPrWorkflow.matchAll(
        /^\s+uses:\s+(paperclipai\/paperclip\/\.github\/workflows\/pr-trusted\.yml)@([^\s#]+)$/gmu,
      ),
    ];

    expect(trustedWorkflowCalls).toHaveLength(1);
    expect(trustedWorkflowCalls[0]?.[1]).toBe(
      "paperclipai/paperclip/.github/workflows/pr-trusted.yml",
    );
    expect(trustedWorkflowCalls[0]?.[2]).toBe(
      ordinaryPrTrustedWorkflowRevision,
    );
  });

  it("keeps pnpm bootstrap registry telemetry out of trusted workflow setup", async () => {
    for (const workflowName of [
      "runner-full-stack-e2e.yml",
      "pr-trusted.yml",
    ]) {
      const workflow = await readFile(
        path.join(repositoryRoot, ".github/workflows", workflowName),
        "utf8",
      );
      const pnpmSetupSteps = workflow
        .split(/\n(?= {6}- )/u)
        .filter((step) => step.includes("uses: pnpm/action-setup@"));

      expect(pnpmSetupSteps, workflowName).toHaveLength(
        workflowName === "pr-trusted.yml" ? 8 : 7,
      );
      for (const step of pnpmSetupSteps) {
        expect(step, workflowName).toContain('NPM_CONFIG_AUDIT: "false"');
        expect(step, workflowName).toContain('NPM_CONFIG_FUND: "false"');
        expect(step, workflowName).toContain(
          'NPM_CONFIG_UPDATE_NOTIFIER: "false"',
        );
      }
      for (const variable of [
        "NPM_CONFIG_AUDIT",
        "NPM_CONFIG_FUND",
        "NPM_CONFIG_UPDATE_NOTIFIER",
      ]) {
        expect(
          workflow.match(new RegExp(`${variable}:`, "gu")),
          workflowName,
        ).toHaveLength(pnpmSetupSteps.length);
      }
    }
  });

  it("installs a modern Node runtime before every trusted pnpm bootstrap", async () => {
    const workflows = [
      {
        name: "runner-full-stack-e2e.yml",
        expectedCachedSetupNodeSteps: 3,
      },
      {
        name: "pr-trusted.yml",
        // PR #13300 restores shared stores directly without setup-node cache writes.
        expectedCachedSetupNodeSteps: 0,
      },
    ];

    for (const { name, expectedCachedSetupNodeSteps } of workflows) {
      const workflow = await readFile(
        path.join(repositoryRoot, ".github/workflows", name),
        "utf8",
      );
      const steps = workflow.split(/\n(?= {6}- )/u);
      const pnpmSetupStepIndexes = steps.flatMap((step, index) =>
        step.includes("uses: pnpm/action-setup@") ? [index] : [],
      );

      expect(pnpmSetupStepIndexes, name).toHaveLength(
        name === "pr-trusted.yml" ? 8 : 7,
      );
      for (const pnpmSetupStepIndex of pnpmSetupStepIndexes) {
        const pnpmSetupStep = steps[pnpmSetupStepIndex]!;
        const nodeBootstrapStep = steps[pnpmSetupStepIndex - 1]!;
        expect(nodeBootstrapStep, name).toContain("uses: actions/setup-node@");
        expect(nodeBootstrapStep, name).not.toContain("cache: pnpm");

        const nodeVersionMatch = nodeBootstrapStep.match(
          /^\s*node-version:\s*["']?(\d+)(?:\.(\d+))?/mu,
        );
        expect(nodeVersionMatch, name).not.toBeNull();
        const nodeMajor = Number(nodeVersionMatch![1]);
        const nodeMinor = Number(nodeVersionMatch![2] ?? 0);
        expect(
          nodeMajor > 22 || (nodeMajor === 22 && nodeMinor >= 13),
          `${name} must install Node >=22.13 before pnpm/action-setup`,
        ).toBe(true);

        const conditionPattern = /^ {6}- if:\s*(.+)$/mu;
        expect(nodeBootstrapStep.match(conditionPattern)?.[1] ?? null).toBe(
          pnpmSetupStep.match(conditionPattern)?.[1] ?? null,
        );
      }

      expect(workflow.match(/^\s+cache: pnpm$/gmu) ?? [], name).toHaveLength(
        expectedCachedSetupNodeSteps,
      );
    }
  });

  it("gates every provider-secret job with stable actor IDs", async () => {
    const workflows = await Promise.all(
      [
        "runner-full-stack-e2e.yml",
        "runner-live-evals.yml",
        "runner-protocol-live-evals.yml",
        "e2e.yml",
      ].map(async (name) => ({
        name,
        contents: await readFile(
          path.join(repositoryRoot, ".github/workflows", name),
          "utf8",
        ),
      })),
    );

    for (const { name, contents } of workflows) {
      const authorize = contents.indexOf("  authorize:");
      const reauthorize = contents.indexOf("Reauthorize");
      const paidCheckout = contents.indexOf("actions/checkout@", reauthorize);
      const providerAccess = contents.search(
        /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./,
      );
      expect(
        authorize,
        `${name} must have an authorization job`,
      ).toBeGreaterThan(0);
      expect(
        reauthorize,
        `${name} must reauthorize partial job reruns`,
      ).toBeGreaterThan(authorize);
      expect(
        paidCheckout,
        `${name} must authorize before checkout`,
      ).toBeGreaterThan(reauthorize);
      expect(
        providerAccess,
        `${name} must authorize before provider access`,
      ).toBeGreaterThan(reauthorize);
      expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS");
      expect(contents).toContain("github.actor_id");
      expect(contents).toContain("github.triggering_actor");
      expect(contents).toContain("refs/heads/$DEFAULT_BRANCH");
      expect(contents).toContain("needs: authorize");
      expect(contents).toContain("name: runner-e2e-paid");
      expect(contents).not.toMatch(
        /^\s*(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
      );
      const actionReferences = [
        ...contents.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gm),
      ].map((match) => match[1]!);
      expect(actionReferences.length).toBeGreaterThan(0);
      for (const reference of actionReferences) {
        expect(reference).toMatch(/^[^@]+@[0-9a-f]{40}$/);
      }
    }

    const fullStack = workflows[0]!.contents;
    const paidJob = fullStack.slice(
      fullStack.indexOf("  test:"),
      fullStack.indexOf("  report:"),
    );
    const authorizeJob = fullStack.slice(
      fullStack.indexOf("  authorize:"),
      fullStack.indexOf("  target_lock:"),
    );
    const targetLockJob = fullStack.slice(
      fullStack.indexOf("  target_lock:"),
      fullStack.indexOf("  catalog:"),
    );
    const daytonaImageJob = fullStack.slice(
      fullStack.indexOf("  daytona_image:"),
      fullStack.indexOf("  build_runner_artifacts:"),
    );
    expect(daytonaImageJob).toMatch(buildRunnerNeeds);
    expect(daytonaImageJob).toContain(
      "runs-on: ${{ needs.authorize.outputs.test_runner }}",
    );
    expect(daytonaImageJob).not.toContain("name: runner-e2e-paid");
    expect(daytonaImageJob).not.toMatch(
      /(?:(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI)_API_KEY|GROK_AUTH_JSON)/,
    );
    expect(authorizeJob).toContain(
      "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
    );
    expect(authorizeJob).toContain("github_runner='ubuntu-latest'");
    expect(authorizeJob).toContain(
      "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
    );
    expect(authorizeJob).toContain(
      "playwright_channel: ${{ steps.runner.outputs.playwright_channel }}",
    );
    expect(authorizeJob).toContain('echo "playwright_channel=chrome"');
    expect(authorizeJob).toContain('echo "playwright_channel="');
    expect(authorizeJob).toContain(
      "Resolve requested repository branch to an immutable commit",
    );
    expect(authorizeJob).toContain(
      "repos/$REPOSITORY/branches/$encoded_branch",
    );
    expect(authorizeJob).toContain('echo "sha=$target_sha"');
    expect(authorizeJob).toContain(
      "target_ref: ${{ steps.target.outputs.ref }}",
    );
    expect(authorizeJob).toContain('echo "ref=refs/heads/$TARGET_BRANCH"');
    expect(authorizeJob).not.toContain("actions/checkout@");
    expect(authorizeJob).not.toContain("pnpm install");
    expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
    expect(targetLockJob).toContain("needs: authorize");
    expect(targetLockJob).toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(targetLockJob).toContain("persist-credentials: false");
    expect(targetLockJob).toContain(
      "pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only",
    );
    expect(targetLockJob).toContain(
      "artifact_id: ${{ steps.upload.outputs.artifact-id }}",
    );
    expect(targetLockJob).toContain("lock_sha256:");
    expect(targetLockJob).toContain(
      "runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}",
    );
    expect(targetLockJob).not.toContain("name: runner-e2e-paid");
    expect(targetLockJob).not.toMatch(
      /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
    );
    expect(paidJob).toContain(
      "runs-on: ${{ needs.authorize.outputs.test_runner }}",
    );
    expect(paidJob).toMatch(fullStackTestNeeds);
    expect(paidJob).toContain("name: runner-e2e-paid");
    expect(paidJob).toMatch(
      /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
    );
    expect(paidJob).toContain(
      "pnpm exec playwright install --with-deps --only-shell chromium",
    );
    expect(paidJob).toContain("name: Qualify preinstalled Chrome");
    expect(paidJob).toContain(
      "if: needs.authorize.outputs.playwright_channel == 'chrome'",
    );
    expect(paidJob).toContain("google-chrome --version");
    expect(paidJob).toContain(
      "if: needs.authorize.outputs.playwright_channel != 'chrome'",
    );
    expect(paidJob).toContain(
      "PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }}",
    );
    expect(paidJob).not.toContain(
      "pnpm exec playwright install --with-deps chromium",
    );
    const paidInstall = paidJob.indexOf(
      "pnpm install --frozen-lockfile --ignore-scripts",
    );
    const daytonaPluginPreparation = paidJob.indexOf(
      "Prepare bundled Daytona plugin without dependency lifecycle scripts",
    );
    const awsFfmpegInstall = paidJob.indexOf(
      "- name: Install Playwright FFmpeg on AWS runner",
    );
    const hostedChromiumInstall = paidJob.indexOf(
      "- name: Install Chromium headless shell on GitHub-hosted fallback",
    );
    const everydayOraclePreparation = paidJob.indexOf(
      "- name: Prepare pinned Python artifact oracle image",
    );
    const paidExecution = paidJob.indexOf("- name: Run bounded campaign and retain feedback review");
    expect(paidInstall).toBeGreaterThan(0);
    expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);
    expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation);
    expect(hostedChromiumInstall).toBeGreaterThan(awsFfmpegInstall);
    expect(everydayOraclePreparation).toBeGreaterThan(hostedChromiumInstall);
    expect(paidExecution).toBeGreaterThan(awsFfmpegInstall);
    expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation);
    expect(paidExecution).toBeGreaterThan(everydayOraclePreparation);
    const grokPreparation = paidJob.indexOf("- name: Install checksum-verified Grok executable");
    expect(grokPreparation).toBeGreaterThan(paidInstall);
    expect(paidExecution).toBeGreaterThan(grokPreparation);
    expect(paidJob).toContain("if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-acpx-grok' || matrix.profileId == 'runner-acpx-grok-subscription')");
    expect(paidJob).toContain("run: sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok");

    const everydayOracleStep = paidJob.slice(
      everydayOraclePreparation,
      paidExecution,
    );
    expect(everydayOracleStep).toContain(
      "if: (matrix.suiteId == 'everyday-workflows' || matrix.suiteId == 'grok-qualification' || matrix.suiteId == 'grok-subscription-qualification') && (matrix.caseId == 'build-revise' || matrix.caseId == 'delegate-feedback' || matrix.caseId == 'agent-review-handoff' || matrix.caseId == 'hire-reuse' || matrix.caseId == 'recover-controller' || matrix.caseId == 'stop-redirect')",
    );
    expect(everydayOracleStep).toContain(
      `oracle_image='${everydayOracleImage}'`,
    );
    expect(everydayOracleStep).toContain(
      "timeout 30s docker version --format '{{.Server.Version}}'",
    );
    expect(everydayOracleStep).toContain(
      'timeout 120s docker pull "$oracle_image"',
    );
    expect(everydayOracleStep).toContain(
      "timeout 30s docker image inspect \"$oracle_image\" --format '{{.Id}}'",
    );
    const artifactSource = await readFile(
      path.join(repositoryRoot, "tests/runner-e2e/everyday-artifact.py"),
      "utf8",
    );
    const artifactImage = artifactSource.match(
      /^SANDBOX_IMAGE = '([^']+)'$/mu,
    )?.[1];
    expect(artifactImage).toBe(everydayOracleImage);
    expect(everydayOracleStep).not.toMatch(/secrets\./u);
    const awsFfmpegStep = paidJob.slice(
      awsFfmpegInstall,
      hostedChromiumInstall,
    );
    expect(awsFfmpegStep).toContain(
      "if: needs.authorize.outputs.playwright_channel == 'chrome'",
    );
    expect(awsFfmpegStep).toContain("pnpm exec playwright install ffmpeg");
    expect(awsFfmpegStep).not.toMatch(
      /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
    );
    const preparedBeforeProviderAccess = paidJob.slice(
      daytonaPluginPreparation,
      paidExecution,
    );
    expect(preparedBeforeProviderAccess).toContain(
      "if: matrix.environmentId == 'daytona'",
    );
    expect(preparedBeforeProviderAccess).toContain(
      'test -f "$daytona_root/pnpm-lock.yaml"',
    );
    expect(preparedBeforeProviderAccess).toContain(
      "pnpm install --ignore-workspace --frozen-lockfile --ignore-scripts",
    );
    expect(preparedBeforeProviderAccess).not.toContain("--no-lockfile");
    expect(preparedBeforeProviderAccess).toContain(
      "node scripts/link-plugin-dev-sdk.mjs",
    );
    expect(preparedBeforeProviderAccess).toContain(
      '"@paperclipai/plugin-daytona"',
    );
    expect(preparedBeforeProviderAccess).toContain('"@paperclipai/plugin-sdk"');
    expect(preparedBeforeProviderAccess).toContain(
      'realpath "$daytona_root/node_modules/@paperclipai/plugin-sdk"',
    );
    expect(preparedBeforeProviderAccess).toContain(
      'pnpm --dir "$daytona_root" build',
    );
    expect(preparedBeforeProviderAccess).not.toContain("secrets.");
    expect(preparedBeforeProviderAccess).not.toContain("pnpm rebuild");
    expect(paidJob.slice(0, paidExecution)).not.toMatch(
      /secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
    );
    expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
    expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
    expect(fullStack).toContain("REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}");
    expect(fullStack).toContain('[ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]');
    expect(fullStack).toContain('[[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]]');

    expect(fullStack).toContain(
      '[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]',
    );
    expect(fullStack).toContain(
      "group: runner-full-stack-e2e-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }}",
    );
    expect(fullStack).toContain(
      "cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}",
    );
    expect(daytonaImageJob).toMatch(
      /- if: needs\.catalog\.outputs\.needs_daytona == 'true'\n\s+uses: actions\/checkout@[0-9a-f]{40}/u,
    );
    for (const stepName of [
      "Download resolved target lockfile",
      "Restore resolved target lockfile",
    ]) {
      expect(daytonaImageJob).toMatch(
        new RegExp(
          `- name: ${stepName}\\n\\s+if: needs\\.catalog\\.outputs\\.needs_daytona == 'true'`,
          "u",
        ),
      );
    }
    expect(daytonaImageJob).toMatch(
      /- name: No Daytona image needed\n\s+id: local_only\n\s+if: needs\.catalog\.outputs\.needs_daytona != 'true'/u,
    );
    expect(daytonaImageJob).toContain('echo "source_revision="');
    expect(daytonaImageJob).toContain('echo "content_id="');
    expect(daytonaImageJob).toContain(
      "IMAGE_CACHE: ghcr.io/paperclipai/paperclip-daytona-runner:e2e-buildcache-amd64",
    );
    expect(daytonaImageJob).toContain(
      "TARGET_REF: ${{ needs.authorize.outputs.target_ref }}",
    );
    expect(daytonaImageJob).toContain(
      "DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}",
    );
    const cacheRead = daytonaImageJob.indexOf(
      '--cache-from "type=registry,ref=${IMAGE_CACHE}"',
    );
    const trustedTargetCheck = daytonaImageJob.indexOf(
      'if [ "$TARGET_REF" = "refs/heads/$DEFAULT_BRANCH" ]; then',
    );
    const cacheWrite = daytonaImageJob.indexOf(
      '--cache-to "type=registry,ref=${IMAGE_CACHE},mode=max"',
    );
    expect(cacheRead).toBeGreaterThan(0);
    expect(trustedTargetCheck).toBeGreaterThan(cacheRead);
    expect(cacheWrite).toBeGreaterThan(trustedTargetCheck);
    expect(daytonaImageJob.slice(trustedTargetCheck, cacheWrite)).not.toContain(
      "secrets.",
    );
    const targetCodeJobs = [
      fullStack.slice(
        fullStack.indexOf("  catalog:"),
        fullStack.indexOf("  daytona_image:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  daytona_image:"),
        fullStack.indexOf("  build_runner_artifacts:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  build_runner_artifacts:"),
        fullStack.indexOf("  build_remote_provider_pack:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  build_remote_provider_pack:"),
        fullStack.indexOf("  test:"),
      ),
      paidJob,
    ];
    for (const targetCodeJob of targetCodeJobs) {
      const checkout = targetCodeJob.indexOf("actions/checkout@");
      const downloadLock = targetCodeJob.indexOf(
        "Download resolved target lockfile",
      );
      const restoreLock = targetCodeJob.indexOf(
        "Restore resolved target lockfile",
      );
      const setupNode = targetCodeJob.indexOf("actions/setup-node@");
      const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile");
      expect(checkout).toBeGreaterThan(0);
      expect(downloadLock).toBeGreaterThan(checkout);
      expect(restoreLock).toBeGreaterThan(downloadLock);
      if (setupNode >= 0) {
        expect(setupNode).toBeGreaterThan(restoreLock);
      }
      if (install >= 0) {
        expect(install).toBeGreaterThan(restoreLock);
      }
      expect(targetCodeJob).toContain(
        "artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}",
      );
      expect(targetCodeJob).toContain(
        "EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}",
      );
    }
    expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
      5,
    );
    expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
      5,
    );
    expect(
      fullStack.match(
        /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
      ),
    ).toHaveLength(6);
    expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
    expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8);
    expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
    expect(fullStack).toContain(
      "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
    );
    const reportJob = fullStack.slice(
      fullStack.indexOf("  report:"),
      fullStack.indexOf("  publish_history:"),
    );
    const historyJob = fullStack.slice(fullStack.indexOf("  publish_history:"));
    expect(reportJob).toContain("ref: ${{ github.sha }}");
    expect(reportJob).not.toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(reportJob).not.toContain("Download resolved target lockfile");
    expect(historyJob).toContain("ref: ${{ github.sha }}");
    expect(historyJob).not.toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(historyJob).not.toContain("Download resolved target lockfile");
    expect(fullStack).toContain(
      "if: always() && !cancelled() && needs.catalog.result == 'success'",
    );
    for (const targetProvenanceJob of [paidJob, reportJob]) {
      expect(targetProvenanceJob).toContain(
        "PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}",
      );
      expect(targetProvenanceJob).toContain(
        "PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }}",
      );
    }
    for (const [secret, condition] of Object.entries({
      OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
      ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
      OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'",
      XAI_API_KEY: "matrix.credentialName == 'XAI_API_KEY'",
      GROK_AUTH_JSON: "matrix.credentialName == 'GROK_AUTH_JSON'",
      COPILOT_GITHUB_TOKEN: "matrix.credentialName == 'COPILOT_GITHUB_TOKEN'",
      DAYTONA_API_KEY: "matrix.environmentId == 'daytona'",
    })) {
      expect(fullStack).toContain(
        `${secret}: \${{ ${condition} && secrets.${secret} || '' }}`,
      );
    }
  });

  it("keeps provider credentials inside explicitly gated paid workflows", async () => {
    const workflowDirectory = path.join(repositoryRoot, ".github/workflows");
    const allowedProviderWorkflows = new Set([
      "e2e.yml",
      "runner-full-stack-e2e.yml",
      "runner-live-evals.yml",
      "runner-protocol-live-evals.yml",
    ]);
    const names = (await readdir(workflowDirectory)).filter((name) =>
      /\.ya?ml$/.test(name),
    );

    for (const name of names) {
      const contents = await readFile(
        path.join(workflowDirectory, name),
        "utf8",
      );
      const providerSecretReferences = [
        ...contents.matchAll(
          /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY|COPILOT_GITHUB_TOKEN)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|XAI_API_KEY|GROK_AUTH_JSON|DAYTONA_API_KEY|COPILOT_GITHUB_TOKEN)['"]\])/g,
        ),
      ];
      if (providerSecretReferences.length > 0) {
        expect(
          allowedProviderWorkflows.has(name),
          `${name} must not receive provider credentials`,
        ).toBe(true);
      }
    }
  });

  it("uses bounded daily and weekly Product schedules while preserving Runner schedules", async () => {
    const workflows = await Promise.all(
      [
        "runner-full-stack-e2e.yml",
        "runner-live-evals.yml",
        "runner-protocol-live-evals.yml",
      ].map((name) =>
        readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
      ),
    );
    for (const [index, workflow] of workflows.entries()) {
      const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(match => match[1]!);
      if (index === 0) {
        expect(crons).toEqual(["47 8 * * *", "47 9 * * 0"]);
        expect(workflow).toContain("vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'");
        expect(workflow).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"');
        expect(workflow).toContain('echo "max_parallel=1"');
        expect(workflow).toContain("--max-automatic-retries 0");
        const paid = workflow.split("- name: Run bounded campaign and retain feedback review")[1]!.split("- name: Run paid cell")[0]!;
        expect([...paid.matchAll(/secrets\.([A-Z_]+)/g)].map(match => match[1])).toEqual(["BRAINTRUST_API_KEY"]);
      } else {
        expect(crons).toHaveLength(1);
        expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
      }
      expect(workflow).toContain("workflow_dispatch:");
    }
  });

  it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
      "utf8",
    );
    const buildJobStart = workflow.indexOf("  build_runner_artifacts:");
    const testJobStart = workflow.indexOf("  test:", buildJobStart);
    const reportJobStart = workflow.indexOf("  report:", testJobStart);
    const buildJob = workflow.slice(buildJobStart, testJobStart);
    const testJob = workflow.slice(testJobStart, reportJobStart);

    expect(buildJobStart).toBeGreaterThan(0);
    expect(testJobStart).toBeGreaterThan(buildJobStart);
    expect(buildJob).toMatch(buildRunnerNeeds);
    expect(buildJob).toMatch(buildRemoteProviderPackNeeds);
    expect(buildJob).not.toContain("environment:");
    expect(buildJob).not.toContain("secrets.");
    expect(
      buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g),
    ).toHaveLength(2);
    expect(buildJob).toContain(
      "pnpm --filter @paperclipai/paperclip-runner build:typescript",
    );
    expect(buildJob).toContain(
      "pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
    );
    expect(buildJob).toContain(
      "node packages/paperclip-runner/scripts/build-provider-pack.mjs",
    );
    expect(buildJob).toContain(
      "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
    );
    expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
    expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
    expect(buildJob).toContain(
      "build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}",
    );
    expect(buildJob).toContain(
      "needs.build_runner_artifacts.outputs.build_artifact_name",
    );
    expect(buildJob).toContain(
      "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
    );
    expect(buildJob).toContain(
      "runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
    );
    expect(buildJob).toContain(
      "runner-e2e-provider-pack-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}",
    );
    expect(buildJob).not.toContain("runner-e2e-build-${GITHUB_SHA}");
    expect(buildJob).not.toContain("runner-e2e-provider-pack-${GITHUB_SHA}");
    expect(workflow).toContain("needs_runner_typescript=");
    expect(workflow).toContain("needs_native_binaries=");
    expect(workflow).toContain("needs_remote_provider_pack=");

    expect(testJob).toMatch(fullStackTestNeeds);
    expect(testJob).toContain("Download immutable campaign outputs");
    expect(
      testJob.match(
        /if: startsWith\(matrix\.profileId, 'runner-'\) \|\| matrix\.suiteId == 'openrouter-model-breadth'/gu,
      ),
    ).toHaveLength(2);
    expect(testJob).toContain("Download immutable remote provider pack");
    expect(testJob).toContain(
      "needs.build_runner_artifacts.outputs.build_artifact_name",
    );
    expect(testJob).toContain(
      "needs.build_remote_provider_pack.outputs.provider_pack_artifact_name",
    );
    expect(testJob).toContain("sha256sum --check");
    expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
      testJob.indexOf("tar --extract"),
    );
    expect(testJob).toContain(
      "test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
    );
    expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
    expect(workflow).toContain("Qualify local provider Node interpreter");
    expect(testJob).toContain(
      "Materialize verified pinned OpenCode executable",
    );
    expect(testJob).toContain(
      "matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'",
    );
    expect(testJob).toContain(
      "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
    );
    expect(testJob).not.toContain("postinstall.mjs");
    expect(testJob).not.toContain("pnpm rebuild");
    expect(testJob).not.toContain("build:typescript");
    expect(testJob).not.toContain("build:runner-binaries");
    expect(testJob).not.toContain("build-provider-pack.mjs");
  });

  it("uses the reviewed AWS Chrome channel without weakening the local executable override", async () => {
    const config = await readFile(
      path.join(repositoryRoot, "tests/runner-e2e/playwright.config.ts"),
      "utf8",
    );

    expect(config).toContain(
      "process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL?.trim()",
    );
    expect(config).toContain("{ channel: playwrightChannel }");
    expect(config).toContain(
      "process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE?.trim()",
    );
    expect(config).toContain(
      "PAPERCLIP_PLAYWRIGHT_CHANNEL and PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE are mutually exclusive",
    );
  });

  it("keeps reporting jobs on the committed lockfile without registry re-resolution", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8",
    );
    for (const jobName of ["report", "publish_history"]) {
      const job = workflow.split(`\n  ${jobName}:`)[1]!.split(/\n  [a-z_]+:/u)[0]!;
      const install = job.indexOf("pnpm install --frozen-lockfile --ignore-scripts");
      expect(job).toContain("ref: ${{ github.sha }}");
      expect(job).not.toContain("resolved-target-lockfile");
      expect(job, jobName).not.toContain("--no-frozen-lockfile");
      expect(job, jobName).not.toContain("--lockfile-only");
      expect(install, jobName).toBeGreaterThan(job.indexOf("ref: ${{ github.sha }}"));
      const credentials = job.indexOf("aws-actions/configure-aws-credentials@");
      if (credentials >= 0) expect(install).toBeLessThan(credentials);
    }
  });

  it("binds rerun evidence and Pages artifacts to the exact workflow attempt", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
      "utf8",
    );
    const reportStart = workflow.indexOf("  report:");
    const publisherStart = workflow.indexOf("  publish_history:", reportStart);
    const report = workflow.slice(reportStart, publisherStart);
    const publisher = workflow.slice(publisherStart);

    expect(report).toContain("actions: read");
    expect(report).toContain(
      '"repos/$REPOSITORY/actions/runs/$RUN_ID/jobs?filter=all&per_page=100"',
    );
    expect(report).toContain("gh api --paginate --slurp");
    expect(report).toContain(
      '"repos/$REPOSITORY/actions/runs/$RUN_ID/attempts/$attempt"',
    );
    expect(report).toContain("--jq '{run_attempt, run_started_at}'");
    expect(report).toContain("pattern: runner-e2e-${{ github.run_id }}-*-*");
    expect(report).not.toContain(
      "pattern: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-*",
    );
    expect(report.match(/merge-multiple: false/g)).toHaveLength(2);
    expect(report).toContain("Select latest workflow attempt per cell");
    expect(report).toContain("tests/runner-e2e/select-rerun-artifacts.ts");
    expect(report).toContain(
      "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e",
    );
    expect(report).toContain(
      "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}",
    );
    const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}";
    expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);
    expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`);
    expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'");
    expect(
      report.indexOf("Select latest workflow attempt per cell"),
    ).toBeLessThan(report.indexOf("Collect blob reports"));
    expect(publisher).toContain(
      'echo "name=github-pages-${{ github.run_id }}-${{ github.run_attempt }}"',
    );
    expect(publisher).toContain(
      "pages_artifact_name: ${{ steps.pages_artifact_name.outputs.name }}",
    );
    expect(publisher).toContain(
      "name: ${{ steps.pages_artifact_name.outputs.name }}",
    );
    expect(publisher).toContain(
      "artifact_name: ${{ needs.publish_history.outputs.pages_artifact_name }}",
    );
  });

  it("keeps recurring reports private while preserving a content-free liveness receipt", async () => {
    const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
    const paid = workflow.split("\n  test:")[1]!.split("\n  report:")[0]!;
    const manual = paid.split("\n  test_recurring:")[0]!;
    const recurring = paid.split("\n  test_recurring:")[1]!;
    expect(manual).not.toContain("id-token: write");
    expect(manual).toContain("if: needs.catalog.outputs.operations_cadence == ''");
    expect(recurring).toContain("id-token: write");
    expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");
    expect(recurring).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");
    expect(recurring).toContain("steps: *paid_test_steps");
    expect(manual).toContain("steps: &paid_test_steps");
    const report = workflow.split("\n  report:")[1]!.split("\n  publish_history:")[0]!;
    const step = (source: string, name: string) => source.split(`      - name: ${name}\n`)[1]!.split(/\n      - /u)[0]!;
    for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) {
      expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''");
    }
    expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''");
    expect(step(paid, "Run bounded campaign and retain feedback review")).not.toContain("AWS_");
    for (const job of [paid, report]) {
      const credentials = step(job, "Obtain scoped private history credentials");
      expect(credentials).toContain("output-env-credentials: false");
      expect(credentials).toContain("disable-retry: true");
      expect(credentials).toContain("RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN");
    }
    expect(paid.indexOf("Verify private evidence retention before paid execution")).toBeLessThan(paid.indexOf("Run bounded campaign and retain feedback review"));
    expect(report).toContain("report > runner-e2e-merged-report/report.log 2>&1");
    const marker = step(report, "Retain report availability for the schedule monitor");
    expect(marker).toContain("steps.private_report.outcome == 'success'");
    expect(marker).toContain("path: private-report-status.json");
    expect(report).toContain('echo \'{"version":1,"privateReportRetained":true}\' > private-report-status.json');
    expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/);
  });

  it("isolates private archive trust from workflows that execute target-branch code", async () => {
    const workflows = path.join(repositoryRoot, ".github/workflows");
    const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8");
    const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8");
    const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"];
    const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]);
    expect(subjects).toEqual(environments);
    const recurring = source.split("\n  test_recurring:")[1]!.split("\n  report:")[0]!;
    const report = source.split("\n  report:")[1]!.split("\n  publish_history:")[0]!;
    for (const [index, job] of [recurring, report].entries()) {
      expect(job).toContain(`name: ${environments[index]}`);
      expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)");
    }
    expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha");
    for (const filename of await readdir(workflows)) {
      if (!/\.ya?ml$/u.test(filename) || filename === "runner-full-stack-e2e.yml") continue;
      const other = await readFile(path.join(workflows, filename), "utf8");
      for (const environment of environments) expect(other, filename).not.toContain(environment);
    }
    const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");
    expect(protocol).toContain("name: runner-e2e-paid");
    expect(protocol).toContain("needs.authorize.outputs.target_sha");
  });

  it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
      "utf8",
    );
    const publisher = workflow.slice(workflow.indexOf("  publish_history:"));
    expect(publisher).toContain("id-token: write");
    expect(publisher).toContain("name: runner-e2e-history");
    expect(publisher).toContain("aws-actions/configure-aws-credentials@");
    expect(publisher).toContain("RUNNER_E2E_HISTORY_AWS_ROLE_ARN");
    expect(publisher).not.toContain("cache: pnpm");
    expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/);
    expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/);
    expect(workflow).toContain("history_source_ready");
    expect(workflow).toContain("Verify normalized history source report");
    expect(workflow).not.toContain("sanitized_screenshot=");
    expect(workflow).toContain(
      "Publish S3 history and Pages bundle with declared screenshots",
    );
    expect(workflow).toContain(
      "Publish trusted summary and declared screenshots to public bundles",
    );
    expect(publisher).toContain(
      "pnpm exec playwright install --with-deps --only-shell chromium",
    );
    expect(workflow).toContain(
      "Package pruned dashboard with declared screenshots for GitHub Pages",
    );
    expect(workflow).toContain("path: runner-e2e-merged-report/pages");
    expect(workflow).toContain(
      "Publish latest dashboard with declared screenshots",
    );
    expect(workflow).not.toContain("dashboard_ready");
    expect(workflow).not.toContain("Publish latest screenshot dashboard");
    expect(
      workflow.indexOf("pnpm test:e2e:runner:history:publish"),
    ).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));
  });

  it("does not restore dependency caches or run install scripts in OIDC-authorized jobs", async () => {
    const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8");
    const jobs = workflow.split(/\n(?=  [a-z_]+:)/u);
    const paidSteps = workflow.split("steps: &paid_test_steps")[1]!.split("\n  test_recurring:")[0]!;
    const oidcJobs = jobs.filter(job => job.includes("id-token: write"));
    expect(oidcJobs.length).toBeGreaterThanOrEqual(3);
    for (const job of oidcJobs) {
      const steps = job + (job.includes("steps: *paid_test_steps") ? paidSteps : "");
      expect(steps, job.split("\n")[0]).not.toMatch(/cache:\s*(?:pnpm|npm|yarn)\b/u);
      expect(steps).not.toMatch(/actions\/cache(?:\/restore)?@/u);
      for (const install of steps.matchAll(/\bpnpm install[^\n]*/gu)) {
        expect(install[0]).toContain("--ignore-scripts");
      }
    }
    for (const name of ["report", "publish_history"]) {
      const job = jobs.find(value => value.startsWith(`  ${name}:`))!;
      const installs = [...job.matchAll(/\bpnpm install[^\n]*/gu)];
      expect(installs).toHaveLength(1);
      for (const install of installs) expect(install[0]).toContain('--store-dir "${{ runner.temp }}/eval-');
    }
  });
});