PaperClipAI/tests/runner-e2e/workflow-security.test.ts

import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";

const repositoryRoot = path.resolve(import.meta.dirname, "../..");
const fullStackTestNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,\s*build_remote_provider_pack,?\s*\]/u;
const buildRunnerNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,?\s*\]/u;
const buildRemoteProviderPackNeeds =
  /needs:\s*\[\s*authorize,\s*target_lock,\s*catalog,\s*daytona_image,\s*build_runner_artifacts,?\s*\]/u;

describe("public repository paid workflow security", () => {
  it("gates every provider-secret job with stable actor IDs", async () => {
    const workflows = await Promise.all(
      ["runner-full-stack-e2e.yml", "runner-live-evals.yml", "e2e.yml"].map(
        async (name) => ({
          name,
          contents: await readFile(
            path.join(repositoryRoot, ".github/workflows", name),
            "utf8",
          ),
        }),
      ),
    );

    for (const { name, contents } of workflows) {
      const authorize = contents.indexOf("  authorize:");
      const reauthorize = contents.indexOf("Reauthorize");
      const paidCheckout = contents.indexOf("actions/checkout@", reauthorize);
      const providerAccess = contents.search(
        /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY:\s*\$\{\{\s*[^}]*secrets\./,
      );
      expect(
        authorize,
        `${name} must have an authorization job`,
      ).toBeGreaterThan(0);
      expect(
        reauthorize,
        `${name} must reauthorize partial job reruns`,
      ).toBeGreaterThan(authorize);
      expect(
        paidCheckout,
        `${name} must authorize before checkout`,
      ).toBeGreaterThan(reauthorize);
      expect(
        providerAccess,
        `${name} must authorize before provider access`,
      ).toBeGreaterThan(reauthorize);
      expect(contents).toContain("RUNNER_E2E_ALLOWED_ACTOR_IDS");
      expect(contents).toContain("github.actor_id");
      expect(contents).toContain("github.triggering_actor");
      expect(contents).toContain("refs/heads/$DEFAULT_BRANCH");
      expect(contents).toContain("needs: authorize");
      expect(contents).toContain("name: runner-e2e-paid");
      expect(contents).not.toMatch(
        /^\s*(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
      );
      const actionReferences = [
        ...contents.matchAll(/^\s*(?:-\s*)?uses:\s*([^\s#]+)/gm),
      ].map((match) => match[1]!);
      expect(actionReferences.length).toBeGreaterThan(0);
      for (const reference of actionReferences) {
        expect(reference).toMatch(/^[^@]+@[0-9a-f]{40}$/);
      }
    }

    const fullStack = workflows[0]!.contents;
    const paidJob = fullStack.slice(
      fullStack.indexOf("  test:"),
      fullStack.indexOf("  report:"),
    );
    const authorizeJob = fullStack.slice(
      fullStack.indexOf("  authorize:"),
      fullStack.indexOf("  target_lock:"),
    );
    const targetLockJob = fullStack.slice(
      fullStack.indexOf("  target_lock:"),
      fullStack.indexOf("  catalog:"),
    );
    expect(authorizeJob).toContain(
      "aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci'",
    );
    expect(authorizeJob).toContain("github_runner='ubuntu-latest'");
    expect(authorizeJob).toContain(
      "AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }}",
    );
    expect(authorizeJob).toContain(
      "Resolve requested repository branch to an immutable commit",
    );
    expect(authorizeJob).toContain(
      "repos/$REPOSITORY/branches/$encoded_branch",
    );
    expect(authorizeJob).toContain('echo "sha=$target_sha"');
    expect(authorizeJob).not.toContain("actions/checkout@");
    expect(authorizeJob).not.toContain("pnpm install");
    expect(targetLockJob).toContain("name: Resolve target pnpm lockfile");
    expect(targetLockJob).toContain("needs: authorize");
    expect(targetLockJob).toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(targetLockJob).toContain("persist-credentials: false");
    expect(targetLockJob).toContain(
      "pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only",
    );
    expect(targetLockJob).toContain(
      "artifact_id: ${{ steps.upload.outputs.artifact-id }}",
    );
    expect(targetLockJob).toContain("lock_sha256:");
    expect(targetLockJob).toContain(
      "runner-e2e-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }}",
    );
    expect(targetLockJob).not.toContain("name: runner-e2e-paid");
    expect(targetLockJob).not.toMatch(
      /(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
    );
    expect(paidJob).toContain(
      "runs-on: ${{ needs.authorize.outputs.test_runner }}",
    );
    expect(paidJob).toMatch(fullStackTestNeeds);
    expect(paidJob).toContain("name: runner-e2e-paid");
    expect(paidJob).toMatch(
      /Reauthorize paid execution before provider access[\s\S]*actions\/checkout@[0-9a-f]{40}[\s\S]*persist-credentials: false[\s\S]*Download resolved target lockfile/,
    );
    const paidInstall = paidJob.indexOf(
      "pnpm install --frozen-lockfile --ignore-scripts",
    );
    const daytonaPluginPreparation = paidJob.indexOf(
      "Prepare bundled Daytona plugin without dependency lifecycle scripts",
    );
    const paidExecution = paidJob.indexOf("- name: Run paid cell");
    expect(paidInstall).toBeGreaterThan(0);
    expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall);
    expect(paidExecution).toBeGreaterThan(daytonaPluginPreparation);
    const preparedBeforeProviderAccess = paidJob.slice(
      daytonaPluginPreparation,
      paidExecution,
    );
    expect(preparedBeforeProviderAccess).toContain(
      "if: matrix.environmentId == 'daytona'",
    );
    expect(preparedBeforeProviderAccess).toContain(
      'test -f "$daytona_root/pnpm-lock.yaml"',
    );
    expect(preparedBeforeProviderAccess).toContain(
      "pnpm install --ignore-workspace --frozen-lockfile --ignore-scripts",
    );
    expect(preparedBeforeProviderAccess).not.toContain("--no-lockfile");
    expect(preparedBeforeProviderAccess).toContain(
      "node scripts/link-plugin-dev-sdk.mjs",
    );
    expect(preparedBeforeProviderAccess).toContain(
      '"@paperclipai/plugin-daytona"',
    );
    expect(preparedBeforeProviderAccess).toContain('"@paperclipai/plugin-sdk"');
    expect(preparedBeforeProviderAccess).toContain(
      'realpath "$daytona_root/node_modules/@paperclipai/plugin-sdk"',
    );
    expect(preparedBeforeProviderAccess).toContain(
      'pnpm --dir "$daytona_root" build',
    );
    expect(preparedBeforeProviderAccess).not.toContain("secrets.");
    expect(preparedBeforeProviderAccess).not.toContain("pnpm rebuild");
    expect(paidJob.slice(0, paidExecution)).not.toMatch(
      /secrets\.(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA)_API_KEY/,
    );
    expect(authorizeJob).toContain('echo "max_parallel_limit=100"');
    expect(fullStack).toContain('[ "$MAX_PARALLEL_LIMIT" -gt 100 ]');
    expect(fullStack).toContain(
      '[ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]',
    );
    expect(fullStack).toContain(
      "group: runner-full-stack-e2e-${{ inputs.target_branch || github.event.repository.default_branch }}",
    );
    expect(fullStack).toContain(
      "cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }}",
    );
    const targetCodeJobs = [
      fullStack.slice(
        fullStack.indexOf("  catalog:"),
        fullStack.indexOf("  daytona_image:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  daytona_image:"),
        fullStack.indexOf("  build_runner_artifacts:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  build_runner_artifacts:"),
        fullStack.indexOf("  build_remote_provider_pack:"),
      ),
      fullStack.slice(
        fullStack.indexOf("  build_remote_provider_pack:"),
        fullStack.indexOf("  test:"),
      ),
      paidJob,
    ];
    for (const targetCodeJob of targetCodeJobs) {
      const checkout = targetCodeJob.indexOf("actions/checkout@");
      const downloadLock = targetCodeJob.indexOf(
        "Download resolved target lockfile",
      );
      const restoreLock = targetCodeJob.indexOf(
        "Restore resolved target lockfile",
      );
      const setupNode = targetCodeJob.indexOf("actions/setup-node@");
      const install = targetCodeJob.indexOf("pnpm install --frozen-lockfile");
      expect(checkout).toBeGreaterThan(0);
      expect(downloadLock).toBeGreaterThan(checkout);
      expect(restoreLock).toBeGreaterThan(downloadLock);
      if (setupNode >= 0) {
        expect(setupNode).toBeGreaterThan(restoreLock);
      }
      if (install >= 0) {
        expect(install).toBeGreaterThan(restoreLock);
      }
      expect(targetCodeJob).toContain(
        "artifact-ids: ${{ needs.target_lock.outputs.artifact_id }}",
      );
      expect(targetCodeJob).toContain(
        "EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }}",
      );
    }
    expect(fullStack.match(/Download resolved target lockfile/g)).toHaveLength(
      5,
    );
    expect(fullStack.match(/Restore resolved target lockfile/g)).toHaveLength(
      5,
    );
    expect(
      fullStack.match(
        /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g,
      ),
    ).toHaveLength(6);
    expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(2);
    expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(8);
    expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}");
    expect(fullStack).toContain(
      "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}",
    );
    const reportJob = fullStack.slice(
      fullStack.indexOf("  report:"),
      fullStack.indexOf("  publish_history:"),
    );
    const historyJob = fullStack.slice(fullStack.indexOf("  publish_history:"));
    expect(reportJob).toContain("ref: ${{ github.sha }}");
    expect(reportJob).not.toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(reportJob).not.toContain("Download resolved target lockfile");
    expect(historyJob).toContain("ref: ${{ github.sha }}");
    expect(historyJob).not.toContain(
      "ref: ${{ needs.authorize.outputs.target_sha }}",
    );
    expect(historyJob).not.toContain("Download resolved target lockfile");
    for (const [secret, condition] of Object.entries({
      OPENAI_API_KEY: "matrix.credentialName == 'OPENAI_API_KEY'",
      ANTHROPIC_API_KEY: "matrix.credentialName == 'ANTHROPIC_API_KEY'",
      OPENROUTER_API_KEY: "matrix.credentialName == 'OPENROUTER_API_KEY'",
      DAYTONA_API_KEY: "matrix.environmentId == 'daytona'",
    })) {
      expect(fullStack).toContain(
        `${secret}: \${{ ${condition} && secrets.${secret} || '' }}`,
      );
    }
  });

  it("keeps provider credentials inside explicitly gated paid workflows", async () => {
    const workflowDirectory = path.join(repositoryRoot, ".github/workflows");
    const allowedProviderWorkflows = new Set([
      "e2e.yml",
      "runner-full-stack-e2e.yml",
      "runner-live-evals.yml",
    ]);
    const names = (await readdir(workflowDirectory)).filter((name) =>
      /\.ya?ml$/.test(name),
    );

    for (const name of names) {
      const contents = await readFile(
        path.join(workflowDirectory, name),
        "utf8",
      );
      const providerSecretReferences = [
        ...contents.matchAll(
          /secrets(?:\.(?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)\b|\[['"](?:OPENAI_API_KEY|ANTHROPIC_API_KEY|OPENROUTER_API_KEY|DAYTONA_API_KEY)['"]\])/g,
        ),
      ];
      if (providerSecretReferences.length > 0) {
        expect(
          allowedProviderWorkflows.has(name),
          `${name} must not receive provider credentials`,
        ).toBe(true);
      }
    }
  });

  it("runs paid scheduled campaigns only on Sundays", async () => {
    const workflows = await Promise.all(
      ["runner-full-stack-e2e.yml", "runner-live-evals.yml"].map((name) =>
        readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"),
      ),
    );
    for (const workflow of workflows) {
      const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(
        (match) => match[1]!,
      );
      expect(crons).toHaveLength(1);
      expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/);
      expect(workflow).toContain("workflow_dispatch:");
    }
  });

  it("builds runner outputs once without provider credentials and verifies them in every paid cell", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
      "utf8",
    );
    const buildJobStart = workflow.indexOf("  build_runner_artifacts:");
    const testJobStart = workflow.indexOf("  test:", buildJobStart);
    const reportJobStart = workflow.indexOf("  report:", testJobStart);
    const buildJob = workflow.slice(buildJobStart, testJobStart);
    const testJob = workflow.slice(testJobStart, reportJobStart);

    expect(buildJobStart).toBeGreaterThan(0);
    expect(testJobStart).toBeGreaterThan(buildJobStart);
    expect(buildJob).toMatch(buildRunnerNeeds);
    expect(buildJob).toMatch(buildRemoteProviderPackNeeds);
    expect(buildJob).not.toContain("environment:");
    expect(buildJob).not.toContain("secrets.");
    expect(
      buildJob.match(/pnpm install --frozen-lockfile --ignore-scripts/g),
    ).toHaveLength(2);
    expect(buildJob).toContain(
      "pnpm --filter @paperclipai/paperclip-runner build:typescript",
    );
    expect(buildJob).toContain(
      "pnpm --filter @paperclipai/paperclip-runner build:runner-binaries",
    );
    expect(buildJob).toContain(
      "node packages/paperclip-runner/scripts/build-provider-pack.mjs",
    );
    expect(buildJob).toContain(
      "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
    );
    expect(buildJob).toContain("runner-e2e-build-bundle.tar.gz.sha256");
    expect(buildJob).toContain("runner-e2e-provider-pack.tar.gz.sha256");
    expect(buildJob).toContain(
      "build_artifact_name: ${{ steps.build_artifact_name.outputs.name }}",
    );
    expect(buildJob).toContain(
      "needs.build_runner_artifacts.outputs.build_artifact_name",
    );
    expect(buildJob).toContain(
      "provider_pack_artifact_name: ${{ steps.provider_pack_artifact_name.outputs.name }}",
    );
    expect(workflow).toContain("needs_runner_typescript=");
    expect(workflow).toContain("needs_native_binaries=");
    expect(workflow).toContain("needs_remote_provider_pack=");

    expect(testJob).toMatch(fullStackTestNeeds);
    expect(testJob).toContain("Download immutable campaign outputs");
    expect(testJob).toContain("Download immutable remote provider pack");
    expect(testJob).toContain(
      "needs.build_runner_artifacts.outputs.build_artifact_name",
    );
    expect(testJob).toContain(
      "needs.build_remote_provider_pack.outputs.provider_pack_artifact_name",
    );
    expect(testJob).toContain("sha256sum --check");
    expect(testJob.indexOf("sha256sum --check")).toBeLessThan(
      testJob.indexOf("tar --extract"),
    );
    expect(testJob).toContain(
      "test -x packages/paperclip-runner/runner/target/debug/paperclip-runnerd",
    );
    expect(testJob).toContain(".payload.runnerSourceRevision == $revision");
    expect(workflow).toContain("Qualify local provider Node interpreter");
    expect(testJob).toContain(
      "Materialize verified pinned OpenCode executable",
    );
    expect(testJob).toContain(
      "matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'",
    );
    expect(testJob).toContain(
      "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",
    );
    expect(testJob).not.toContain("postinstall.mjs");
    expect(testJob).not.toContain("pnpm rebuild");
    expect(testJob).not.toContain("build:typescript");
    expect(testJob).not.toContain("build:runner-binaries");
    expect(testJob).not.toContain("build-provider-pack.mjs");
  });

  it("uses environment-scoped OIDC for a no-delete history publisher", async () => {
    const workflow = await readFile(
      path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"),
      "utf8",
    );
    const publisher = workflow.slice(workflow.indexOf("  publish_history:"));
    expect(publisher).toContain("id-token: write");
    expect(publisher).toContain("name: runner-e2e-history");
    expect(publisher).toContain("aws-actions/configure-aws-credentials@");
    expect(publisher).toContain("RUNNER_E2E_HISTORY_AWS_ROLE_ARN");
    expect(publisher).not.toContain("cache: pnpm");
    expect(publisher).not.toMatch(/AWS_(?:ACCESS|SECRET)_KEY/);
    expect(publisher).not.toMatch(/aws s3 (?:rm|sync .*--delete)/);
    expect(workflow).toContain("history_source_ready");
    expect(workflow).toContain(
      "Verify history source report and private screenshot evidence",
    );
    expect(workflow).toContain("private_screenshot=");
    expect(workflow).toContain("Publish pruned immutable history");
    expect(workflow).toContain("Publish latest structured dashboard");
    expect(workflow).not.toContain("dashboard_ready");
    expect(workflow).not.toContain("Publish latest screenshot dashboard");
    expect(
      workflow.indexOf("pnpm test:e2e:runner:history:publish"),
    ).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@"));
  });
});