name: Chat catalog TestOps
run-name: Chat catalog TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}

on:
  workflow_dispatch:
    inputs:
      operation:
        type: choice
        options: [evaluate, bootstrap]
        default: evaluate
        description: Run the exact TestOps selection, or register its catalog plan
      ALLURE_JOB_RUN_ID:
        type: string
        required: false
        description: TestOps job-run identity; source, model and clocks are fixed
      ALLURE_USERNAME:
        type: string
        required: false
        description: Reserved service metadata; never grants authority

permissions:
  contents: read
  actions: read

concurrency:
  group: chat-catalog-testops-streaming-v1
  cancel-in-progress: false

env:
  NODE_OPTIONS: ''
  NODE_PATH: ''
  AWS_EC2_METADATA_DISABLED: 'true'

jobs:
  authorize:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      operation: ${{ steps.guard.outputs.operation }}
      target_sha: ${{ steps.guard.outputs.target_sha }}
      runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
    steps:
      - &checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          ref: ${{ github.sha }}
          persist-credentials: false
      - &node
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: 24
      - &guard
        name: Verify actual main-only dispatch and both actors
        id: guard
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
        run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard

  select:
    name: Register the dedicated plan or validate its real TestOps selection
    needs: authorize
    runs-on: ubuntu-latest
    timeout-minutes: 15
    environment: runner-e2e-testops-reporting
    outputs:
      artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
      selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
      execution_ids: ${{ steps.selection.outputs.execution_ids }}
      matrix: ${{ steps.selection.outputs.matrix }}
      native_matrix: ${{ steps.selection.outputs.native_matrix }}
      external_matrix: ${{ steps.selection.outputs.external_matrix }}
      native_count: ${{ steps.selection.outputs.native_count }}
      external_count: ${{ steps.selection.outputs.external_count }}
      parallelism: ${{ steps.selection.outputs.parallelism }}
      policy: ${{ steps.selection.outputs.policy }}
    steps:
      - *checkout
      - *node
      - *guard
      - if: needs.authorize.outputs.operation != 'launch-plan'
        uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
        with:
          allurectl-version: 2.23.1
          allure-endpoint: https://paperclip.testops.cloud
          allure-project-id: '1'
          allure-token: ${{ secrets.ALLURE_TOKEN }}
      - name: Register all 244 catalog cases without synthetic results
        if: needs.authorize.outputs.operation == 'bootstrap'
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
          ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
          ALLURE_ENDPOINT: https://paperclip.testops.cloud
          ALLURE_PROJECT_ID: '1'
          ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results
          CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
        run: |
          mkdir -m 700 "$ALLURE_RESULTS"
          allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap
          node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context
      - name: Validate case, plan, job, launch and same-run source before provider access
        id: selection
        if: needs.authorize.outputs.operation == 'evaluate'
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
          ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
          ALLURE_ENDPOINT: https://paperclip.testops.cloud
          ALLURE_PROJECT_ID: '1'
          ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
          ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results
          CHAT_FULL_CATALOG_TESTOPS_BINDING: ${{ vars.CHAT_FULL_CATALOG_TESTOPS_BINDING }}
        run: |
          mkdir -m 700 "$ALLURE_RESULTS"
          allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection
      - name: Retain token-free registration receipts
        if: always() && needs.authorize.outputs.operation != 'evaluate'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: chat-completion-testops-setup-${{ github.run_id }}-1
          path: |
            ${{ runner.temp }}/chat-completion-testops-bootstrap/
          retention-days: 30
          if-no-files-found: error
      - name: Retain the exact authenticated selection
        id: selection_upload
        if: steps.selection.outcome == 'success'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: chat-completion-testops-selection-${{ github.run_id }}-1
          path: testops-pilot-selection.json
          retention-days: 30
          if-no-files-found: error

  build:
    name: Build the immutable catalog source without service or model credentials
    needs: [authorize, select]
    if: needs.authorize.outputs.operation == 'evaluate'
    runs-on: ${{ needs.authorize.outputs.runner }}
    timeout-minutes: 45
    outputs:
      artifact_id: ${{ steps.bundle.outputs.artifact-id }}
    steps:
      - *checkout
      - *node
      - *guard
      - &pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
        env:
          NPM_CONFIG_AUDIT: 'false'
          NPM_CONFIG_FUND: 'false'
          NPM_CONFIG_UPDATE_NOTIFIER: 'false'
        with:
          version: 9.15.4
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          ref: ${{ needs.authorize.outputs.target_sha }}
          persist-credentials: false
      - name: Reject target-owned trusted helper paths
        id: target_boundary
        run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          ref: ${{ github.sha }}
          path: .testops-trusted
          persist-credentials: false
      - name: Resolve the fixed candidate graph with lifecycle scripts disabled
        run: |
          pnpm install --lockfile-only --ignore-scripts
          pnpm install --frozen-lockfile --ignore-scripts
      - name: Build existing TypeScript and native Runner outputs
        run: |
          pnpm --filter @paperclipai/paperclip-runner build:typescript
      - name: Exercise the real workflow and pinned source host contract before model access
        run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs
      - name: Build native Runner outputs
        run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
      - name: Package the resolved lock and exact source outputs
        env:
          TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
        run: |
          set -euo pipefail
          test "$(git rev-parse HEAD)" = "$TARGET_SHA"
          tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
          sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256
      - name: Retain credential-free immutable outputs
        id: bundle
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: chat-completion-build-${{ github.run_id }}-1
          path: |
            chat-completion-build.tar.gz
            chat-completion-build.tar.gz.sha256
          retention-days: 1
          if-no-files-found: error

  chat:
    name: ${{ matrix.executionId }}
    needs: [authorize, select, build]
    if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.native_count != '0'
    strategy:
      fail-fast: false
      max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}
      matrix: ${{ fromJSON(needs.select.outputs.native_matrix) }}
    permissions:
      contents: read
      actions: read
      id-token: write
    uses: ./.github/workflows/chat-catalog-testops-case.yml
    with:
      protocol: streaming-v1
      cell: ${{ toJSON(matrix) }}
      target_sha: ${{ needs.authorize.outputs.target_sha }}
      runner: ${{ needs.authorize.outputs.runner }}
      selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
      selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
      build_artifact_id: ${{ needs.build.outputs.artifact_id }}
      ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
    secrets:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}

  chat_external:
    name: ${{ matrix.executionId }}
    needs: [authorize, select, build]
    if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.external_count != '0'
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}
    permissions:
      contents: read
      actions: read
      id-token: write
    uses: ./.github/workflows/chat-catalog-testops-case.yml
    with:
      protocol: streaming-v1
      cell: ${{ toJSON(matrix) }}
      target_sha: ${{ needs.authorize.outputs.target_sha }}
      runner: ${{ needs.authorize.outputs.runner }}
      selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
      selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
      build_artifact_id: ${{ needs.build.outputs.artifact_id }}
      ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
    secrets:
      OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}

  aggregate:
    name: Validate and merge every selected metadata-only projection
    needs: [authorize, select, build, chat, chat_external]
    if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure')
    runs-on: ubuntu-latest
    timeout-minutes: 15
    outputs:
      artifact_id: ${{ steps.projection.outputs.artifact-id }}
    steps:
      - *checkout
      - *node
      - *guard
      - *pnpm
      - name: Install trusted reporting dependencies without lifecycle scripts
        run: |
          node tests/runner-e2e/prepare-report-lock.mjs
          pnpm install --frozen-lockfile --ignore-scripts
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          artifact-ids: ${{ needs.select.outputs.artifact_id }}
          path: chat-completion-testops-selection
      # A single pattern match is flattened by this pinned action. Bind its
      # exact name and explicit child destination instead of inferring a name.
      - if: fromJSON(needs.select.outputs.execution_ids)[1] == null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: chat-catalog-case-bundle-${{ github.run_id }}-1-0
          path: catalog-bundles/chat-catalog-case-bundle-${{ github.run_id }}-1-0
      - if: fromJSON(needs.select.outputs.execution_ids)[1] != null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
          merge-multiple: false
          path: catalog-bundles
      - name: Gather only safe singleton projections from immutable bundles
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
          ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
          EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
          CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles
        run: node tests/runner-e2e/chat-completion-testops-cli.mjs gather-case-projections catalog-projections
      - name: Authenticate selection and validate all singleton exports before aggregation
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
          ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
          EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
          TESTOPS_CATALOG_SELECTION: chat-completion-testops-selection/testops-pilot-selection.json
        run: |
          node tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection "$TESTOPS_CATALOG_SELECTION"
          node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-full-catalog-testops-merge.ts
      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        id: projection
        with:
          name: chat-catalog-testops-projection-${{ github.run_id }}-1
          path: runner-e2e-testops/
          retention-days: 30
          if-no-files-found: error

  verify:
    name: Verify final coverage without uploading any result again
    needs: [authorize, select, aggregate]
    if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.aggregate.result == 'success'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    environment: runner-e2e-testops-reporting
    steps:
      - *checkout
      - *node
      - *guard
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          artifact-ids: ${{ needs.aggregate.outputs.artifact_id }}
          path: runner-e2e-testops
      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          artifact-ids: ${{ needs.select.outputs.artifact_id }}
          path: chat-completion-testops-selection
      # A single pattern match is flattened by this pinned action. Bind its
      # exact name and explicit child destination instead of inferring a name.
      - if: fromJSON(needs.select.outputs.execution_ids)[1] == null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: chat-catalog-case-bundle-${{ github.run_id }}-1-0
          path: catalog-bundles/chat-catalog-case-bundle-${{ github.run_id }}-1-0
      - if: fromJSON(needs.select.outputs.execution_ids)[1] != null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
          merge-multiple: false
          path: catalog-bundles
      # A single pattern match is flattened by this pinned action. Bind its
      # exact name and explicit child destination instead of inferring a name.
      - if: fromJSON(needs.select.outputs.execution_ids)[1] == null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          name: chat-catalog-case-publication-${{ github.run_id }}-1-0
          path: catalog-publications/chat-catalog-case-publication-${{ github.run_id }}-1-0
      - if: fromJSON(needs.select.outputs.execution_ids)[1] != null
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          pattern: chat-catalog-case-publication-${{ github.run_id }}-1-*
          merge-multiple: false
          path: catalog-publications
      - name: Authenticate every frozen publication and verify the exact original selection
        id: verification
        env:
          GH_TOKEN: ${{ github.token }}
          TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
          ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
          ALLURE_ENDPOINT: https://paperclip.testops.cloud
          ALLURE_PROJECT_ID: '1'
          ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
          EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
          CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles
          CHAT_TESTOPS_PUBLICATIONS: ${{ github.workspace }}/catalog-publications
        run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-stream runner-e2e-testops
      - name: Retain closed final coverage verification
        if: always() && !cancelled()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: chat-completion-testops-verification-${{ github.run_id }}-1
          path: ${{ runner.temp }}/chat-completion-testops-verify-stream/
          retention-days: 30
          if-no-files-found: error