PaperClipAI/.github/workflows/runner-chaos-evals.yml

name: Runner Chaos Evals

on:
  schedule:
    - cron: "43 7 * * 0"
  workflow_dispatch:
  workflow_call:
    inputs:
      ref:
        description: Commit SHA, branch, or tag to verify before release
        required: false
        type: string

concurrency:
  # Reusable calls inherit the caller's workflow name. Cloud readiness and
  # Release verify the same SHA independently and must not cancel each other.
  group: runner-chaos-evals-${{ github.workflow }}-${{ inputs.ref || github.ref }}
  cancel-in-progress: true

jobs:
  chaos_and_recovery:
    name: Restart, replay, trace, and recovery faults
    runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
    timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }}
    permissions:
      contents: read

    steps:
      - name: Checkout repository
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          ref: ${{ inputs.ref || github.sha }}

      - name: Setup pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
        with:
          version: 9.15.4

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: 24
          cache: pnpm

      - name: Install dependencies
        run: pnpm install --no-frozen-lockfile

      - name: Build eval and Runner contracts
        run: |
          pnpm --filter @paperclipai/paperclip-eval-kernel build
          pnpm --filter @paperclipai/paperclip-runner report:runner-chaos-evals

      - name: Run Runner fault and replay suites
        run: |
          pnpm --filter @paperclipai/paperclip-runner exec vitest run \
            src/eval/workflow-evals.test.ts \
            src/native-session-runtime.test.ts \
            src/live/live-session.test.ts \
            src/live/turn-stream.test.ts \
            src/protocol/replay-contract.test.ts \
            src/drivers/opencode/mcp-bridge.test.ts \
            src/drivers/acpx/runtime-host.test.ts

      - name: Build server test dependencies
        run: pnpm --filter @paperclipai/plugin-sdk ensure-build-deps

      - name: Run server finalization and recovery suites
        run: |
          pnpm --filter @paperclipai/server exec vitest run \
            src/__tests__/native-finalization-recovery.test.ts \
            src/__tests__/heartbeat-process-recovery.test.ts \
            src/__tests__/heartbeat-comment-wake-batching.test.ts \
            src/__tests__/heartbeat-dependency-scheduling.test.ts \
            src/__tests__/provider-trace-store.test.ts \
            src/services/issue-thread-interaction-resolution.test.ts \
            src/services/recovery/successful-run-handoff.test.ts

      - name: Upload chaos eval bundle
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: runner-chaos-evals-${{ github.run_id }}
          path: packages/paperclip-runner/.paperclip-local/evals/workflows/
          retention-days: 30
          if-no-files-found: error