name: Chat feature TestOps run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} on: workflow_dispatch: inputs: operation: type: choice options: [evaluate, bootstrap, launch-plan] default: evaluate description: Fixed evaluation, zero-result registration, or one TestOps plan launch ALLURE_JOB_RUN_ID: type: string required: false description: TestOps supplies its job-run identity; no test/source selector is accepted ALLURE_USERNAME: type: string required: false description: Reserved TestOps service metadata; never used as authorization permissions: contents: read concurrency: group: chat-feature-testops cancel-in-progress: false jobs: authorize: runs-on: ubuntu-latest timeout-minutes: 5 outputs: sha: ${{ steps.guard.outputs.sha }} operation: ${{ steps.guard.outputs.operation }} triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci steps: - &checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.sha }} persist-credentials: false - &node uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - &guard name: Verify fixed default-source dispatch and both actor identities id: guard env: GH_TOKEN: ${{ github.token }} TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} NODE_OPTIONS: '' NODE_PATH: '' run: node tests/runner-e2e/testops-chat-dispatch.ts select: name: Validate TestOps selection or register zero-result setup needs: authorize runs-on: ubuntu-latest timeout-minutes: 10 environment: runner-e2e-testops-reporting permissions: contents: read actions: read outputs: artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} sha256: ${{ steps.selection.outputs.sha256 }} steps: - *checkout - *node - *guard - &allure if: needs.authorize.outputs.operation != 'launch-plan' uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 with: allurectl-version: 2.23.1 allure-endpoint: https://paperclip.testops.cloud allure-project-id: '1' allure-token: ${{ secrets.ALLURE_TOKEN }} - name: Register the public job without an evaluation or a synthetic result if: needs.authorize.outputs.operation == 'bootstrap' env: ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} ALLURE_ENDPOINT: https://paperclip.testops.cloud ALLURE_PROJECT_ID: '1' CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results run: | mkdir -m 700 "$ALLURE_RESULTS" allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - name: Validate the one real TestOps selection before any build or provider access id: selection if: needs.authorize.outputs.operation == 'evaluate' env: ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} ALLURE_ENDPOINT: https://paperclip.testops.cloud ALLURE_PROJECT_ID: '1' ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results run: | mkdir -m 700 "$ALLURE_RESULTS" allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection - name: Start exactly the validated TestOps plan and exit without waiting for its child if: needs.authorize.outputs.operation == 'launch-plan' env: ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} ALLURE_ENDPOINT: https://paperclip.testops.cloud ALLURE_PROJECT_ID: '1' CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} GH_TOKEN: ${{ github.token }} run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan - name: Retain only closed metadata from zero-result setup if: always() && needs.authorize.outputs.operation == 'bootstrap' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-testops-setup-${{ github.run_id }}-1 path: | ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json ${{ runner.temp }}/chat-testops-bootstrap-state/once.json ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json ${{ runner.temp }}/chat-testops-bootstrap-state/job.json ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json retention-days: 30 if-no-files-found: error - name: Retain the one plan-launch receipt and stop state if: always() && needs.authorize.outputs.operation == 'launch-plan' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-testops-plan-launch-${{ github.run_id }}-1 path: | ${{ runner.temp }}/chat-testops-launch-plan-state/once.json ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json retention-days: 30 if-no-files-found: error - name: Retain only the token-free exact selection receipt id: selection_upload if: steps.selection.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-testops-selection-${{ github.run_id }}-1 path: testops-selection.json retention-days: 30 if-no-files-found: error build: name: Build the immutable shipped native candidate without model credentials needs: [authorize, select] if: needs.authorize.outputs.operation == 'evaluate' runs-on: ${{ needs.authorize.outputs.runner }} timeout-minutes: 45 outputs: artifact_id: ${{ steps.bundle.outputs.artifact-id }} steps: - *checkout - *node - *guard - &pnpm uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 env: NPM_CONFIG_AUDIT: 'false' NPM_CONFIG_FUND: 'false' NPM_CONFIG_UPDATE_NOTIFIER: 'false' with: version: 9.15.4 - name: Install only the source-reviewed graph without lifecycle scripts run: | node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json pnpm install --frozen-lockfile --ignore-scripts - name: Build the existing TypeScript and native runner owners run: | pnpm --filter @paperclipai/paperclip-runner build:typescript pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - name: Package source-bound outputs and resolved lock run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$GITHUB_SHA" node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 - name: Retain credential-free immutable outputs id: bundle uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 path: | chat-testops-build.tar.gz chat-testops-build.tar.gz.sha256 retention-days: 1 if-no-files-found: error chat: name: One real native OpenCode backlog request on AWS needs: [authorize, select, build] if: needs.authorize.outputs.operation == 'evaluate' runs-on: ${{ needs.authorize.outputs.runner }} timeout-minutes: 32 environment: runner-e2e-private-paid permissions: contents: read id-token: write env: &retention PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} TARGET_SHA: ${{ needs.authorize.outputs.sha }} MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} NODE_OPTIONS: '' NODE_PATH: '' AWS_EC2_METADATA_DISABLED: 'true' steps: - *checkout - *node - *guard - *pnpm - name: Check out the same admitted source for trusted retention uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.authorize.outputs.sha }} path: .testops-trusted persist-credentials: false - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.build.outputs.artifact_id }} path: chat-testops-build - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.select.outputs.artifact_id }} path: chat-testops-selection - name: Verify immutable outputs and token-free TestOps selection env: EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} run: | set -euo pipefail (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json cmp chat-testops-lock.json chat-testops-worker-lock.json cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml pnpm install --frozen-lockfile --ignore-scripts (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") - name: Materialize and qualify pinned OpenCode runtime and browser run: | node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' google-chrome --version pnpm exec playwright install ffmpeg - name: Prepare private evidence with shipped-main authority id: private env: TARGET_ROOT: ${{ github.workspace }} run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - name: Obtain only the exact private cell storage session id: cell_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} inline-session-policy: ${{ steps.private.outputs.session_policy }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Verify strict private custody before the model key env: &cell_session PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - name: Execute only the fixed original request with zero retries env: OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - name: Retain exact original grade and private evidence, including failures if: always() && steps.cell_credentials.outcome == 'success' env: *cell_session run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell report: name: Retain private originals and export only reviewed fixture results needs: [authorize, select, chat] if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' runs-on: ubuntu-latest timeout-minutes: 15 environment: runner-e2e-private-history permissions: contents: read actions: read id-token: write env: *retention steps: - *checkout - *node - *guard - *pnpm - name: Install trusted reporting dependencies only run: | node tests/runner-e2e/prepare-report-lock.mjs pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" - name: Prepare exact private report and cell-read policies id: session run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - name: Obtain private report writer id: report_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} inline-session-policy: ${{ steps.session.outputs.session_policy }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Obtain exact private cell reader id: read_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Download the exact private original and verify all inventories env: PRIVATE_REPORT_ROOT: ${{ github.workspace }} SELECTED_CELL_INDEX: '0' AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} run: | node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - name: Build the existing normalized campaign without exposing original logs env: PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} run: | mkdir -p runner-e2e-merged-report pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - name: Retain the complete merged report privately even when its grade failed id: private_report if: always() && steps.report_credentials.outcome == 'success' env: PRIVATE_REPORT_ROOT: ${{ github.workspace }} PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - name: Export only the bound normalized result, JUnit and marked PNGs id: export if: always() && steps.private_report.outcome == 'success' env: TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts - name: Upload only the sanitized TestOps projection if: always() && steps.export.outcome == 'success' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-e2e-testops-${{ github.run_id }}-1 path: runner-e2e-testops/ retention-days: 30 if-no-files-found: error upload: name: Report the original outcome to its exact TestOps case needs: [authorize, select, chat, report] if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' runs-on: ubuntu-latest timeout-minutes: 10 environment: runner-e2e-testops-reporting permissions: contents: read actions: read steps: - *checkout - *node - *guard - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-e2e-testops-${{ github.run_id }}-1 path: runner-e2e-testops - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.select.outputs.artifact_id }} path: chat-testops-selection - *allure - name: Verify and upload only the bound safe case outcome id: allure_upload env: &testops_reporter ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} ALLURE_ENDPOINT: https://paperclip.testops.cloud ALLURE_PROJECT_ID: '1' ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops - name: Verify the actual TestOps result after upload processing id: verify_upload if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') env: *testops_reporter run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops - name: Retain only closed actual TestOps verification metadata if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-testops-verification-${{ github.run_id }}-1 path: | ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json retention-days: 30 if-no-files-found: error