diff --git a/.github/workflows/chat-completion-testops.yml b/.github/workflows/chat-completion-testops.yml deleted file mode 100644 index 7016f76dfec..00000000000 --- a/.github/workflows/chat-completion-testops.yml +++ /dev/null @@ -1,504 +0,0 @@ -name: Chat completion TestOps -run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Run the exact TestOps selection, register its plan, or launch it once - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps job-run identity; source, model and clocks are fixed - ALLURE_USERNAME: - type: string - required: false - description: Reserved service metadata; never grants authority - -permissions: - contents: read - actions: read - -concurrency: - group: chat-completion-testops - cancel-in-progress: false - -env: - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - operation: ${{ steps.guard.outputs.operation }} - target_sha: ${{ steps.guard.outputs.target_sha }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify actual main-only dispatch and both actors - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard - - select: - name: Register the dedicated plan or validate its real TestOps selection - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-testops-reporting - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} - execution_ids: ${{ steps.selection.outputs.execution_ids }} - matrix: ${{ steps.selection.outputs.matrix }} - policy: ${{ steps.selection.outputs.policy }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register three exact cases with no synthetic test result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap - node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context - - name: Validate case, plan, job, launch and same-run source before provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection - - name: Request the one reviewed plan launch and exit - if: needs.authorize.outputs.operation == 'launch-plan' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }} - CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan - - name: Retain token-free registration and launch receipts - if: always() && needs.authorize.outputs.operation != 'evaluate' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-bootstrap/ - ${{ runner.temp }}/chat-completion-testops-launch-plan/ - retention-days: 30 - if-no-files-found: error - - name: Retain the exact authenticated selection - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-selection-${{ github.run_id }}-1 - path: testops-pilot-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable catalog source without service or model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - name: Resolve the fixed candidate graph with lifecycle scripts disabled - run: | - pnpm install --lockfile-only --ignore-scripts - pnpm install --frozen-lockfile --ignore-scripts - - name: Build existing TypeScript and native Runner outputs - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - - name: Exercise the real workflow and pinned source host contract before model access - run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs - - name: Build native Runner outputs - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package the resolved lock and exact source outputs - env: - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$TARGET_SHA" - tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-build-${{ github.run_id }}-1 - path: | - chat-completion-build.tar.gz - chat-completion-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: ${{ matrix.executionId }} - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - strategy: - fail-fast: false - max-parallel: 1 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - actions: read - id-token: write - env: &retention - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }} - MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }} - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - steps: - - *checkout - - *node - - *guard - - *pnpm - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-completion-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - name: Verify source-bound build and authenticated selection before credentials - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" - node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json - (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) - tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") - - name: Qualify the pinned provider runtime and preinstalled browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Bind exact private evidence under the authenticated TestOps authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only this cell's exact-key storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify private custody before delivering the model key - env: &cell_session - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the selected original case with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain original evidence after success or failure - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and build the exact safe TestOps projection - needs: [authorize, select, build, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure') - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies with lifecycle scripts disabled - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" - - name: Prepare unchanged exact-key report and cell policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact first-cell reader - id: reader_0 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the first exact original - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact second-cell reader - id: reader_1 - if: fromJSON(steps.session.outputs.cell_count) > 1 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_1 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the second exact original - if: steps.reader_1.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '1' - AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact third-cell reader - id: reader_2 - if: fromJSON(steps.session.outputs.cell_count) > 2 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_2 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the third exact original - if: steps.reader_2.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '2' - AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Verify every selected original and its completeness receipt - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing report while preserving failed source grades - id: normalize - continue-on-error: true - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only exact reviewed synthetic fixture results - id: export - if: always() && steps.private_report.outcome == 'success' - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts - - name: Require a complete safe projection even after a failed normalization - if: always() - env: - EXPORT_OUTCOME: ${{ steps.export.outcome }} - run: test "$EXPORT_OUTCOME" = success - - name: Retain the metadata-only TestOps projection - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Record and independently verify each TestOps result - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - *allure - - name: Upload original outcomes, checks and private-evidence receipts - id: allure_upload - env: &upload_env - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results - run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops - - name: Verify actual selected results and complete result-owned attachments - id: verification - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *upload_env - run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops - - name: Retain closed TestOps verification receipts - if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-verify-upload/ - ${{ runner.temp }}/chat-completion-testops-upload/exported.json - retention-days: 30 - if-no-files-found: error diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml deleted file mode 100644 index 826a340ae91..00000000000 --- a/.github/workflows/chat-feature-testops.yml +++ /dev/null @@ -1,451 +0,0 @@ -name: Chat feature TestOps -run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Fixed evaluation, zero-result registration, or one TestOps plan launch - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps supplies its job-run identity; no test/source selector is accepted - ALLURE_USERNAME: - type: string - required: false - description: Reserved TestOps service metadata; never used as authorization - -permissions: - contents: read - -concurrency: - group: chat-feature-testops - cancel-in-progress: false - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - sha: ${{ steps.guard.outputs.sha }} - operation: ${{ steps.guard.outputs.operation }} - triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify fixed default-source dispatch and both actor identities - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - NODE_OPTIONS: '' - NODE_PATH: '' - run: node tests/runner-e2e/testops-chat-dispatch.ts - - select: - name: Validate TestOps selection or register zero-result setup - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - sha256: ${{ steps.selection.outputs.sha256 }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register the public job without an evaluation or a synthetic result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap - node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - - name: Validate the one real TestOps selection before any build or provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection - - name: Start exactly the validated TestOps plan and exit without waiting for its child - if: needs.authorize.outputs.operation == 'launch-plan' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} - CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - GH_TOKEN: ${{ github.token }} - run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan - - name: Retain only closed metadata from zero-result setup - if: always() && needs.authorize.outputs.operation == 'bootstrap' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json - ${{ runner.temp }}/chat-testops-bootstrap-state/once.json - ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json - ${{ runner.temp }}/chat-testops-bootstrap-state/job.json - ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain the one plan-launch receipt and stop state - if: always() && needs.authorize.outputs.operation == 'launch-plan' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-plan-launch-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-launch-plan-state/once.json - ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json - ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain only the token-free exact selection receipt - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-selection-${{ github.run_id }}-1 - path: testops-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable shipped native candidate without model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - name: Install only the source-reviewed graph without lifecycle scripts - run: | - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - pnpm install --frozen-lockfile --ignore-scripts - - name: Build the existing TypeScript and native runner owners - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package source-bound outputs and resolved lock - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 - path: | - chat-testops-build.tar.gz - chat-testops-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: One real native OpenCode backlog request on AWS - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - id-token: write - env: &retention - PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 - TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TARGET_SHA: ${{ needs.authorize.outputs.sha }} - MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Check out the same admitted source for trusted retention - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-testops-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - name: Verify immutable outputs and token-free TestOps selection - env: - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) - tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" - node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json - cmp chat-testops-lock.json chat-testops-worker-lock.json - cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") - - name: Materialize and qualify pinned OpenCode runtime and browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Prepare private evidence with shipped-main authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only the exact private cell storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify strict private custody before the model key - env: &cell_session - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the fixed original request with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain exact original grade and private evidence, including failures - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and export only reviewed fixture results - needs: [authorize, select, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies only - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" - - name: Prepare exact private report and cell-read policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact private cell reader - id: read_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the exact private original and verify all inventories - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: | - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing normalized campaign without exposing original logs - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately even when its grade failed - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only the bound normalized result, JUnit and marked PNGs - id: export - if: always() && steps.private_report.outcome == 'success' - env: - TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts - - name: Upload only the sanitized TestOps projection - if: always() && steps.export.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Report the original outcome to its exact TestOps case - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - *allure - - name: Verify and upload only the bound safe case outcome - id: allure_upload - env: &testops_reporter - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results - run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops - - name: Verify the actual TestOps result after upload processing - id: verify_upload - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *testops_reporter - run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops - - name: Retain only closed actual TestOps verification metadata - if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json - ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json - retention-days: 30 - if-no-files-found: error diff --git a/doc/evals.md b/doc/evals.md index 4071cf01866..d0ea2fa5ec3 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -57,8 +57,9 @@ OpenCode Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, using ordinary production instructions, real browser messages and independent state/provenance checks. A separately selected native OpenCode/DeepSeek1017 -backlog cell measures TestOps integration acceptance with the same oracle and -original bounds. Its source registration is not an AWS or TestOps pass. The +backlog cell supplied historical TestOps integration acceptance with the same +oracle and original bounds. Its former smoke plan and the three-case completion +pilot are retired; their cases and measurements remain in history. The [full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops) maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and fixtures use a preserved immutable catalog revision; trusted workflow helpers run diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 97290f343e1..12dbe906011 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -579,8 +579,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), - readWorkflow("chat-feature-testops.yml"), - readWorkflow("chat-completion-testops.yml"), readWorkflow("chat-catalog-testops.yml"), ]; @@ -621,8 +619,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", "chat-catalog-testops.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); @@ -705,38 +701,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { } continue; } - if (name === "chat-feature-testops.yml") { - assert.equal(providerJobs.length, 1); - const assertBoundary = (block) => { - assert.match(block, /^ chat:\n/); - assert.match(block, /\n needs: \[authorize, select, build\]\n/); - assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); - assert.match(block, /\n environment: runner-e2e-private-paid\n/); - assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); - assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/); - assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); - assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/); - assert.doesNotMatch(block, /ALLURE_TOKEN/); - assert.equal([...block.matchAll(providerSecretReference)].length, 1); - assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); - assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference)); - }; - const block = providerJobs[0]; - assertBoundary(block); - assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); - assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/); - assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/); - assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/); - for (const mutant of [ - block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), - block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), - block.replace(" - *guard\n", ""), - block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"), - block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"), - block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"), - ]) assert.throws(() => assertBoundary(mutant)); - continue; - } + for (const block of providerJobs) { if (name === "runner-protocol-live-evals.yml") { assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}")); diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index a66cf38a374..01d7c354b23 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -181,82 +181,31 @@ backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. -## TestOps completion pilot +## Retired TestOps plans -The separate `Chat completion TestOps` workflow connects only these preserved -catalog cases to the private Paperclip Evals project: +The one-case **Chat feature evals** plan and three-case +**Chat completion pilot — OpenCode 1017** plan are retired. Their scenarios are +included in the [full catalog](#full-catalog-in-testops). Use that plan or select +cases from it; do not recreate the smoke or pilot plans. -- `projects.project-work` -- `projects.project-defaults` -- `instance-operations.restart-service` +The former `chat-feature-testops.yml` and `chat-completion-testops.yml` workflows +are removed. Their bootstrap functions reject registration before any API call +or mutation marker. Shared transport, evidence, export and verification helpers +remain for the full-catalog workflow and retained historical evidence. Retiring a +plan does not delete its cases, results, launches or private evidence. -It evaluates the immutable catalog source in -`chat-completion-testops-contract.mjs`. This keeps the broader catalog on its -integration branch. The source includes the bounded TestOps host prerequisite -from #18515. Changing the pin requires source review. The original DeepSeek -1017 model, case clocks, independent checks, and zero retries remain fixed. -Source registration does not establish a new live result. - -Run the named plan **Chat completion pilot — OpenCode 1017** in TestOps. The -adapter validates the actual project, plan, job, launch and selected cases before -build or provider access. A selected subset runs only those cases. The AWS matrix -has concurrency one and does not cancel sibling cases after a failure. - -The main-only reporter uses the existing protected `ALLURE_TOKEN`; the evaluated -app and worker never receive it. Original evidence stays in the existing private -AWS storage with the unchanged validators, scanners, byte bounds and exact-key -policies. A separate trusted projection carries verdicts, checks, accounting and -private-evidence receipts. The final PNGs are verified in protected storage; -only their checksums and availability are recorded in TestOps. GitHub artifacts -contain metadata only. Raw logs, screenshots, hidden -reasoning, API snapshots, credentials and provider sessions are not attachments. -Original failed grades and cleanup failures remain failed. Same-run readback -requires matching results and complete result-owned attachment metadata within -the existing bounded processing window; it does not claim remote byte hashes. - -Initial setup uses the existing workflow operations: - -1. Dispatch reviewed `master` once with `operation=bootstrap`. This creates or - reconciles only the three exact automated cases and the dedicated plan. It - registers the GitHub job with an empty Allure results directory. It reuses - only the existing enabled GitHub integration from - `CHAT_TESTOPS_INTEGRATION_ID`. No model or fabricated result is created. - The observation receipt captures a closed parameter-shape inspection before - binding. Only the observed single `Branch/defaultValue` definition for - `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases - while preserving that definition; the plan uses its default branch. - Assignment inspection records unassigned and existing-job counts before - binding. A single unassigned bucket must cover the exact selected cases; - foreign jobs and unknown or incomplete buckets stop before writes. -2. Read the closed registration receipt. Store its observed `binding` JSON as - `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. -3. Use TestOps **Run**. For the initial operator launch, set an exact next - workflow run number in `CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER`, set a new - 32-hex `CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE`, then dispatch `launch-plan` - once. Its request produces the TestOps-owned `evaluate` dispatch. Do not - manually dispatch an additional evaluation after an uncertain launch. -4. Verify the fresh TestOps launch, all selected case results, source links, - independent checks, private-evidence receipts and cleanup. Inspect screenshots - through authenticated private-history retrieval. Retain earlier failed attempts. - -Launch preflight searches use TestOps' typed name filter, encoded as Base64 of -UTF-8 JSON in the `search` query parameter. Plain name text returns HTTP 400. -The backlog and completion launchers share this encoding; the full-catalog -launcher uses the completion adapter. Existing nonce and once-only guards still -apply before a plan-run request. - -The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username -metadata. It rejects arbitrary source/model/budget selectors, branch workflows, -replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and -actor identities. The existing one-case backlog plan remains independent. +Retained launch helpers encode TestOps name filters as Base64 of UTF-8 JSON in +`search`. Plain name text returns HTTP 400. Their nonce and once-only guards +remain unchanged; this transport contract does not restore the retired plans. ## Full catalog in TestOps The [Chat catalog TestOps workflow](../../.github/workflows/chat-catalog-testops.yml) registers all 244 scenarios from the 35 YAML areas in a separate **Chat feature catalog — OpenCode 1017 (candidate)** plan. Its source-derived -manifest is `chat-full-catalog-testops-manifest.json`. The existing backlog plan -and three-case completion pilot keep their own jobs, selections and history. +manifest is `chat-full-catalog-testops-manifest.json`. This is the active chat +plan. The former backlog and three-case pilot plans are retired; their cases +and measured history are preserved. Legacy aliases reuse existing TestOps case identities without renaming them. The catalog contains 236 Agent Chat routes, five browser boundaries, one external @@ -322,77 +271,3 @@ and credentials do not enter the public artifact path. Additional credentials reach only cases that declare them. External inputs use the existing selected-case private-input owner, disposable-resource checks and oracle commitments. Unconfigured inputs remain setup blockers. - -## TestOps-backed AWS backlog cell - -The dedicated `Chat feature TestOps` workflow accepts only the shipped default -branch and the fixed OpenCode backlog cell. `evaluate` is the default operation -and requires TestOps' `ALLURE_JOB_RUN_ID`. A protected reporting job verifies the -project, singleton plan, assigned public workflow, selected case and current -GitHub run before any build or provider access. The evaluated process receives -no TestOps token. It retains the original one-query, one-work-run, 120-second -turn, 420-second attempt, 300-second settled-state and zero-retry limits. - -Two metadata operations prepare that route. `bootstrap` registers the actual -public CI job through pinned `allurectl` with an empty results directory and -reconciles only the dedicated case and singleton plan. It produces no passing -test result. After the registered job and protected case/plan/job identifiers -are verified, `launch-plan` sends one TestOps plan-run request and exits. -The TestOps job then dispatches `evaluate`; an operator must not dispatch a -second evaluation manually. Missing or ambiguous registration and selection -metadata stops before inference. - -The workflow reuses the existing AWS fleet and strict private evidence storage. -Its shipped-main authority is separate from manual candidate approval. Original -logs, provider records and failure evidence remain private. The reporting adapter -exports only a source/run-bound normalized result, JUnit and screenshots already -marked by the owned fixture screenshot producer with matching file hashes. -A separate protected reporting job converts that projection to the exact TestOps -case. Failed grades and cleanup failures remain failed. - -Reporting setup uses the master-only `runner-e2e-testops-reporting` environment, -with the existing reporting token delivered through the reviewed sealed-box -bootstrap. `CHAT_TESTOPS_CASE_ID`, `CHAT_TESTOPS_PLAN_ID` and -`CHAT_TESTOPS_JOB_ID` must name the observed dedicated mapping; the numeric -`CHAT_TESTOPS_GITHUB_WORKFLOW_ID` supports TestOps' registered workflow URL. -No arbitrary source, model, case, retry or budget input is accepted. - -The initial `launch-plan` administrator operation also requires a protected -`CHAT_TESTOPS_LAUNCH_RUN_NUMBER`. After registration, the trusted operator creates -this variable once with the reviewed next workflow run number and reads it back. -The launcher compares GitHub's actual run number, attempt, workflow, source and -actors before any TestOps mutation. A later dispatch or replay cannot reuse that -approved number. Missing or uncertain metadata stops; an uncertain request needs -reconciliation rather than a new send. The nonce and launch search provide -additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of -this one-time administrator fence. The launch nonce is protected configuration, -not a dispatch input. Job parameter IDs are opaque. The launcher preserves the -API's exact ID/value pair for `refs/heads/master` and uses the fixed workflow's -`evaluate` default. An operation parameter is used only when the protected -`CHAT_TESTOPS_OPERATION_PARAMETER_ID` matches a returned ID with a `bootstrap` -or `evaluate` value; the launcher then sets it to `evaluate`. Other values, -duplicate IDs and conflicting configuration are rejected. No relationship to -global environment-variable IDs or names is inferred. - -`CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub -integration. The launcher verifies its project, type and enabled state, then binds -only the newly registered public job to it. It never changes integration settings -or credentials. Since the job response does not expose that association, only the -real TestOps-to-GitHub dispatch can qualify it. - -After `allurectl watch` finishes, the same protected reporter checks the actual -TestOps job-run, launch, selected case, result source/status and attachment -metadata. Processing reads are bounded to 12 polls, five-second gaps and a -120-second processing deadline. Unknown or missing results remain unverified. -The check downloads no attachment content and claims no server-side attachment -byte hash. Its closed receipt is retained separately. An upload success alone -is not a passing feature result, and verifying an original failure cannot turn -that failure into a pass. - -Build and worker installs use the existing reviewed report-lock policy. Only its -committed input or output lock is accepted; its authenticated correction is the -only allowed normalization. Dependency manifests, patch inputs and policy must -remain unchanged. Installs are frozen with lifecycle scripts disabled. The -credential-free build rechecks that boundary before packaging, and the worker -matches the source/lock receipt before installing. Arbitrary resolver drift is -rejected before provider access. diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index 5b9f883c935..7d02341af3e 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -135,32 +135,12 @@ export async function inspectPilotCatalog(api) { } export async function bootstrapPilotPlan(api, integrationId, markOnce, context = {}) { + // The pilot plan is retired. Reject before API reads or mutation markers. + if (context.catalog !== 'full') reject('plan_retired'); if (!positive(integrationId)) reject('integration'); const integration = await api(`/api/integration/${integrationId}/project/1`); if (integration.id !== integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) reject('integration'); - if (context.catalog === 'full') return { ...await reconcileFullCatalog(api, markOnce), integrationId }; - const inspected = await inspectPilotCatalog(api); - if (inspected.plans.length && inspected.cases.length !== cases.length) reject('plan_conflict'); - const selected = [...inspected.cases]; - for (const row of cases) { - if (selected.some(value => value.key === row.key)) continue; - await markOnce(`create_case_${row.key}`); - const created = await api('/api/testcase', 'POST', { projectId: 1, name: row.name, fullName: row.fullName, automated: true }); - if (!positive(created?.id)) reject('created_case'); - assertCase(created, { ...row, caseId: created.id }); - selected.push({ ...row, caseId: created.id }); - } - const ordered = cases.map(row => selected.find(value => value.key === row.key)); - let planId = inspected.plans[0]?.id; - if (!planId) { - await markOnce('create_plan'); - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: ordered.map(row => row.caseId) } }); - if (!positive(created?.id)) reject('created_plan'); planId = created.id; - } - const binding = { planId, integrationId, cases: ordered }; - assertPlan(await api(`/api/testplan/${planId}`), binding); - return binding; + return { ...await reconcileFullCatalog(api, markOnce), integrationId }; } export async function completePilotRegistration(api, prepared, official, context, workflowId, markOnce, observe = async () => {}) { diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index 4ee58ebe003..aee4be2527c 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -180,35 +180,9 @@ test('an uncertain launch response is never automatically resent', async () => { assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); }); -test('registration creates only the three missing cases and the dedicated plan after once markers', async () => { - const calls = []; let nextCase = 100; - const get = async (route, method = 'GET', body) => { - calls.push({ route, method, body }); - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - if (route === '/api/testcase' && method === 'POST') return { ...body, id: nextCase++ }; - if (route === '/api/testplan' && method === 'POST') return { ...plan, name: planName }; - if (route === '/api/testplan/10') return plan; - assert.fail(route); - }; - const value = await bootstrapPilotPlan(get, 2, async stage => calls.push({ marker: stage })); - assert.equal(value.cases.length, 3); assert.equal(value.planId, 10); - assert.equal(calls.filter(row => row.method === 'POST').length, 4); - for (const [i, row] of calls.entries()) if (row.method === 'POST') assert.ok(calls[i - 1].marker); -}); - -test('uncertain case creation is never automatically resent', async () => { - let writes = 0, markers = 0; - const get = async (route, method = 'GET') => { - if (method === 'POST') { writes++; throw new Error('uncertain transport'); } - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - assert.fail(route); - }; - await assert.rejects(bootstrapPilotPlan(get, 2, async () => { markers++; }), /uncertain transport/); - assert.equal(writes, 1); assert.equal(markers, 1); +test('retired pilot bootstrap cannot recreate a plan or call the integration API', async () => { + await assert.rejects(bootstrapPilotPlan(() => assert.fail('retired route must not call the API'), 2, + () => assert.fail('retired route must not mark a mutation')), /chat_completion_testops_plan_retired/); }); function api(replace = {}) { diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs index c77d89031a8..c1199bbeb40 100644 --- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -96,37 +96,8 @@ test('the executable guard binds the actual original GitHub run before service o } finally { await rm(root, { recursive: true, force: true }); } }); -test('the real workflow separates metadata authority, provider execution and result reporting', async () => { - const text = await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8'); - const w = parse(text); - assert.deepEqual(Object.keys(w.on.workflow_dispatch.inputs).sort(), ['ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'operation']); - assert.equal(w.concurrency['cancel-in-progress'], false); - assert.equal(w.jobs.chat.strategy['max-parallel'], 1); assert.equal(w.jobs.chat.strategy['fail-fast'], false); - assert.equal(w.jobs.select.environment, 'runner-e2e-testops-reporting'); - assert.equal(w.jobs.chat.environment, 'runner-e2e-private-paid'); - assert.equal(w.jobs.report.environment, 'runner-e2e-private-history'); - assert.equal(w.jobs.upload.environment, 'runner-e2e-testops-reporting'); - for (const [name, job] of Object.entries(w.jobs)) { - assert.ok(job.steps.slice(0, 3).some(step => step.run?.endsWith('chat-completion-testops-cli.mjs guard')), name); - if (name !== 'chat') assert.ok(!JSON.stringify(job).includes('secrets.OPENROUTER_API_KEY'), name); - if (!['select', 'upload'].includes(name)) assert.ok(!JSON.stringify(job).includes('secrets.ALLURE_TOKEN'), name); - } - assert.ok(w.jobs.upload.if.includes("needs.report.result == 'success'")); - assert.ok(w.jobs.report.needs.includes('build')); - assert.ok(w.jobs.report.if.includes("needs.build.result == 'success'")); - assert.ok(w.jobs.report.if.includes("needs.chat.result == 'failure'")); - assert.ok(!text.includes('<<:')); assert.ok(!text.includes('target_branch:')); - assert.equal(w.jobs.build.steps.find(step => step.name?.startsWith('Resolve the fixed')).run.includes('--ignore-scripts'), true); - const paid = w.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); - assert.equal(paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, 'v1'); - assert.equal(paid.env.ALLURE_JOB_RUN_ID, '${{ inputs.ALLURE_JOB_RUN_ID }}'); - assert.ok(paid.run.includes('--max-automatic-retries 0')); - assert.ok(w.jobs.build.steps.some(step => step.run?.includes('chat-completion-testops-host-control.mjs'))); - assert.ok(w.jobs.chat.steps.indexOf(paid) > w.jobs.chat.steps.findIndex(step => step.name?.startsWith('Verify source-bound'))); -}); - test('target checkout cannot populate or overwrite the trusted helper path', async () => { - const definition = parse(await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8')); + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); try { const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 2b4a4f1d049..47fc4939da8 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -53,7 +53,7 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'testops_case_alias_conflict', 'testops_plan_name_conflict', 'testops_plan_identity_invalid', 'testops_integration_identity_invalid', 'testops_bootstrap_identity_invalid', 'testops_plan_without_matching_case', 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', - 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', + 'testops_plan_retired', 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); @@ -139,32 +139,10 @@ export async function inspectChatMetadata(api) { return { projectId: 1, caseConflict: cases.size > 1, cases: [...cases.values()], plans: plans.map(value => ({ id: value.id, name: planName, count: value.testCasesCount })), integrations }; } -export async function ensureChatPlan(api, inspected, markOnce) { - if (inspected.projectId !== 1 || inspected.plans.length > 1) fail('testops_bootstrap_identity_invalid'); - if (inspected.caseConflict === true || inspected.cases.length > 1) fail('testops_case_alias_conflict'); - let selected = inspected.cases[0], plan = inspected.plans[0]; - if (plan && !selected) fail('testops_plan_without_matching_case'); - if (!selected || !plan) await markOnce(); - if (!selected) { - const created = await api('/api/testcase', 'POST', { projectId: 1, name: 'Create a task backlog through chat', fullName, - automated: true, description: 'Canonical scenario: tasks.create-backlog. Readiness and recorded outcomes remain separate.', - expectedResult: 'The real agent creates the requested backlog; public Product E2E assertions and cleanup must pass.' }); - if (!id(created?.id) || created.projectId !== 1 || created.fullName !== fullName) fail('testops_created_case_identity_invalid'); - selected = { id: created.id, fullName, automated: created.automated === true }; - } - if (!plan) { - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: [selected.id] } }); - if (!id(created?.id) || created.projectId !== 1 || created.name !== planName) fail('testops_created_plan_identity_invalid'); - plan = { id: created.id, name: planName }; - } - const actual = await api(`/api/testplan/${plan.id}`); - if (actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== 1 - || actual.treeSelection?.inverted === true || !same(actual.treeSelection?.leafsInclude, [selected.id]) - || (actual.treeSelection?.groupsInclude?.length ?? 0) > 0 || (actual.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('testops_plan_scope_mismatch'); - return { projectId: 1, case: selected, plan: { id: plan.id, name: planName, count: 1 }, - caseUrl: `${endpoint}/project/1/test-cases/${selected.id}`, planUrl: `${endpoint}/testplan/${plan.id}`, - actualResult: null, note: 'Plan preparation is not an executed measurement or a passing result.' }; +// Kept as an explicit retirement boundary for callers of the former smoke lane. +// Catalog registration owns the shared cases; deleting the old plan must persist. +export async function ensureChatPlan() { + fail('testops_plan_retired'); } diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 318c616208c..68f689beee6 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -31,64 +31,14 @@ test('multiple alias identities, foreign project and incomplete metadata stop be await assert.rejects(inspectChatMetadata(async () => ({ content: [], totalPages: 11, totalElements: 1100, number: 0 })), /catalog_incomplete/); }); -test('existing exact plan reuses case and plan without a write', async () => { - const result = await ensureChatPlan(async route => { assert.equal(route, '/api/testplan/8'); return plan; }, - await inspectChatMetadata(inspectApi()), () => assert.fail('must not mark or write')); - assert.equal(result.case.id, 91); - assert.equal(result.plan.id, 8); - assert.equal(result.actualResult, null); -}); - -test('bootstrap creates only the missing exact plan after exclusive marker and reads back scope', async () => { - const calls = []; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - if (method === 'POST') { assert.equal(calls[0].route, 'MARK'); assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [91] } }); } - return plan; - }, { projectId: 1, cases: [caseRow], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(calls.filter(row => row.method === 'POST').length, 1); - assert.equal(result.plan.count, 1); -}); - -test('empty project bootstrap links the genuinely created case into only the new plan', async () => { - const calls = []; - const createdCase = { ...caseRow, id: 92 }; - const createdPlan = { ...plan, treeSelection: { inverted: false, leafsInclude: [92] } }; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - assert.equal(calls[0].route, 'MARK'); - if (route === '/api/testcase') { - assert.equal(body.projectId, 1); assert.equal(body.fullName, fullName); assert.equal(body.automated, true); - return createdCase; - } - if (route === '/api/testplan') { - assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [92] } }); - return createdPlan; - } - assert.equal(route, '/api/testplan/8'); assert.equal(method, undefined); return createdPlan; - }, { projectId: 1, cases: [], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(result.case.id, 92); assert.equal(result.plan.id, 8); - assert.deepEqual(calls.map(call => call.route), ['MARK', '/api/testcase', '/api/testplan', '/api/testplan/8']); -}); - -test('known conflicting plan stops before creating an otherwise missing case or marker', async () => { - await assert.rejects(ensureChatPlan(() => assert.fail('no API write or read needed'), - { projectId: 1, cases: [], plans: [plan] }, () => assert.fail('must not mark')), /plan_without_matching_case/); -}); - -test('new case is created only after alias reconciliation; an uncertain mutation is never resent', async () => { - let writes = 0, markers = 0; - await assert.rejects(ensureChatPlan(async (_route, method) => { - assert.equal(method, 'POST'); writes++; throw new Error('uncertain transport'); - }, { projectId: 1, cases: [], plans: [] }, async () => { markers++; }), /uncertain/); - assert.equal(writes, 1); assert.equal(markers, 1); -}); - -test('existing broad or conflicting plan never expands or changes its scope', async () => { - for (const actual of [{ ...plan, testCasesCount: 2 }, { ...plan, treeSelection: { leafsInclude: [92] } }, - { ...plan, treeSelection: { leafsInclude: [91], inverted: true } }, { ...plan, projectId: 2 }]) { - await assert.rejects(ensureChatPlan(async (_route, method) => { assert.equal(method, undefined); return actual; }, - { projectId: 1, cases: [caseRow], plans: [plan] }, () => assert.fail('no writes')), /scope_mismatch/); +test('retired singleton bootstrap cannot recreate an absent plan or mutate retained metadata', async () => { + for (const inspected of [ + { projectId: 1, cases: [], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [plan] }, + ]) { + await assert.rejects(ensureChatPlan(() => assert.fail('retired route must not call the API'), inspected, + () => assert.fail('retired route must not mark a mutation')), /testops_plan_retired/); } }); diff --git a/tests/runner-e2e/chat-feature-testops-cli.test.mjs b/tests/runner-e2e/chat-feature-testops-cli.test.mjs index 107f046b109..70238eb15f0 100644 --- a/tests/runner-e2e/chat-feature-testops-cli.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-cli.test.mjs @@ -9,7 +9,7 @@ const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/ GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; -test('actual metadata bootstrap caller creates no result or agent and reuses exact case/plan', async t => { +test('actual retired singleton caller refuses plan recreation and creates no result', async t => { const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-bootstrap-'))); t.after(() => rm(temporary, { recursive: true })); const state = path.join(temporary, 'state'), requests = []; @@ -30,9 +30,10 @@ test('actual metadata bootstrap caller creates no result or agent and reuses exa return response({ id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { leafsInclude: [91], inverted: false } }); }; assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, - CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 0); - const plan = JSON.parse(await readFile(path.join(state, 'plan.json'))); - assert.equal(plan.actualResult, null); + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 1); + const failure = JSON.parse(await readFile(path.join(state, 'failure.json'))); + assert.equal(failure.code, 'testops_plan_retired'); + await assert.rejects(readFile(path.join(state, 'plan.json')), { code: 'ENOENT' }); assert.equal(requests.filter(value => value.method === 'POST').length, 1, 'only auth token exchange; no case/plan write'); await assert.rejects(readFile(path.join(state, 'allure-results.json')), { code: 'ENOENT' }); }); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 917d6912f90..65c32e7a7e9 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -244,18 +244,6 @@ test('post-watch foreign backend binding returns safe nonzero and preserves orig assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); }); -test('bootstrap workflow captures official context only after watch and retains exactly named closed receipts', async () => { - const workflow = await readFile(new URL('../../.github/workflows/chat-feature-testops.yml', import.meta.url), 'utf8'); - const start = workflow.indexOf(' - name: Register the public job without an evaluation or a synthetic result'); - const end = workflow.indexOf(' - name: Validate the one real TestOps selection', start); - const step = workflow.slice(start, end); - assert.ok(step.includes("if: needs.authorize.outputs.operation == 'bootstrap'")); - assert.match(step, /allurectl watch -- node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap\n\s+node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap-context/); - assert.equal((step.match(/allurectl watch/g) ?? []).length, 1); assert.ok(!step.includes('continue-on-error')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration.json')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json')); -}); - test('post-watch refuses nonprivate state custody before any command auth or receipt write', async t => { const f = await bootstrapContextFixture(t); await chmod(f.state, 0o755); const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); diff --git a/tests/runner-e2e/testops-chat-workflow.test.ts b/tests/runner-e2e/testops-chat-workflow.test.ts deleted file mode 100644 index 14484cbd30e..00000000000 --- a/tests/runner-e2e/testops-chat-workflow.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { readFileSync } from "node:fs"; -import { parse } from "yaml"; -import { expect, it } from "vitest"; -import { validateChatTestOpsDispatch } from "./testops-chat-dispatch.js"; - -const workflow = parse(readFileSync(new URL("../../.github/workflows/chat-feature-testops.yml", import.meta.url), "utf8")); -function context() { - return { env: { GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", - GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", GITHUB_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", GITHUB_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]" }, - event: { repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, ref: "refs/heads/master", sender: { id: 123 }, inputs: { ALLURE_JOB_RUN_ID: "456" } as Record } }; -} -it("admits the actual TestOps service input and only the closed metadata bootstrap alternative", () => { - const f = context(); expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - f.event.inputs = { operation: "bootstrap" }; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("bootstrap"); - f.event.inputs.operation = "launch-plan"; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("launch-plan"); - f.event.inputs.ALLURE_JOB_RUN_ID = "456"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); - delete f.event.inputs.ALLURE_JOB_RUN_ID; - f.event.inputs.operation = "other"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); -}); -it.each(["unlisted-trigger", "wrong-source", "replay", "extra-input", "missing-jobrun", "foreign-repo"])("rejects %s before any service credential operation", kind => { - const f = context(); let trigger = 123; - if (kind === "unlisted-trigger") trigger = 456; - if (kind === "wrong-source") f.env.GITHUB_REF = "refs/heads/unreviewed"; - if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; - if (kind === "extra-input") f.event.inputs.target_branch = "other"; - if (kind === "missing-jobrun") f.event.inputs = {}; - if (kind === "foreign-repo") f.event.repository.full_name = "foreign/repo"; - expect(() => validateChatTestOpsDispatch(f.env, f.event, trigger)).toThrow(); -}); -it("keeps provider and TestOps service credentials in separate jobs", () => { - const jobs = workflow.jobs; - for (const [id, job] of Object.entries(jobs) as Array<[string, any]>) { - const text = JSON.stringify(job); - if (text.includes("secrets.OPENROUTER_API_KEY")) expect(id).toBe("chat"); - if (text.includes("secrets.ALLURE_TOKEN")) expect(["select", "upload"]).toContain(id); - if (id !== "chat") expect(text).not.toContain("pnpm test:e2e:runner --"); - expect(job.steps.some((step: any) => step.run === "node tests/runner-e2e/testops-chat-dispatch.ts")).toBe(true); - } - expect(jobs.chat.environment).toBe("runner-e2e-private-paid"); - expect(jobs.select.environment).toBe("runner-e2e-testops-reporting"); - expect(jobs.upload.environment).toBe("runner-e2e-testops-reporting"); -}); -it("requires token-isolated selection before the fixed AWS run and keeps bootstrap outside all model jobs", () => { - expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); - expect(Object.keys(workflow.on.workflow_dispatch.inputs)).toEqual(["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"]); - expect(workflow.on.workflow_dispatch.inputs.operation.default).toBe("evaluate"); - expect(workflow.jobs.chat.needs).toContain("select"); expect(workflow.jobs.build.needs).toContain("select"); - expect(workflow.jobs.chat.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.jobs.build.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - const paid = workflow.jobs.chat.steps.find((s: any) => s.env?.OPENROUTER_API_KEY); - expect(paid.run).toContain("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0"); - expect(workflow.jobs.chat.steps.findIndex((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection")).toBeLessThan(workflow.jobs.chat.steps.indexOf(paid)); - expect(workflow.jobs.authorize.outputs.runner).toBe("runs-on/fleet=paperclip-public-pr-x64/env=public-ci"); - expect(workflow.jobs.chat["runs-on"]).toBe("${{ needs.authorize.outputs.runner }}"); -}); -it("uploads only the trusted safe projection after successful strict private report retention", () => { - const steps = workflow.jobs.report.steps; - expect(steps.find((s: any) => s.id === "export").if).toBe("always() && steps.private_report.outcome == 'success'"); - expect(steps.find((s: any) => s.with?.name?.startsWith("runner-e2e-testops-")).with.path).toBe("runner-e2e-testops/"); - expect(JSON.stringify(workflow)).not.toContain("MANUAL_PRIVATE_APPROVALS_JSON"); - expect(Object.values(workflow.jobs).flatMap((j: any) => j.steps).filter((s: any) => s.uses?.startsWith("actions/upload-artifact")).some((s: any) => s.with.path === "tests/runner-e2e/results/")).toBe(false); -}); - -it("initializes the actual TestOps association and never fabricates setup results", () => { - const steps = workflow.jobs.select.steps; - const installer = steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")); - expect(installer.uses).toBe("allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db"); - expect(installer.with["allurectl-version"]).toBe("2.23.1"); - const bootstrap = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs bootstrap")); - expect(bootstrap.run).toContain("mkdir -m 700"); - expect(bootstrap.run).toContain("allurectl watch -- node"); - expect(bootstrap.env.ALLURE_RESULTS).toBe("${{ runner.temp }}/chat-testops-bootstrap-results"); - const select = steps.find((s: any) => s.id === "selection"); - expect(select.run).toContain("allurectl watch -- node"); - for (const name of ["CHAT_TESTOPS_CASE_ID", "CHAT_TESTOPS_PLAN_ID", "CHAT_TESTOPS_JOB_ID"]) - expect(select.env[name]).toBe(`\${{ vars.${name} }}`); - const metadata = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-setup-")); - expect(metadata.if).toBe("always() && needs.authorize.outputs.operation == 'bootstrap'"); - expect(metadata.with.path).not.toContain("results"); -}); -it("authenticates the same-run selection again before converting and uploading", () => { - const steps = workflow.jobs.upload.steps; - const receipt = steps.find((s: any) => s.with?.path === "chat-testops-selection"); - expect(receipt.with["artifact-ids"]).toBe("${{ needs.select.outputs.artifact_id }}"); - const upload = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs upload")); - expect(upload.run).toContain("allurectl watch -- node"); - expect(upload.env.EXPECTED_SELECTION_SHA256).toBe("${{ needs.select.outputs.sha256 }}"); - expect(steps.indexOf(receipt)).toBeLessThan(steps.indexOf(upload)); - expect(JSON.stringify(workflow.jobs.select)).not.toContain("OPENROUTER_API_KEY"); - expect(JSON.stringify(workflow.jobs.upload)).not.toContain("OPENROUTER_API_KEY"); -}); - -it("keeps the one plan launch outside Allure watch and every AWS/model job", () => { - const steps = workflow.jobs.select.steps; - const launch = steps.find((s: any) => s.run?.endsWith("testops-chat-reporter.mjs launch-plan")); - expect(launch.if).toBe("needs.authorize.outputs.operation == 'launch-plan'"); - expect(launch.run).not.toContain("watch"); - expect(launch.env.CHAT_TESTOPS_OPERATION_PARAMETER_ID).toBe("${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}"); - expect(launch.env.CHAT_TESTOPS_INTEGRATION_ID).toBe("${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_NONCE).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}"); - expect(launch.env.TESTOPS_TRIGGERING_ACTOR_ID).toBe("${{ needs.authorize.outputs.triggering_actor_id }}"); - expect(launch.env.TESTOPS_ALLOWED_ACTOR_IDS).toBe("${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - expect(launch.env.GH_TOKEN).toBe("${{ github.token }}"); - expect(launch.run).not.toContain("sleep"); - expect(steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")).if).toBe("needs.authorize.outputs.operation != 'launch-plan'"); - for (const name of ["build", "chat"]) expect(workflow.jobs[name].if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.concurrency["cancel-in-progress"]).toBe(false); -}); - -it("accepts the documented TestOps username metadata without treating it as an actor", () => { - const f = context(); f.event.inputs.ALLURE_USERNAME = "service-metadata"; - expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - expect(() => validateChatTestOpsDispatch(f.env, f.event, 456)).toThrow(); - expect(JSON.stringify(workflow.jobs)).not.toContain("inputs.ALLURE_USERNAME"); -}); - -it("includes every native Node helper control in the normal unit owner", () => { - const launcher = readFileSync(new URL("./run-unit.mjs", import.meta.url), "utf8"); - const config = readFileSync(new URL("./vitest.config.ts", import.meta.url), "utf8"); - for (const file of ["chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"]) { - expect(launcher).toContain(`"tests/runner-e2e/${file}"`); - expect(config).toContain(`"${file}"`); - } -}); - -it("verifies the actual server outcome after watch without masking an original failed upload", () => { - const steps = workflow.jobs.upload.steps; - const watch = steps.find((s: any) => s.id === "allure_upload"); - const verify = steps.find((s: any) => s.id === "verify_upload"); - expect(watch.run).toContain("allurectl watch -- node"); - expect(watch["continue-on-error"]).toBeUndefined(); - expect(verify.run).toBe("node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops"); - expect(verify.if).toBe("always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')"); - expect(verify.env).toEqual(watch.env); - expect(steps.indexOf(verify)).toBeGreaterThan(steps.indexOf(watch)); - const proof = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-verification-")); - expect(proof.with.path.split("\n").filter(Boolean)).toHaveLength(2); - expect(proof.with.path).not.toContain("allure-results"); -}); - -it("carries only the existing source-reviewed lock through build and worker install", () => { - const build = workflow.jobs.build.steps; - const install = build.find((s: any) => s.run?.includes("pnpm install --frozen-lockfile")); - expect(install.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(install.run).not.toContain("--lockfile-only"); - const archive = build.find((s: any) => s.run?.includes("tar -czf chat-testops-build")); - expect(archive.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(archive.run).toContain("chat-testops-lock.json pnpm-lock.yaml"); - const worker = workflow.jobs.chat.steps.find((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection"); - expect(worker.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json"); - expect(worker.run).toContain("cmp chat-testops-lock.json chat-testops-worker-lock.json"); - expect(worker.run.indexOf("cmp chat-testops-lock.json")).toBeLessThan(worker.run.indexOf("pnpm install --frozen-lockfile")); -}); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 91c0e3e9f29..ca84b9de5ab 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -606,8 +606,6 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -893,24 +891,7 @@ describe("public repository paid workflow security", () => { for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); - if (["chat-feature-testops.yml", "chat-completion-testops.yml"].includes(filename)) { - const guard = filename === "chat-feature-testops.yml" ? "testops-chat-dispatch.ts" : "chat-completion-testops-cli.mjs guard"; - expect(other).toContain(`run: node tests/runner-e2e/${guard}`); - expect(other).toContain("ref: ${{ github.sha }}"); - expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; - expect(chat).toContain("environment: runner-e2e-private-paid"); - expect(chat).toContain("needs: [authorize, select, build]"); - expect(chat).toContain(" - *guard"); - expect(chat).toContain("verify-selection"); - expect(chat).not.toContain("ALLURE_TOKEN"); - expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); - const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; - expect(report).toContain("environment: runner-e2e-private-history"); - expect(report).toContain("manual-private-transfer.ts report-session"); - expect(report).not.toContain("ALLURE_TOKEN"); - continue; - } + for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");