From 97c4dc969e13364464765ffbde74955005bf44d3 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:07:32 -0600 Subject: [PATCH] test(evals): retire duplicate chat TestOps plans (#18625) ## Thinking Path > - Paperclip manages AI agents and preserves their work history. > - Product E2E cases use TestOps plans to select real AWS evaluations. > - The full chat catalog now includes the former smoke and completion-pilot cases. > - The two old bootstrap workflows can recreate their smaller plans after deletion. > - This pull request removes those workflows and stops their plan-creation functions before API access. > - The full-catalog workflow keeps its shared clients, evidence rules, case identities, and measured results. ## Linked Issues or Issue Description Refs #18542, #18516. The user requested removal of the duplicate smoke and completion-pilot plans and investigation of their recreation paths. Both old workflows accepted manual dispatch only. Their bootstrap operations created plans by name when the plans were absent. They had no schedules in their workflow definitions. The old workflow records were disabled before plan removal. An existing pilot execution was allowed to finish successfully. This change does not cancel executions or delete cases, results, launches, or private evidence. ## What Changed - Remove `chat-feature-testops.yml` and `chat-completion-testops.yml`. - Make the former singleton plan creator reject every call before API reads or mutation markers. - Reject pilot bootstrap before API access. Keep the full-catalog branch and its integration validation. - Remove tests that require the retired workflow files. Keep shared helper and full-catalog controls. Run the trusted-checkout boundary test against the active catalog workflow. - Replace old setup instructions with retirement guidance and the active full-catalog route. The master sync preserves the incoming typed launch-search fix from #18620, including both regression tests. The only conflict was the retired pilot documentation. Its search-format note remains as a retained-helper contract. ## Verification - Two new retirement controls fail against the parent code because it reaches an API or mutation marker. Both pass after the change. - Six owning Node suites after the master sync: 146 passed (144 before the sync). Shared contract: 22 passed. Catalog and retained export controls: 12 passed. Focused workflow and release controls: 7 passed. - Two modified API modules pass Node syntax checks. `git diff --check` passes. - The pure workflow-security file has the same four failures and 13 passes on both the untouched parent and this branch. Two assertions expect eight trusted bootstrap steps where master now has nine. Two lists omit the already-shipped catalog workflow. This change does not repair or relax those unrelated assertions. - A broader release-test invocation had 73 passes and 20 sandbox failures from blocked mock HTTP listeners or npm permissions. Its focused affected workflow control passes. A standard Vitest setup attempt stopped on a missing `acpx/runtime` bundle dependency before tests. No local app, database, provider, or native runtime was qualified. - Current-head PR CI passes: 48 successful jobs and one skipped job, including both required checks. Greptile is 5/5. Independent source review passes. No new model run is needed for this retirement. ## Risks - Existing integrations that point at either retired workflow will stop dispatching. This is the requested behavior; use the full catalog instead. - Historical evidence readers still use shared helpers. Those helpers remain, including the transport and official-context parser used by the active catalog. - Workflow removal is a repository change. TestOps plan deletion and disabling obsolete executors are separate service actions. Neither action should delete shared cases or measurements. ## Model Used OpenAI `gpt-6.1-sol` through Codex, with source editing, shell tools, and independent source review. The context-window size is not asserted. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge Co-authored-by: Paperclip --- .github/workflows/chat-completion-testops.yml | 504 ------------------ .github/workflows/chat-feature-testops.yml | 451 ---------------- doc/evals.md | 5 +- .../release-verify-workflow.test.mjs | 37 +- tests/runner-e2e/CHAT-FEATURES.md | 157 +----- .../chat-completion-testops-api.mjs | 26 +- .../chat-completion-testops-api.test.mjs | 32 +- .../chat-completion-testops-contract.test.mjs | 31 +- tests/runner-e2e/chat-feature-testops-api.mjs | 32 +- .../chat-feature-testops-api.test.mjs | 66 +-- .../chat-feature-testops-cli.test.mjs | 9 +- .../runner-e2e/testops-chat-reporter.test.mjs | 12 - .../runner-e2e/testops-chat-workflow.test.ts | 154 ------ tests/runner-e2e/workflow-security.test.ts | 21 +- 14 files changed, 46 insertions(+), 1491 deletions(-) delete mode 100644 .github/workflows/chat-completion-testops.yml delete mode 100644 .github/workflows/chat-feature-testops.yml delete mode 100644 tests/runner-e2e/testops-chat-workflow.test.ts diff --git a/.github/workflows/chat-completion-testops.yml b/.github/workflows/chat-completion-testops.yml deleted file mode 100644 index 7016f76dfec..00000000000 --- a/.github/workflows/chat-completion-testops.yml +++ /dev/null @@ -1,504 +0,0 @@ -name: Chat completion TestOps -run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Run the exact TestOps selection, register its plan, or launch it once - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps job-run identity; source, model and clocks are fixed - ALLURE_USERNAME: - type: string - required: false - description: Reserved service metadata; never grants authority - -permissions: - contents: read - actions: read - -concurrency: - group: chat-completion-testops - cancel-in-progress: false - -env: - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - operation: ${{ steps.guard.outputs.operation }} - target_sha: ${{ steps.guard.outputs.target_sha }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify actual main-only dispatch and both actors - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard - - select: - name: Register the dedicated plan or validate its real TestOps selection - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-testops-reporting - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} - execution_ids: ${{ steps.selection.outputs.execution_ids }} - matrix: ${{ steps.selection.outputs.matrix }} - policy: ${{ steps.selection.outputs.policy }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register three exact cases with no synthetic test result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap - node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context - - name: Validate case, plan, job, launch and same-run source before provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection - - name: Request the one reviewed plan launch and exit - if: needs.authorize.outputs.operation == 'launch-plan' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }} - CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan - - name: Retain token-free registration and launch receipts - if: always() && needs.authorize.outputs.operation != 'evaluate' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-bootstrap/ - ${{ runner.temp }}/chat-completion-testops-launch-plan/ - retention-days: 30 - if-no-files-found: error - - name: Retain the exact authenticated selection - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-selection-${{ github.run_id }}-1 - path: testops-pilot-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable catalog source without service or model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - name: Resolve the fixed candidate graph with lifecycle scripts disabled - run: | - pnpm install --lockfile-only --ignore-scripts - pnpm install --frozen-lockfile --ignore-scripts - - name: Build existing TypeScript and native Runner outputs - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - - name: Exercise the real workflow and pinned source host contract before model access - run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs - - name: Build native Runner outputs - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package the resolved lock and exact source outputs - env: - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$TARGET_SHA" - tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-build-${{ github.run_id }}-1 - path: | - chat-completion-build.tar.gz - chat-completion-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: ${{ matrix.executionId }} - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - strategy: - fail-fast: false - max-parallel: 1 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - actions: read - id-token: write - env: &retention - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }} - MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }} - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - steps: - - *checkout - - *node - - *guard - - *pnpm - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-completion-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - name: Verify source-bound build and authenticated selection before credentials - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" - node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json - (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) - tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") - - name: Qualify the pinned provider runtime and preinstalled browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Bind exact private evidence under the authenticated TestOps authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only this cell's exact-key storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify private custody before delivering the model key - env: &cell_session - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the selected original case with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain original evidence after success or failure - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and build the exact safe TestOps projection - needs: [authorize, select, build, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure') - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies with lifecycle scripts disabled - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" - - name: Prepare unchanged exact-key report and cell policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact first-cell reader - id: reader_0 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the first exact original - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact second-cell reader - id: reader_1 - if: fromJSON(steps.session.outputs.cell_count) > 1 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_1 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the second exact original - if: steps.reader_1.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '1' - AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact third-cell reader - id: reader_2 - if: fromJSON(steps.session.outputs.cell_count) > 2 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_2 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the third exact original - if: steps.reader_2.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '2' - AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Verify every selected original and its completeness receipt - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing report while preserving failed source grades - id: normalize - continue-on-error: true - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only exact reviewed synthetic fixture results - id: export - if: always() && steps.private_report.outcome == 'success' - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts - - name: Require a complete safe projection even after a failed normalization - if: always() - env: - EXPORT_OUTCOME: ${{ steps.export.outcome }} - run: test "$EXPORT_OUTCOME" = success - - name: Retain the metadata-only TestOps projection - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Record and independently verify each TestOps result - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - *allure - - name: Upload original outcomes, checks and private-evidence receipts - id: allure_upload - env: &upload_env - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results - run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops - - name: Verify actual selected results and complete result-owned attachments - id: verification - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *upload_env - run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops - - name: Retain closed TestOps verification receipts - if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-verify-upload/ - ${{ runner.temp }}/chat-completion-testops-upload/exported.json - retention-days: 30 - if-no-files-found: error diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml deleted file mode 100644 index 826a340ae91..00000000000 --- a/.github/workflows/chat-feature-testops.yml +++ /dev/null @@ -1,451 +0,0 @@ -name: Chat feature TestOps -run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Fixed evaluation, zero-result registration, or one TestOps plan launch - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps supplies its job-run identity; no test/source selector is accepted - ALLURE_USERNAME: - type: string - required: false - description: Reserved TestOps service metadata; never used as authorization - -permissions: - contents: read - -concurrency: - group: chat-feature-testops - cancel-in-progress: false - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - sha: ${{ steps.guard.outputs.sha }} - operation: ${{ steps.guard.outputs.operation }} - triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify fixed default-source dispatch and both actor identities - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - NODE_OPTIONS: '' - NODE_PATH: '' - run: node tests/runner-e2e/testops-chat-dispatch.ts - - select: - name: Validate TestOps selection or register zero-result setup - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - sha256: ${{ steps.selection.outputs.sha256 }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register the public job without an evaluation or a synthetic result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap - node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - - name: Validate the one real TestOps selection before any build or provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection - - name: Start exactly the validated TestOps plan and exit without waiting for its child - if: needs.authorize.outputs.operation == 'launch-plan' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} - CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - GH_TOKEN: ${{ github.token }} - run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan - - name: Retain only closed metadata from zero-result setup - if: always() && needs.authorize.outputs.operation == 'bootstrap' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json - ${{ runner.temp }}/chat-testops-bootstrap-state/once.json - ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json - ${{ runner.temp }}/chat-testops-bootstrap-state/job.json - ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain the one plan-launch receipt and stop state - if: always() && needs.authorize.outputs.operation == 'launch-plan' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-plan-launch-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-launch-plan-state/once.json - ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json - ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain only the token-free exact selection receipt - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-selection-${{ github.run_id }}-1 - path: testops-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable shipped native candidate without model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - name: Install only the source-reviewed graph without lifecycle scripts - run: | - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - pnpm install --frozen-lockfile --ignore-scripts - - name: Build the existing TypeScript and native runner owners - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package source-bound outputs and resolved lock - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 - path: | - chat-testops-build.tar.gz - chat-testops-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: One real native OpenCode backlog request on AWS - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - id-token: write - env: &retention - PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 - TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TARGET_SHA: ${{ needs.authorize.outputs.sha }} - MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Check out the same admitted source for trusted retention - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-testops-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - name: Verify immutable outputs and token-free TestOps selection - env: - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) - tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" - node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json - cmp chat-testops-lock.json chat-testops-worker-lock.json - cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") - - name: Materialize and qualify pinned OpenCode runtime and browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Prepare private evidence with shipped-main authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only the exact private cell storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify strict private custody before the model key - env: &cell_session - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the fixed original request with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain exact original grade and private evidence, including failures - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and export only reviewed fixture results - needs: [authorize, select, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies only - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" - - name: Prepare exact private report and cell-read policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact private cell reader - id: read_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the exact private original and verify all inventories - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: | - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing normalized campaign without exposing original logs - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately even when its grade failed - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only the bound normalized result, JUnit and marked PNGs - id: export - if: always() && steps.private_report.outcome == 'success' - env: - TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts - - name: Upload only the sanitized TestOps projection - if: always() && steps.export.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Report the original outcome to its exact TestOps case - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - *allure - - name: Verify and upload only the bound safe case outcome - id: allure_upload - env: &testops_reporter - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results - run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops - - name: Verify the actual TestOps result after upload processing - id: verify_upload - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *testops_reporter - run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops - - name: Retain only closed actual TestOps verification metadata - if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json - ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json - retention-days: 30 - if-no-files-found: error diff --git a/doc/evals.md b/doc/evals.md index 4071cf01866..d0ea2fa5ec3 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -57,8 +57,9 @@ OpenCode Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, using ordinary production instructions, real browser messages and independent state/provenance checks. A separately selected native OpenCode/DeepSeek1017 -backlog cell measures TestOps integration acceptance with the same oracle and -original bounds. Its source registration is not an AWS or TestOps pass. The +backlog cell supplied historical TestOps integration acceptance with the same +oracle and original bounds. Its former smoke plan and the three-case completion +pilot are retired; their cases and measurements remain in history. The [full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops) maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and fixtures use a preserved immutable catalog revision; trusted workflow helpers run diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 97290f343e1..12dbe906011 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -579,8 +579,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), - readWorkflow("chat-feature-testops.yml"), - readWorkflow("chat-completion-testops.yml"), readWorkflow("chat-catalog-testops.yml"), ]; @@ -621,8 +619,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", "chat-catalog-testops.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); @@ -705,38 +701,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { } continue; } - if (name === "chat-feature-testops.yml") { - assert.equal(providerJobs.length, 1); - const assertBoundary = (block) => { - assert.match(block, /^ chat:\n/); - assert.match(block, /\n needs: \[authorize, select, build\]\n/); - assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); - assert.match(block, /\n environment: runner-e2e-private-paid\n/); - assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); - assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/); - assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); - assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/); - assert.doesNotMatch(block, /ALLURE_TOKEN/); - assert.equal([...block.matchAll(providerSecretReference)].length, 1); - assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); - assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference)); - }; - const block = providerJobs[0]; - assertBoundary(block); - assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); - assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/); - assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/); - assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/); - for (const mutant of [ - block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), - block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), - block.replace(" - *guard\n", ""), - block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"), - block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"), - block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"), - ]) assert.throws(() => assertBoundary(mutant)); - continue; - } + for (const block of providerJobs) { if (name === "runner-protocol-live-evals.yml") { assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}")); diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index a66cf38a374..01d7c354b23 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -181,82 +181,31 @@ backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. -## TestOps completion pilot +## Retired TestOps plans -The separate `Chat completion TestOps` workflow connects only these preserved -catalog cases to the private Paperclip Evals project: +The one-case **Chat feature evals** plan and three-case +**Chat completion pilot — OpenCode 1017** plan are retired. Their scenarios are +included in the [full catalog](#full-catalog-in-testops). Use that plan or select +cases from it; do not recreate the smoke or pilot plans. -- `projects.project-work` -- `projects.project-defaults` -- `instance-operations.restart-service` +The former `chat-feature-testops.yml` and `chat-completion-testops.yml` workflows +are removed. Their bootstrap functions reject registration before any API call +or mutation marker. Shared transport, evidence, export and verification helpers +remain for the full-catalog workflow and retained historical evidence. Retiring a +plan does not delete its cases, results, launches or private evidence. -It evaluates the immutable catalog source in -`chat-completion-testops-contract.mjs`. This keeps the broader catalog on its -integration branch. The source includes the bounded TestOps host prerequisite -from #18515. Changing the pin requires source review. The original DeepSeek -1017 model, case clocks, independent checks, and zero retries remain fixed. -Source registration does not establish a new live result. - -Run the named plan **Chat completion pilot — OpenCode 1017** in TestOps. The -adapter validates the actual project, plan, job, launch and selected cases before -build or provider access. A selected subset runs only those cases. The AWS matrix -has concurrency one and does not cancel sibling cases after a failure. - -The main-only reporter uses the existing protected `ALLURE_TOKEN`; the evaluated -app and worker never receive it. Original evidence stays in the existing private -AWS storage with the unchanged validators, scanners, byte bounds and exact-key -policies. A separate trusted projection carries verdicts, checks, accounting and -private-evidence receipts. The final PNGs are verified in protected storage; -only their checksums and availability are recorded in TestOps. GitHub artifacts -contain metadata only. Raw logs, screenshots, hidden -reasoning, API snapshots, credentials and provider sessions are not attachments. -Original failed grades and cleanup failures remain failed. Same-run readback -requires matching results and complete result-owned attachment metadata within -the existing bounded processing window; it does not claim remote byte hashes. - -Initial setup uses the existing workflow operations: - -1. Dispatch reviewed `master` once with `operation=bootstrap`. This creates or - reconciles only the three exact automated cases and the dedicated plan. It - registers the GitHub job with an empty Allure results directory. It reuses - only the existing enabled GitHub integration from - `CHAT_TESTOPS_INTEGRATION_ID`. No model or fabricated result is created. - The observation receipt captures a closed parameter-shape inspection before - binding. Only the observed single `Branch/defaultValue` definition for - `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases - while preserving that definition; the plan uses its default branch. - Assignment inspection records unassigned and existing-job counts before - binding. A single unassigned bucket must cover the exact selected cases; - foreign jobs and unknown or incomplete buckets stop before writes. -2. Read the closed registration receipt. Store its observed `binding` JSON as - `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. -3. Use TestOps **Run**. For the initial operator launch, set an exact next - workflow run number in `CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER`, set a new - 32-hex `CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE`, then dispatch `launch-plan` - once. Its request produces the TestOps-owned `evaluate` dispatch. Do not - manually dispatch an additional evaluation after an uncertain launch. -4. Verify the fresh TestOps launch, all selected case results, source links, - independent checks, private-evidence receipts and cleanup. Inspect screenshots - through authenticated private-history retrieval. Retain earlier failed attempts. - -Launch preflight searches use TestOps' typed name filter, encoded as Base64 of -UTF-8 JSON in the `search` query parameter. Plain name text returns HTTP 400. -The backlog and completion launchers share this encoding; the full-catalog -launcher uses the completion adapter. Existing nonce and once-only guards still -apply before a plan-run request. - -The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username -metadata. It rejects arbitrary source/model/budget selectors, branch workflows, -replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and -actor identities. The existing one-case backlog plan remains independent. +Retained launch helpers encode TestOps name filters as Base64 of UTF-8 JSON in +`search`. Plain name text returns HTTP 400. Their nonce and once-only guards +remain unchanged; this transport contract does not restore the retired plans. ## Full catalog in TestOps The [Chat catalog TestOps workflow](../../.github/workflows/chat-catalog-testops.yml) registers all 244 scenarios from the 35 YAML areas in a separate **Chat feature catalog — OpenCode 1017 (candidate)** plan. Its source-derived -manifest is `chat-full-catalog-testops-manifest.json`. The existing backlog plan -and three-case completion pilot keep their own jobs, selections and history. +manifest is `chat-full-catalog-testops-manifest.json`. This is the active chat +plan. The former backlog and three-case pilot plans are retired; their cases +and measured history are preserved. Legacy aliases reuse existing TestOps case identities without renaming them. The catalog contains 236 Agent Chat routes, five browser boundaries, one external @@ -322,77 +271,3 @@ and credentials do not enter the public artifact path. Additional credentials reach only cases that declare them. External inputs use the existing selected-case private-input owner, disposable-resource checks and oracle commitments. Unconfigured inputs remain setup blockers. - -## TestOps-backed AWS backlog cell - -The dedicated `Chat feature TestOps` workflow accepts only the shipped default -branch and the fixed OpenCode backlog cell. `evaluate` is the default operation -and requires TestOps' `ALLURE_JOB_RUN_ID`. A protected reporting job verifies the -project, singleton plan, assigned public workflow, selected case and current -GitHub run before any build or provider access. The evaluated process receives -no TestOps token. It retains the original one-query, one-work-run, 120-second -turn, 420-second attempt, 300-second settled-state and zero-retry limits. - -Two metadata operations prepare that route. `bootstrap` registers the actual -public CI job through pinned `allurectl` with an empty results directory and -reconciles only the dedicated case and singleton plan. It produces no passing -test result. After the registered job and protected case/plan/job identifiers -are verified, `launch-plan` sends one TestOps plan-run request and exits. -The TestOps job then dispatches `evaluate`; an operator must not dispatch a -second evaluation manually. Missing or ambiguous registration and selection -metadata stops before inference. - -The workflow reuses the existing AWS fleet and strict private evidence storage. -Its shipped-main authority is separate from manual candidate approval. Original -logs, provider records and failure evidence remain private. The reporting adapter -exports only a source/run-bound normalized result, JUnit and screenshots already -marked by the owned fixture screenshot producer with matching file hashes. -A separate protected reporting job converts that projection to the exact TestOps -case. Failed grades and cleanup failures remain failed. - -Reporting setup uses the master-only `runner-e2e-testops-reporting` environment, -with the existing reporting token delivered through the reviewed sealed-box -bootstrap. `CHAT_TESTOPS_CASE_ID`, `CHAT_TESTOPS_PLAN_ID` and -`CHAT_TESTOPS_JOB_ID` must name the observed dedicated mapping; the numeric -`CHAT_TESTOPS_GITHUB_WORKFLOW_ID` supports TestOps' registered workflow URL. -No arbitrary source, model, case, retry or budget input is accepted. - -The initial `launch-plan` administrator operation also requires a protected -`CHAT_TESTOPS_LAUNCH_RUN_NUMBER`. After registration, the trusted operator creates -this variable once with the reviewed next workflow run number and reads it back. -The launcher compares GitHub's actual run number, attempt, workflow, source and -actors before any TestOps mutation. A later dispatch or replay cannot reuse that -approved number. Missing or uncertain metadata stops; an uncertain request needs -reconciliation rather than a new send. The nonce and launch search provide -additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of -this one-time administrator fence. The launch nonce is protected configuration, -not a dispatch input. Job parameter IDs are opaque. The launcher preserves the -API's exact ID/value pair for `refs/heads/master` and uses the fixed workflow's -`evaluate` default. An operation parameter is used only when the protected -`CHAT_TESTOPS_OPERATION_PARAMETER_ID` matches a returned ID with a `bootstrap` -or `evaluate` value; the launcher then sets it to `evaluate`. Other values, -duplicate IDs and conflicting configuration are rejected. No relationship to -global environment-variable IDs or names is inferred. - -`CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub -integration. The launcher verifies its project, type and enabled state, then binds -only the newly registered public job to it. It never changes integration settings -or credentials. Since the job response does not expose that association, only the -real TestOps-to-GitHub dispatch can qualify it. - -After `allurectl watch` finishes, the same protected reporter checks the actual -TestOps job-run, launch, selected case, result source/status and attachment -metadata. Processing reads are bounded to 12 polls, five-second gaps and a -120-second processing deadline. Unknown or missing results remain unverified. -The check downloads no attachment content and claims no server-side attachment -byte hash. Its closed receipt is retained separately. An upload success alone -is not a passing feature result, and verifying an original failure cannot turn -that failure into a pass. - -Build and worker installs use the existing reviewed report-lock policy. Only its -committed input or output lock is accepted; its authenticated correction is the -only allowed normalization. Dependency manifests, patch inputs and policy must -remain unchanged. Installs are frozen with lifecycle scripts disabled. The -credential-free build rechecks that boundary before packaging, and the worker -matches the source/lock receipt before installing. Arbitrary resolver drift is -rejected before provider access. diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index 5b9f883c935..7d02341af3e 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -135,32 +135,12 @@ export async function inspectPilotCatalog(api) { } export async function bootstrapPilotPlan(api, integrationId, markOnce, context = {}) { + // The pilot plan is retired. Reject before API reads or mutation markers. + if (context.catalog !== 'full') reject('plan_retired'); if (!positive(integrationId)) reject('integration'); const integration = await api(`/api/integration/${integrationId}/project/1`); if (integration.id !== integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) reject('integration'); - if (context.catalog === 'full') return { ...await reconcileFullCatalog(api, markOnce), integrationId }; - const inspected = await inspectPilotCatalog(api); - if (inspected.plans.length && inspected.cases.length !== cases.length) reject('plan_conflict'); - const selected = [...inspected.cases]; - for (const row of cases) { - if (selected.some(value => value.key === row.key)) continue; - await markOnce(`create_case_${row.key}`); - const created = await api('/api/testcase', 'POST', { projectId: 1, name: row.name, fullName: row.fullName, automated: true }); - if (!positive(created?.id)) reject('created_case'); - assertCase(created, { ...row, caseId: created.id }); - selected.push({ ...row, caseId: created.id }); - } - const ordered = cases.map(row => selected.find(value => value.key === row.key)); - let planId = inspected.plans[0]?.id; - if (!planId) { - await markOnce('create_plan'); - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: ordered.map(row => row.caseId) } }); - if (!positive(created?.id)) reject('created_plan'); planId = created.id; - } - const binding = { planId, integrationId, cases: ordered }; - assertPlan(await api(`/api/testplan/${planId}`), binding); - return binding; + return { ...await reconcileFullCatalog(api, markOnce), integrationId }; } export async function completePilotRegistration(api, prepared, official, context, workflowId, markOnce, observe = async () => {}) { diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index 4ee58ebe003..aee4be2527c 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -180,35 +180,9 @@ test('an uncertain launch response is never automatically resent', async () => { assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); }); -test('registration creates only the three missing cases and the dedicated plan after once markers', async () => { - const calls = []; let nextCase = 100; - const get = async (route, method = 'GET', body) => { - calls.push({ route, method, body }); - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - if (route === '/api/testcase' && method === 'POST') return { ...body, id: nextCase++ }; - if (route === '/api/testplan' && method === 'POST') return { ...plan, name: planName }; - if (route === '/api/testplan/10') return plan; - assert.fail(route); - }; - const value = await bootstrapPilotPlan(get, 2, async stage => calls.push({ marker: stage })); - assert.equal(value.cases.length, 3); assert.equal(value.planId, 10); - assert.equal(calls.filter(row => row.method === 'POST').length, 4); - for (const [i, row] of calls.entries()) if (row.method === 'POST') assert.ok(calls[i - 1].marker); -}); - -test('uncertain case creation is never automatically resent', async () => { - let writes = 0, markers = 0; - const get = async (route, method = 'GET') => { - if (method === 'POST') { writes++; throw new Error('uncertain transport'); } - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - assert.fail(route); - }; - await assert.rejects(bootstrapPilotPlan(get, 2, async () => { markers++; }), /uncertain transport/); - assert.equal(writes, 1); assert.equal(markers, 1); +test('retired pilot bootstrap cannot recreate a plan or call the integration API', async () => { + await assert.rejects(bootstrapPilotPlan(() => assert.fail('retired route must not call the API'), 2, + () => assert.fail('retired route must not mark a mutation')), /chat_completion_testops_plan_retired/); }); function api(replace = {}) { diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs index c77d89031a8..c1199bbeb40 100644 --- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -96,37 +96,8 @@ test('the executable guard binds the actual original GitHub run before service o } finally { await rm(root, { recursive: true, force: true }); } }); -test('the real workflow separates metadata authority, provider execution and result reporting', async () => { - const text = await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8'); - const w = parse(text); - assert.deepEqual(Object.keys(w.on.workflow_dispatch.inputs).sort(), ['ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'operation']); - assert.equal(w.concurrency['cancel-in-progress'], false); - assert.equal(w.jobs.chat.strategy['max-parallel'], 1); assert.equal(w.jobs.chat.strategy['fail-fast'], false); - assert.equal(w.jobs.select.environment, 'runner-e2e-testops-reporting'); - assert.equal(w.jobs.chat.environment, 'runner-e2e-private-paid'); - assert.equal(w.jobs.report.environment, 'runner-e2e-private-history'); - assert.equal(w.jobs.upload.environment, 'runner-e2e-testops-reporting'); - for (const [name, job] of Object.entries(w.jobs)) { - assert.ok(job.steps.slice(0, 3).some(step => step.run?.endsWith('chat-completion-testops-cli.mjs guard')), name); - if (name !== 'chat') assert.ok(!JSON.stringify(job).includes('secrets.OPENROUTER_API_KEY'), name); - if (!['select', 'upload'].includes(name)) assert.ok(!JSON.stringify(job).includes('secrets.ALLURE_TOKEN'), name); - } - assert.ok(w.jobs.upload.if.includes("needs.report.result == 'success'")); - assert.ok(w.jobs.report.needs.includes('build')); - assert.ok(w.jobs.report.if.includes("needs.build.result == 'success'")); - assert.ok(w.jobs.report.if.includes("needs.chat.result == 'failure'")); - assert.ok(!text.includes('<<:')); assert.ok(!text.includes('target_branch:')); - assert.equal(w.jobs.build.steps.find(step => step.name?.startsWith('Resolve the fixed')).run.includes('--ignore-scripts'), true); - const paid = w.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); - assert.equal(paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, 'v1'); - assert.equal(paid.env.ALLURE_JOB_RUN_ID, '${{ inputs.ALLURE_JOB_RUN_ID }}'); - assert.ok(paid.run.includes('--max-automatic-retries 0')); - assert.ok(w.jobs.build.steps.some(step => step.run?.includes('chat-completion-testops-host-control.mjs'))); - assert.ok(w.jobs.chat.steps.indexOf(paid) > w.jobs.chat.steps.findIndex(step => step.name?.startsWith('Verify source-bound'))); -}); - test('target checkout cannot populate or overwrite the trusted helper path', async () => { - const definition = parse(await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8')); + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); try { const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 2b4a4f1d049..47fc4939da8 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -53,7 +53,7 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'testops_case_alias_conflict', 'testops_plan_name_conflict', 'testops_plan_identity_invalid', 'testops_integration_identity_invalid', 'testops_bootstrap_identity_invalid', 'testops_plan_without_matching_case', 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', - 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', + 'testops_plan_retired', 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); @@ -139,32 +139,10 @@ export async function inspectChatMetadata(api) { return { projectId: 1, caseConflict: cases.size > 1, cases: [...cases.values()], plans: plans.map(value => ({ id: value.id, name: planName, count: value.testCasesCount })), integrations }; } -export async function ensureChatPlan(api, inspected, markOnce) { - if (inspected.projectId !== 1 || inspected.plans.length > 1) fail('testops_bootstrap_identity_invalid'); - if (inspected.caseConflict === true || inspected.cases.length > 1) fail('testops_case_alias_conflict'); - let selected = inspected.cases[0], plan = inspected.plans[0]; - if (plan && !selected) fail('testops_plan_without_matching_case'); - if (!selected || !plan) await markOnce(); - if (!selected) { - const created = await api('/api/testcase', 'POST', { projectId: 1, name: 'Create a task backlog through chat', fullName, - automated: true, description: 'Canonical scenario: tasks.create-backlog. Readiness and recorded outcomes remain separate.', - expectedResult: 'The real agent creates the requested backlog; public Product E2E assertions and cleanup must pass.' }); - if (!id(created?.id) || created.projectId !== 1 || created.fullName !== fullName) fail('testops_created_case_identity_invalid'); - selected = { id: created.id, fullName, automated: created.automated === true }; - } - if (!plan) { - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: [selected.id] } }); - if (!id(created?.id) || created.projectId !== 1 || created.name !== planName) fail('testops_created_plan_identity_invalid'); - plan = { id: created.id, name: planName }; - } - const actual = await api(`/api/testplan/${plan.id}`); - if (actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== 1 - || actual.treeSelection?.inverted === true || !same(actual.treeSelection?.leafsInclude, [selected.id]) - || (actual.treeSelection?.groupsInclude?.length ?? 0) > 0 || (actual.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('testops_plan_scope_mismatch'); - return { projectId: 1, case: selected, plan: { id: plan.id, name: planName, count: 1 }, - caseUrl: `${endpoint}/project/1/test-cases/${selected.id}`, planUrl: `${endpoint}/testplan/${plan.id}`, - actualResult: null, note: 'Plan preparation is not an executed measurement or a passing result.' }; +// Kept as an explicit retirement boundary for callers of the former smoke lane. +// Catalog registration owns the shared cases; deleting the old plan must persist. +export async function ensureChatPlan() { + fail('testops_plan_retired'); } diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 318c616208c..68f689beee6 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -31,64 +31,14 @@ test('multiple alias identities, foreign project and incomplete metadata stop be await assert.rejects(inspectChatMetadata(async () => ({ content: [], totalPages: 11, totalElements: 1100, number: 0 })), /catalog_incomplete/); }); -test('existing exact plan reuses case and plan without a write', async () => { - const result = await ensureChatPlan(async route => { assert.equal(route, '/api/testplan/8'); return plan; }, - await inspectChatMetadata(inspectApi()), () => assert.fail('must not mark or write')); - assert.equal(result.case.id, 91); - assert.equal(result.plan.id, 8); - assert.equal(result.actualResult, null); -}); - -test('bootstrap creates only the missing exact plan after exclusive marker and reads back scope', async () => { - const calls = []; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - if (method === 'POST') { assert.equal(calls[0].route, 'MARK'); assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [91] } }); } - return plan; - }, { projectId: 1, cases: [caseRow], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(calls.filter(row => row.method === 'POST').length, 1); - assert.equal(result.plan.count, 1); -}); - -test('empty project bootstrap links the genuinely created case into only the new plan', async () => { - const calls = []; - const createdCase = { ...caseRow, id: 92 }; - const createdPlan = { ...plan, treeSelection: { inverted: false, leafsInclude: [92] } }; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - assert.equal(calls[0].route, 'MARK'); - if (route === '/api/testcase') { - assert.equal(body.projectId, 1); assert.equal(body.fullName, fullName); assert.equal(body.automated, true); - return createdCase; - } - if (route === '/api/testplan') { - assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [92] } }); - return createdPlan; - } - assert.equal(route, '/api/testplan/8'); assert.equal(method, undefined); return createdPlan; - }, { projectId: 1, cases: [], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(result.case.id, 92); assert.equal(result.plan.id, 8); - assert.deepEqual(calls.map(call => call.route), ['MARK', '/api/testcase', '/api/testplan', '/api/testplan/8']); -}); - -test('known conflicting plan stops before creating an otherwise missing case or marker', async () => { - await assert.rejects(ensureChatPlan(() => assert.fail('no API write or read needed'), - { projectId: 1, cases: [], plans: [plan] }, () => assert.fail('must not mark')), /plan_without_matching_case/); -}); - -test('new case is created only after alias reconciliation; an uncertain mutation is never resent', async () => { - let writes = 0, markers = 0; - await assert.rejects(ensureChatPlan(async (_route, method) => { - assert.equal(method, 'POST'); writes++; throw new Error('uncertain transport'); - }, { projectId: 1, cases: [], plans: [] }, async () => { markers++; }), /uncertain/); - assert.equal(writes, 1); assert.equal(markers, 1); -}); - -test('existing broad or conflicting plan never expands or changes its scope', async () => { - for (const actual of [{ ...plan, testCasesCount: 2 }, { ...plan, treeSelection: { leafsInclude: [92] } }, - { ...plan, treeSelection: { leafsInclude: [91], inverted: true } }, { ...plan, projectId: 2 }]) { - await assert.rejects(ensureChatPlan(async (_route, method) => { assert.equal(method, undefined); return actual; }, - { projectId: 1, cases: [caseRow], plans: [plan] }, () => assert.fail('no writes')), /scope_mismatch/); +test('retired singleton bootstrap cannot recreate an absent plan or mutate retained metadata', async () => { + for (const inspected of [ + { projectId: 1, cases: [], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [plan] }, + ]) { + await assert.rejects(ensureChatPlan(() => assert.fail('retired route must not call the API'), inspected, + () => assert.fail('retired route must not mark a mutation')), /testops_plan_retired/); } }); diff --git a/tests/runner-e2e/chat-feature-testops-cli.test.mjs b/tests/runner-e2e/chat-feature-testops-cli.test.mjs index 107f046b109..70238eb15f0 100644 --- a/tests/runner-e2e/chat-feature-testops-cli.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-cli.test.mjs @@ -9,7 +9,7 @@ const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/ GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; -test('actual metadata bootstrap caller creates no result or agent and reuses exact case/plan', async t => { +test('actual retired singleton caller refuses plan recreation and creates no result', async t => { const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-bootstrap-'))); t.after(() => rm(temporary, { recursive: true })); const state = path.join(temporary, 'state'), requests = []; @@ -30,9 +30,10 @@ test('actual metadata bootstrap caller creates no result or agent and reuses exa return response({ id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { leafsInclude: [91], inverted: false } }); }; assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, - CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 0); - const plan = JSON.parse(await readFile(path.join(state, 'plan.json'))); - assert.equal(plan.actualResult, null); + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 1); + const failure = JSON.parse(await readFile(path.join(state, 'failure.json'))); + assert.equal(failure.code, 'testops_plan_retired'); + await assert.rejects(readFile(path.join(state, 'plan.json')), { code: 'ENOENT' }); assert.equal(requests.filter(value => value.method === 'POST').length, 1, 'only auth token exchange; no case/plan write'); await assert.rejects(readFile(path.join(state, 'allure-results.json')), { code: 'ENOENT' }); }); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 917d6912f90..65c32e7a7e9 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -244,18 +244,6 @@ test('post-watch foreign backend binding returns safe nonzero and preserves orig assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); }); -test('bootstrap workflow captures official context only after watch and retains exactly named closed receipts', async () => { - const workflow = await readFile(new URL('../../.github/workflows/chat-feature-testops.yml', import.meta.url), 'utf8'); - const start = workflow.indexOf(' - name: Register the public job without an evaluation or a synthetic result'); - const end = workflow.indexOf(' - name: Validate the one real TestOps selection', start); - const step = workflow.slice(start, end); - assert.ok(step.includes("if: needs.authorize.outputs.operation == 'bootstrap'")); - assert.match(step, /allurectl watch -- node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap\n\s+node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap-context/); - assert.equal((step.match(/allurectl watch/g) ?? []).length, 1); assert.ok(!step.includes('continue-on-error')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration.json')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json')); -}); - test('post-watch refuses nonprivate state custody before any command auth or receipt write', async t => { const f = await bootstrapContextFixture(t); await chmod(f.state, 0o755); const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); diff --git a/tests/runner-e2e/testops-chat-workflow.test.ts b/tests/runner-e2e/testops-chat-workflow.test.ts deleted file mode 100644 index 14484cbd30e..00000000000 --- a/tests/runner-e2e/testops-chat-workflow.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { readFileSync } from "node:fs"; -import { parse } from "yaml"; -import { expect, it } from "vitest"; -import { validateChatTestOpsDispatch } from "./testops-chat-dispatch.js"; - -const workflow = parse(readFileSync(new URL("../../.github/workflows/chat-feature-testops.yml", import.meta.url), "utf8")); -function context() { - return { env: { GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", - GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", GITHUB_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", GITHUB_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]" }, - event: { repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, ref: "refs/heads/master", sender: { id: 123 }, inputs: { ALLURE_JOB_RUN_ID: "456" } as Record } }; -} -it("admits the actual TestOps service input and only the closed metadata bootstrap alternative", () => { - const f = context(); expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - f.event.inputs = { operation: "bootstrap" }; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("bootstrap"); - f.event.inputs.operation = "launch-plan"; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("launch-plan"); - f.event.inputs.ALLURE_JOB_RUN_ID = "456"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); - delete f.event.inputs.ALLURE_JOB_RUN_ID; - f.event.inputs.operation = "other"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); -}); -it.each(["unlisted-trigger", "wrong-source", "replay", "extra-input", "missing-jobrun", "foreign-repo"])("rejects %s before any service credential operation", kind => { - const f = context(); let trigger = 123; - if (kind === "unlisted-trigger") trigger = 456; - if (kind === "wrong-source") f.env.GITHUB_REF = "refs/heads/unreviewed"; - if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; - if (kind === "extra-input") f.event.inputs.target_branch = "other"; - if (kind === "missing-jobrun") f.event.inputs = {}; - if (kind === "foreign-repo") f.event.repository.full_name = "foreign/repo"; - expect(() => validateChatTestOpsDispatch(f.env, f.event, trigger)).toThrow(); -}); -it("keeps provider and TestOps service credentials in separate jobs", () => { - const jobs = workflow.jobs; - for (const [id, job] of Object.entries(jobs) as Array<[string, any]>) { - const text = JSON.stringify(job); - if (text.includes("secrets.OPENROUTER_API_KEY")) expect(id).toBe("chat"); - if (text.includes("secrets.ALLURE_TOKEN")) expect(["select", "upload"]).toContain(id); - if (id !== "chat") expect(text).not.toContain("pnpm test:e2e:runner --"); - expect(job.steps.some((step: any) => step.run === "node tests/runner-e2e/testops-chat-dispatch.ts")).toBe(true); - } - expect(jobs.chat.environment).toBe("runner-e2e-private-paid"); - expect(jobs.select.environment).toBe("runner-e2e-testops-reporting"); - expect(jobs.upload.environment).toBe("runner-e2e-testops-reporting"); -}); -it("requires token-isolated selection before the fixed AWS run and keeps bootstrap outside all model jobs", () => { - expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); - expect(Object.keys(workflow.on.workflow_dispatch.inputs)).toEqual(["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"]); - expect(workflow.on.workflow_dispatch.inputs.operation.default).toBe("evaluate"); - expect(workflow.jobs.chat.needs).toContain("select"); expect(workflow.jobs.build.needs).toContain("select"); - expect(workflow.jobs.chat.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.jobs.build.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - const paid = workflow.jobs.chat.steps.find((s: any) => s.env?.OPENROUTER_API_KEY); - expect(paid.run).toContain("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0"); - expect(workflow.jobs.chat.steps.findIndex((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection")).toBeLessThan(workflow.jobs.chat.steps.indexOf(paid)); - expect(workflow.jobs.authorize.outputs.runner).toBe("runs-on/fleet=paperclip-public-pr-x64/env=public-ci"); - expect(workflow.jobs.chat["runs-on"]).toBe("${{ needs.authorize.outputs.runner }}"); -}); -it("uploads only the trusted safe projection after successful strict private report retention", () => { - const steps = workflow.jobs.report.steps; - expect(steps.find((s: any) => s.id === "export").if).toBe("always() && steps.private_report.outcome == 'success'"); - expect(steps.find((s: any) => s.with?.name?.startsWith("runner-e2e-testops-")).with.path).toBe("runner-e2e-testops/"); - expect(JSON.stringify(workflow)).not.toContain("MANUAL_PRIVATE_APPROVALS_JSON"); - expect(Object.values(workflow.jobs).flatMap((j: any) => j.steps).filter((s: any) => s.uses?.startsWith("actions/upload-artifact")).some((s: any) => s.with.path === "tests/runner-e2e/results/")).toBe(false); -}); - -it("initializes the actual TestOps association and never fabricates setup results", () => { - const steps = workflow.jobs.select.steps; - const installer = steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")); - expect(installer.uses).toBe("allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db"); - expect(installer.with["allurectl-version"]).toBe("2.23.1"); - const bootstrap = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs bootstrap")); - expect(bootstrap.run).toContain("mkdir -m 700"); - expect(bootstrap.run).toContain("allurectl watch -- node"); - expect(bootstrap.env.ALLURE_RESULTS).toBe("${{ runner.temp }}/chat-testops-bootstrap-results"); - const select = steps.find((s: any) => s.id === "selection"); - expect(select.run).toContain("allurectl watch -- node"); - for (const name of ["CHAT_TESTOPS_CASE_ID", "CHAT_TESTOPS_PLAN_ID", "CHAT_TESTOPS_JOB_ID"]) - expect(select.env[name]).toBe(`\${{ vars.${name} }}`); - const metadata = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-setup-")); - expect(metadata.if).toBe("always() && needs.authorize.outputs.operation == 'bootstrap'"); - expect(metadata.with.path).not.toContain("results"); -}); -it("authenticates the same-run selection again before converting and uploading", () => { - const steps = workflow.jobs.upload.steps; - const receipt = steps.find((s: any) => s.with?.path === "chat-testops-selection"); - expect(receipt.with["artifact-ids"]).toBe("${{ needs.select.outputs.artifact_id }}"); - const upload = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs upload")); - expect(upload.run).toContain("allurectl watch -- node"); - expect(upload.env.EXPECTED_SELECTION_SHA256).toBe("${{ needs.select.outputs.sha256 }}"); - expect(steps.indexOf(receipt)).toBeLessThan(steps.indexOf(upload)); - expect(JSON.stringify(workflow.jobs.select)).not.toContain("OPENROUTER_API_KEY"); - expect(JSON.stringify(workflow.jobs.upload)).not.toContain("OPENROUTER_API_KEY"); -}); - -it("keeps the one plan launch outside Allure watch and every AWS/model job", () => { - const steps = workflow.jobs.select.steps; - const launch = steps.find((s: any) => s.run?.endsWith("testops-chat-reporter.mjs launch-plan")); - expect(launch.if).toBe("needs.authorize.outputs.operation == 'launch-plan'"); - expect(launch.run).not.toContain("watch"); - expect(launch.env.CHAT_TESTOPS_OPERATION_PARAMETER_ID).toBe("${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}"); - expect(launch.env.CHAT_TESTOPS_INTEGRATION_ID).toBe("${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_NONCE).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}"); - expect(launch.env.TESTOPS_TRIGGERING_ACTOR_ID).toBe("${{ needs.authorize.outputs.triggering_actor_id }}"); - expect(launch.env.TESTOPS_ALLOWED_ACTOR_IDS).toBe("${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - expect(launch.env.GH_TOKEN).toBe("${{ github.token }}"); - expect(launch.run).not.toContain("sleep"); - expect(steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")).if).toBe("needs.authorize.outputs.operation != 'launch-plan'"); - for (const name of ["build", "chat"]) expect(workflow.jobs[name].if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.concurrency["cancel-in-progress"]).toBe(false); -}); - -it("accepts the documented TestOps username metadata without treating it as an actor", () => { - const f = context(); f.event.inputs.ALLURE_USERNAME = "service-metadata"; - expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - expect(() => validateChatTestOpsDispatch(f.env, f.event, 456)).toThrow(); - expect(JSON.stringify(workflow.jobs)).not.toContain("inputs.ALLURE_USERNAME"); -}); - -it("includes every native Node helper control in the normal unit owner", () => { - const launcher = readFileSync(new URL("./run-unit.mjs", import.meta.url), "utf8"); - const config = readFileSync(new URL("./vitest.config.ts", import.meta.url), "utf8"); - for (const file of ["chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"]) { - expect(launcher).toContain(`"tests/runner-e2e/${file}"`); - expect(config).toContain(`"${file}"`); - } -}); - -it("verifies the actual server outcome after watch without masking an original failed upload", () => { - const steps = workflow.jobs.upload.steps; - const watch = steps.find((s: any) => s.id === "allure_upload"); - const verify = steps.find((s: any) => s.id === "verify_upload"); - expect(watch.run).toContain("allurectl watch -- node"); - expect(watch["continue-on-error"]).toBeUndefined(); - expect(verify.run).toBe("node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops"); - expect(verify.if).toBe("always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')"); - expect(verify.env).toEqual(watch.env); - expect(steps.indexOf(verify)).toBeGreaterThan(steps.indexOf(watch)); - const proof = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-verification-")); - expect(proof.with.path.split("\n").filter(Boolean)).toHaveLength(2); - expect(proof.with.path).not.toContain("allure-results"); -}); - -it("carries only the existing source-reviewed lock through build and worker install", () => { - const build = workflow.jobs.build.steps; - const install = build.find((s: any) => s.run?.includes("pnpm install --frozen-lockfile")); - expect(install.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(install.run).not.toContain("--lockfile-only"); - const archive = build.find((s: any) => s.run?.includes("tar -czf chat-testops-build")); - expect(archive.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(archive.run).toContain("chat-testops-lock.json pnpm-lock.yaml"); - const worker = workflow.jobs.chat.steps.find((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection"); - expect(worker.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json"); - expect(worker.run).toContain("cmp chat-testops-lock.json chat-testops-worker-lock.json"); - expect(worker.run.indexOf("cmp chat-testops-lock.json")).toBeLessThan(worker.run.indexOf("pnpm install --frozen-lockfile")); -}); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 91c0e3e9f29..ca84b9de5ab 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -606,8 +606,6 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -893,24 +891,7 @@ describe("public repository paid workflow security", () => { for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); - if (["chat-feature-testops.yml", "chat-completion-testops.yml"].includes(filename)) { - const guard = filename === "chat-feature-testops.yml" ? "testops-chat-dispatch.ts" : "chat-completion-testops-cli.mjs guard"; - expect(other).toContain(`run: node tests/runner-e2e/${guard}`); - expect(other).toContain("ref: ${{ github.sha }}"); - expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; - expect(chat).toContain("environment: runner-e2e-private-paid"); - expect(chat).toContain("needs: [authorize, select, build]"); - expect(chat).toContain(" - *guard"); - expect(chat).toContain("verify-selection"); - expect(chat).not.toContain("ALLURE_TOKEN"); - expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); - const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; - expect(report).toContain("environment: runner-e2e-private-history"); - expect(report).toContain("manual-private-transfer.ts report-session"); - expect(report).not.toContain("ALLURE_TOKEN"); - continue; - } + for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");