diff --git a/.github/workflows/chat-completion-testops.yml b/.github/workflows/chat-completion-testops.yml new file mode 100644 index 00000000000..7016f76dfec --- /dev/null +++ b/.github/workflows/chat-completion-testops.yml @@ -0,0 +1,504 @@ +name: Chat completion TestOps +run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} + +on: + workflow_dispatch: + inputs: + operation: + type: choice + options: [evaluate, bootstrap, launch-plan] + default: evaluate + description: Run the exact TestOps selection, register its plan, or launch it once + ALLURE_JOB_RUN_ID: + type: string + required: false + description: TestOps job-run identity; source, model and clocks are fixed + ALLURE_USERNAME: + type: string + required: false + description: Reserved service metadata; never grants authority + +permissions: + contents: read + actions: read + +concurrency: + group: chat-completion-testops + cancel-in-progress: false + +env: + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + +jobs: + authorize: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + operation: ${{ steps.guard.outputs.operation }} + target_sha: ${{ steps.guard.outputs.target_sha }} + runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci + steps: + - &checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - &node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - &guard + name: Verify actual main-only dispatch and both actors + id: guard + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard + + select: + name: Register the dedicated plan or validate its real TestOps selection + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: runner-e2e-testops-reporting + outputs: + artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} + selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} + execution_ids: ${{ steps.selection.outputs.execution_ids }} + matrix: ${{ steps.selection.outputs.matrix }} + policy: ${{ steps.selection.outputs.policy }} + steps: + - *checkout + - *node + - *guard + - &allure + if: needs.authorize.outputs.operation != 'launch-plan' + uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 + with: + allurectl-version: 2.23.1 + allure-endpoint: https://paperclip.testops.cloud + allure-project-id: '1' + allure-token: ${{ secrets.ALLURE_TOKEN }} + - name: Register three exact cases with no synthetic test result + if: needs.authorize.outputs.operation == 'bootstrap' + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results + CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} + run: | + mkdir -m 700 "$ALLURE_RESULTS" + allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap + node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context + - name: Validate case, plan, job, launch and same-run source before provider access + id: selection + if: needs.authorize.outputs.operation == 'evaluate' + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results + CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} + run: | + mkdir -m 700 "$ALLURE_RESULTS" + allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection + - name: Request the one reviewed plan launch and exit + if: needs.authorize.outputs.operation == 'launch-plan' + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} + CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }} + CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }} + run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan + - name: Retain token-free registration and launch receipts + if: always() && needs.authorize.outputs.operation != 'evaluate' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-completion-testops-setup-${{ github.run_id }}-1 + path: | + ${{ runner.temp }}/chat-completion-testops-bootstrap/ + ${{ runner.temp }}/chat-completion-testops-launch-plan/ + retention-days: 30 + if-no-files-found: error + - name: Retain the exact authenticated selection + id: selection_upload + if: steps.selection.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-completion-testops-selection-${{ github.run_id }}-1 + path: testops-pilot-selection.json + retention-days: 30 + if-no-files-found: error + + build: + name: Build the immutable catalog source without service or model credentials + needs: [authorize, select] + if: needs.authorize.outputs.operation == 'evaluate' + runs-on: ${{ needs.authorize.outputs.runner }} + timeout-minutes: 45 + outputs: + artifact_id: ${{ steps.bundle.outputs.artifact-id }} + steps: + - *checkout + - *node + - *guard + - &pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: 'false' + NPM_CONFIG_FUND: 'false' + NPM_CONFIG_UPDATE_NOTIFIER: 'false' + with: + version: 9.15.4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + - name: Reject target-owned trusted helper paths + id: target_boundary + run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: .testops-trusted + persist-credentials: false + - name: Resolve the fixed candidate graph with lifecycle scripts disabled + run: | + pnpm install --lockfile-only --ignore-scripts + pnpm install --frozen-lockfile --ignore-scripts + - name: Build existing TypeScript and native Runner outputs + run: | + pnpm --filter @paperclipai/paperclip-runner build:typescript + - name: Exercise the real workflow and pinned source host contract before model access + run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs + - name: Build native Runner outputs + run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + - name: Package the resolved lock and exact source outputs + env: + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$TARGET_SHA" + tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar + sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256 + - name: Retain credential-free immutable outputs + id: bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-completion-build-${{ github.run_id }}-1 + path: | + chat-completion-build.tar.gz + chat-completion-build.tar.gz.sha256 + retention-days: 1 + if-no-files-found: error + + chat: + name: ${{ matrix.executionId }} + needs: [authorize, select, build] + if: needs.authorize.outputs.operation == 'evaluate' + strategy: + fail-fast: false + max-parallel: 1 + matrix: ${{ fromJSON(needs.select.outputs.matrix) }} + runs-on: ${{ needs.authorize.outputs.runner }} + timeout-minutes: 32 + environment: runner-e2e-private-paid + permissions: + contents: read + actions: read + id-token: write + env: &retention + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + steps: + - *checkout + - *node + - *guard + - *pnpm + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.target_sha }} + persist-credentials: false + - name: Reject target-owned trusted helper paths + id: target_boundary + run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: .testops-trusted + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.build.outputs.artifact_id }} + path: chat-completion-build + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.select.outputs.artifact_id }} + path: chat-completion-testops-selection + - name: Verify source-bound build and authenticated selection before credentials + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + run: | + set -euo pipefail + test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" + node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json + (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) + tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" + pnpm install --frozen-lockfile --ignore-scripts + (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") + - name: Qualify the pinned provider runtime and preinstalled browser + run: | + node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs + node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' + google-chrome --version + pnpm exec playwright install ffmpeg + - name: Bind exact private evidence under the authenticated TestOps authority + id: private + env: + TARGET_ROOT: ${{ github.workspace }} + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare + - name: Obtain only this cell's exact-key storage session + id: cell_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.private.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Verify private custody before delivering the model key + env: &cell_session + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify + - name: Execute only the selected original case with zero retries + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} + PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration + PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome + run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 + - name: Retain original evidence after success or failure + if: always() && steps.cell_credentials.outcome == 'success' + env: *cell_session + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell + + report: + name: Retain private originals and build the exact safe TestOps projection + needs: [authorize, select, build, chat] + if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure') + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: runner-e2e-private-history + permissions: + contents: read + actions: read + id-token: write + env: *retention + steps: + - *checkout + - *node + - *guard + - *pnpm + - name: Install trusted reporting dependencies with lifecycle scripts disabled + run: | + node tests/runner-e2e/prepare-report-lock.mjs + pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" + - name: Prepare unchanged exact-key report and cell policies + id: session + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session + - name: Obtain private report writer + id: report_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Obtain exact first-cell reader + id: reader_0 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Download the first exact original + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '0' + AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell + - name: Obtain exact second-cell reader + id: reader_1 + if: fromJSON(steps.session.outputs.cell_count) > 1 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.read_policy_1 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Download the second exact original + if: steps.reader_1.outcome == 'success' + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '1' + AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell + - name: Obtain exact third-cell reader + id: reader_2 + if: fromJSON(steps.session.outputs.cell_count) > 2 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.read_policy_2 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Download the third exact original + if: steps.reader_2.outcome == 'success' + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '2' + AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell + - name: Verify every selected original and its completeness receipt + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads + - name: Build the existing report while preserving failed source grades + id: normalize + continue-on-error: true + env: + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration + run: | + mkdir -p runner-e2e-merged-report + pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 + - name: Retain the complete merged report privately + id: private_report + if: always() && steps.report_credentials.outcome == 'success' + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report + AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report + - name: Export only exact reviewed synthetic fixture results + id: export + if: always() && steps.private_report.outcome == 'success' + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts + - name: Require a complete safe projection even after a failed normalization + if: always() + env: + EXPORT_OUTCOME: ${{ steps.export.outcome }} + run: test "$EXPORT_OUTCOME" = success + - name: Retain the metadata-only TestOps projection + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-completion-testops-projection-${{ github.run_id }}-1 + path: runner-e2e-testops/ + retention-days: 30 + if-no-files-found: error + + upload: + name: Record and independently verify each TestOps result + needs: [authorize, select, chat, report] + if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: runner-e2e-testops-reporting + steps: + - *checkout + - *node + - *guard + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: chat-completion-testops-projection-${{ github.run_id }}-1 + path: runner-e2e-testops + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.select.outputs.artifact_id }} + path: chat-completion-testops-selection + - *allure + - name: Upload original outcomes, checks and private-evidence receipts + id: allure_upload + env: &upload_env + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results + run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops + - name: Verify actual selected results and complete result-owned attachments + id: verification + if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') + env: *upload_env + run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops + - name: Retain closed TestOps verification receipts + if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-completion-testops-verification-${{ github.run_id }}-1 + path: | + ${{ runner.temp }}/chat-completion-testops-verify-upload/ + ${{ runner.temp }}/chat-completion-testops-upload/exported.json + retention-days: 30 + if-no-files-found: error diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 8efc8264bed..34d2853f624 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -580,6 +580,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), readWorkflow("chat-feature-testops.yml"), + readWorkflow("chat-completion-testops.yml"), ]; for (const workflow of actionPinWorkflows) { @@ -620,6 +621,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-live-evals.yml", "runner-protocol-live-evals.yml", "chat-feature-testops.yml", + "chat-completion-testops.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); const providerSecretReference = @@ -652,6 +654,36 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { (block) => [...block.matchAll(providerSecretReference)].length > 0, ); assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`); + if (name === "chat-completion-testops.yml") { + assert.equal(providerJobs.length, 1); + const assertBoundary = (block) => { + assert.match(block, /^ chat:\n/); + assert.match(block, /\n needs: \[authorize, select, build\]\n/); + assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); + assert.match(block, /\n environment: runner-e2e-private-paid\n/); + assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); + assert.match(block, /chat-completion-testops-cli\.mjs verify-selection/); + assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); + assert.match(block, /--id "\$\{\{ matrix\.executionId \}\}" --max-parallel 1 --max-automatic-retries 0/); + assert.match(block, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/); + assert.match(block, /max-parallel: 1/); + assert.doesNotMatch(block, /ALLURE_TOKEN/); + assert.equal([...block.matchAll(providerSecretReference)].length, 1); + assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); + assert.ok(block.indexOf("chat-completion-testops-cli.mjs verify-selection") < block.search(providerSecretReference)); + }; + const block = providerJobs[0]; assertBoundary(block); + assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); + for (const mutant of [ + block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), + block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), + block.replace(" - *guard\n", ""), + block.replace("chat-completion-testops-cli.mjs verify-selection", "chat-completion-testops-cli.mjs unverified-selection"), + block.replace('--id "${{ matrix.executionId }}"', "--all"), + block.replace("PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1", "PAPERCLIP_CHAT_COMPLETION_TESTOPS: unknown"), + ]) assert.throws(() => assertBoundary(mutant)); + continue; + } if (name === "chat-feature-testops.yml") { assert.equal(providerJobs.length, 1); const assertBoundary = (block) => { diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 17b681cf0af..80aaa875fbc 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -181,6 +181,66 @@ backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. +## TestOps completion pilot + +The separate `Chat completion TestOps` workflow connects only these preserved +catalog cases to the private Paperclip Evals project: + +- `projects.project-work` +- `projects.project-defaults` +- `instance-operations.restart-service` + +It evaluates the immutable catalog source in +`chat-completion-testops-contract.mjs`. This keeps the broader catalog on its +integration branch. The source includes the bounded TestOps host prerequisite +from #18515. Changing the pin requires source review. The original DeepSeek +1017 model, case clocks, independent checks, and zero retries remain fixed. +Source registration does not establish a new live result. + +Run the named plan **Chat completion pilot — OpenCode 1017** in TestOps. The +adapter validates the actual project, plan, job, launch and selected cases before +build or provider access. A selected subset runs only those cases. The AWS matrix +has concurrency one and does not cancel sibling cases after a failure. + +The main-only reporter uses the existing protected `ALLURE_TOKEN`; the evaluated +app and worker never receive it. Original evidence stays in the existing private +AWS storage with the unchanged validators, scanners, byte bounds and exact-key +policies. A separate trusted projection carries verdicts, checks, accounting and +private-evidence receipts. The final PNGs are verified in protected storage; +only their checksums and availability are recorded in TestOps. GitHub artifacts +contain metadata only. Raw logs, screenshots, hidden +reasoning, API snapshots, credentials and provider sessions are not attachments. +Original failed grades and cleanup failures remain failed. Same-run readback +requires matching results and complete result-owned attachment metadata within +the existing bounded processing window; it does not claim remote byte hashes. + +Initial setup uses the existing workflow operations: + +1. Dispatch reviewed `master` once with `operation=bootstrap`. This creates or + reconciles only the three exact automated cases and the dedicated plan. It + registers the GitHub job with an empty Allure results directory. It reuses + only the existing enabled GitHub integration from + `CHAT_TESTOPS_INTEGRATION_ID`. No model or fabricated result is created. + The observation receipt captures a closed parameter-shape inspection before + binding. Only the observed single `Branch/defaultValue` definition for + `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases + while preserving that definition; the plan uses its default branch. +2. Read the closed registration receipt. Store its observed `binding` JSON as + `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. +3. Use TestOps **Run**. For the initial operator launch, set an exact next + workflow run number in `CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER`, set a new + 32-hex `CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE`, then dispatch `launch-plan` + once. Its request produces the TestOps-owned `evaluate` dispatch. Do not + manually dispatch an additional evaluation after an uncertain launch. +4. Verify the fresh TestOps launch, all selected case results, source links, + independent checks, private-evidence receipts and cleanup. Inspect screenshots + through authenticated private-history retrieval. Retain earlier failed attempts. + +The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username +metadata. It rejects arbitrary source/model/budget selectors, branch workflows, +replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and +actor identities. The existing one-case backlog plan remains independent. + ## TestOps-backed AWS backlog cell The dedicated `Chat feature TestOps` workflow accepts only the shipped default diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs new file mode 100644 index 00000000000..a10d46b88cc --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -0,0 +1,260 @@ +import { testopsClient, endpoint } from './chat-feature-testops-api.mjs'; +import { cases, planName, targetSha, workflow, positive, reject, validateSelection } from './chat-completion-testops-contract.mjs'; + +export { testopsClient, endpoint }; +const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); +const pageRows = value => { + if (!Array.isArray(value?.content) || value.content.length > 100 || !Number.isSafeInteger(value.totalPages) + || value.totalPages < 0 || value.totalPages > 1) reject('incomplete_page'); + return value.content; +}; +export function bindingFromEnv(env) { + let value; + try { value = JSON.parse(env.CHAT_COMPLETION_TESTOPS_BINDING); } catch { reject('binding_missing'); } + if (!positive(value.planId) || !positive(value.jobId) || !positive(value.integrationId) + || !Array.isArray(value.cases) || value.cases.length !== cases.length) reject('binding'); + const selected = cases.map(row => { + const matched = value.cases.filter(v => v.key === row.key); + if (matched.length !== 1 || !positive(matched[0].caseId)) reject('binding'); + return { ...row, caseId: matched[0].caseId }; + }); + if (new Set(selected.map(row => row.caseId)).size !== cases.length) reject('binding'); + return { planId: value.planId, jobId: value.jobId, integrationId: value.integrationId, cases: selected }; +} +export function isWorkflowUrl(url, workflowId) { + return url === `https://github.com/paperclipai/paperclip/actions/workflows/${workflow.split('/').at(-1)}` + || positive(workflowId) && url === `https://github.com/paperclipai/paperclip/actions/workflows/${workflowId}` + || url === 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+completion+TestOps%22'; +} +function assertCase(actual, expected) { + if (actual.id !== expected.caseId || actual.projectId !== 1 || actual.fullName !== expected.fullName + || actual.automated !== true || actual.deleted === true) reject('case_mismatch'); +} +function assertPlan(actual, binding) { + const selected = binding.cases.map(row => row.caseId).sort((a, b) => a - b); + if (actual.id !== binding.planId || actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== selected.length + || !Array.isArray(actual.treeSelection?.leafsInclude) || !same([...actual.treeSelection.leafsInclude].sort((a, b) => a - b), selected) + || actual.treeSelection.inverted === true || (actual.treeSelection.groupsInclude?.length ?? 0) !== 0 + || (actual.treeSelection.leafsExclude?.length ?? 0) !== 0) reject('plan_mismatch'); +} +function assertJob(job, binding, workflowId) { + if (job.id !== binding.jobId || job.type !== 'github' || !isWorkflowUrl(job.url, workflowId)) reject('job_mismatch'); +} +function assertBranchDefinition(job) { + // Observed job-definition DTO, not the id/value plan-override DTO. + if (!Array.isArray(job.parameters) || job.parameters.length !== 1 || !positive(job.parameters[0]?.id) + || job.parameters[0].name !== 'Branch' || job.parameters[0].defaultValue !== 'refs/heads/master' + || Object.hasOwn(job.parameters[0], 'value')) reject('job_parameters'); +} + +export async function validatePilotSelection(api, env, context, workflowId) { + const binding = bindingFromEnv(env); + for (const row of binding.cases) assertCase(await api(`/api/testcase/${row.caseId}`), row); + assertPlan(await api(`/api/testplan/${binding.planId}`), binding); + const jobs = await api(`/api/testplan/${binding.planId}/job`); + if (!Array.isArray(jobs) || jobs.length !== 1 || jobs[0].job?.id !== binding.jobId || jobs[0].testCasesCount !== cases.length) reject('plan_job_mismatch'); + assertJob(await api(`/api/job/${binding.jobId}`), binding, workflowId); + const run = await api(`/api/jobrun/${env.ALLURE_JOB_RUN_ID}`); + if (run.id !== Number(env.ALLURE_JOB_RUN_ID) || run.job?.id !== binding.jobId || run.job?.type !== 'github' + || !['scheduled', 'in_progress'].includes(run.stage) || !positive(run.launchId) + || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) reject('jobrun_mismatch'); + const launch = await api(`/api/launch/${run.launchId}`); + if (launch.id !== run.launchId || launch.projectId !== 1 || launch.closed === true) reject('launch_mismatch'); + const planned = await api(`/api/jobrun/${run.id}/plan?expected=true`); + if (!Array.isArray(planned) || !planned.length || planned.length > cases.length || new Set(planned.map(row => row.id)).size !== planned.length) reject('selection'); + const selected = planned.map(row => { + const found = binding.cases.find(value => value.caseId === row.id && value.fullName === row.selector); + if (!found) reject('foreign_selection'); + return found; + }); + const value = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, + targetSha, allureJobRunId: env.ALLURE_JOB_RUN_ID, projectId: 1, planId: binding.planId, + jobId: binding.jobId, launchId: launch.id, cases: cases.flatMap(row => selected.filter(value => value.key === row.key)) }; + validateSelection(value, context, env.ALLURE_JOB_RUN_ID); + return value; +} + +export async function inspectPilotCatalog(api) { + const found = new Map(), seen = new Set(); let pages, total; + for (let index = 0; index < 10; index++) { + const page = await api(`/api/testcase?${new URLSearchParams({ projectId: '1', page: String(index), size: '100', sort: 'id,asc' })}`); + if (!Array.isArray(page?.content) || page.content.length > 100 || page.number !== index + || !Number.isSafeInteger(page.totalPages) || page.totalPages < 0 || page.totalPages > 10 + || !Number.isSafeInteger(page.totalElements) || page.totalElements < 0 || page.totalElements > 1000 + || pages !== undefined && (pages !== page.totalPages || total !== page.totalElements)) reject('catalog_incomplete'); + pages = page.totalPages; total = page.totalElements; + for (const row of page.content) { + if (!positive(row.id) || seen.has(row.id)) reject('catalog_incomplete'); + seen.add(row.id); + const actual = await api(`/api/testcase/${row.id}`); + if (actual.id !== row.id || actual.projectId !== 1) reject('case_mismatch'); + const expected = cases.find(value => value.fullName === actual.fullName); + if (!expected) continue; + if (found.has(expected.key)) reject('case_conflict'); + assertCase(actual, { ...expected, caseId: actual.id }); + found.set(expected.key, { ...expected, caseId: actual.id }); + } + if (index + 1 >= pages) break; + } + if (seen.size !== total) reject('catalog_incomplete'); + const plans = pageRows(await api(`/api/testplan?${new URLSearchParams({ projectId: '1', name: planName, page: '0', size: '100' })}`)).filter(row => row.name === planName); + if (plans.length > 1 || plans.some(row => row.projectId !== 1 || !positive(row.id))) reject('plan_conflict'); + return { cases: [...found.values()], plans: plans.map(row => ({ id: row.id, name: row.name })) }; +} + +export async function bootstrapPilotPlan(api, integrationId, markOnce) { + if (!positive(integrationId)) reject('integration'); + const integration = await api(`/api/integration/${integrationId}/project/1`); + if (integration.id !== integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) reject('integration'); + const inspected = await inspectPilotCatalog(api); + if (inspected.plans.length && inspected.cases.length !== cases.length) reject('plan_conflict'); + const selected = [...inspected.cases]; + for (const row of cases) { + if (selected.some(value => value.key === row.key)) continue; + await markOnce(`create_case_${row.key}`); + const created = await api('/api/testcase', 'POST', { projectId: 1, name: row.name, fullName: row.fullName, automated: true }); + if (!positive(created?.id)) reject('created_case'); + assertCase(created, { ...row, caseId: created.id }); + selected.push({ ...row, caseId: created.id }); + } + const ordered = cases.map(row => selected.find(value => value.key === row.key)); + let planId = inspected.plans[0]?.id; + if (!planId) { + await markOnce('create_plan'); + const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, + treeSelection: { inverted: false, leafsInclude: ordered.map(row => row.caseId) } }); + if (!positive(created?.id)) reject('created_plan'); planId = created.id; + } + const binding = { planId, integrationId, cases: ordered }; + assertPlan(await api(`/api/testplan/${planId}`), binding); + return binding; +} + +export async function completePilotRegistration(api, prepared, official, context, workflowId, markOnce, observe = async () => {}) { + const run = await api(`/api/jobrun/${official.jobRunId}`); + if (run.id !== Number(official.jobRunId) || run.job?.type !== 'github' || !positive(run.job?.id) + || run.launchId !== official.launchId || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) reject('registration'); + const launch = await api(`/api/launch/${official.launchId}`); + if (launch.id !== official.launchId || launch.projectId !== 1) reject('registration'); + const binding = { ...prepared, jobId: run.job.id }; + assertPlan(await api(`/api/testplan/${binding.planId}`), binding); + const job = await api(`/api/job/${binding.jobId}`); assertJob(job, binding, workflowId); + await observe({ schema: 'paperclip.chat-completion-testops-registration-observed/v1', ...context, + projectId: 1, jobId: job.id, launchId: official.launchId, allureJobRunId: official.jobRunId, + parameterCount: Array.isArray(job.parameters) ? job.parameters.length : null, + parameters: Array.isArray(job.parameters) ? job.parameters.slice(0, 4).map(row => ({ + id: positive(row?.id) ? row.id : null, name: row?.name === 'Branch' ? 'Branch' : 'UNRESOLVED', + defaultValue: row?.defaultValue === 'refs/heads/master' ? 'refs/heads/master' : 'NOT_PROJECTED', + runtimeValuePresent: Object.hasOwn(row ?? {}, 'value') })) : null, + qualification: 'inspection_only', actualResult: null }); + assertBranchDefinition(job); + const assignments = await api(`/api/testplan/${binding.planId}/job`); + if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row.job?.id !== binding.jobId || row.testCasesCount !== cases.length)) reject('plan_job_mismatch'); + await markOnce('bind_owned_job'); + // PATCH only the documented enable/integration fields. Preserve the opaque + // definition exactly; no guessed parameter-definition write codec is used. + await api(`/api/job/${binding.jobId}`, 'PATCH', { integrationId: binding.integrationId, canRun: true }); + const patched = await api(`/api/job/${binding.jobId}`); assertJob(patched, binding, workflowId); assertBranchDefinition(patched); + if (!same(patched.parameters, job.parameters) || (Object.hasOwn(patched, 'canRun') && patched.canRun !== true)) reject('job_parameters'); + if (!assignments.length) { + await markOnce('assign_plan'); + await api(`/api/testplan/${binding.planId}/assign`, 'POST', { jobId: binding.jobId, + selection: { inverted: false, manual: false, leafsInclude: binding.cases.map(row => row.caseId) } }); + } + const bound = await api(`/api/testplan/${binding.planId}/job`); + if (!Array.isArray(bound) || bound.length !== 1 || bound[0].job?.id !== binding.jobId || bound[0].testCasesCount !== cases.length) reject('plan_job_mismatch'); + return { schema: 'paperclip.chat-completion-testops-registration/v1', ...context, binding, + parametersMode: 'job-defaults', branchDefinition: { id: job.parameters[0].id, name: 'Branch', defaultValue: 'refs/heads/master' }, + planUrl: `${endpoint}/testplan/${binding.planId}`, jobUrl: `${endpoint}/project/1/jobs`, qualification: 'registration_only', actualResult: null }; +} + +export async function launchPilotPlan(api, env, context, workflowId, markOnce) { + const binding = bindingFromEnv(env); + if (!/^[1-9][0-9]{0,15}$/.test(env.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER ?? '') + || env.GITHUB_RUN_NUMBER !== env.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER + || !/^[a-f0-9]{32}$/.test(env.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE ?? '')) reject('launch_fence'); + for (const row of binding.cases) assertCase(await api(`/api/testcase/${row.caseId}`), row); + assertPlan(await api(`/api/testplan/${binding.planId}`), binding); + const job = await api(`/api/job/${binding.jobId}`); assertJob(job, binding, workflowId); + assertBranchDefinition(job); + const assignments = await api(`/api/testplan/${binding.planId}/job`); + if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0].job?.id !== binding.jobId + || assignments[0].testCasesCount !== cases.length) reject('plan_job_mismatch'); + const name = `${planName} · ${env.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE}`; + const existing = pageRows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: name, page: '0', size: '100' })}`)); + if (existing.length) reject('launch_already_requested'); + await markOnce('launch_plan'); + const launched = await api(`/api/testplan/${binding.planId}/run`, 'POST', { launchName: name }); + if (!positive(launched?.id) || launched.projectId !== 1 || launched.name !== name) reject('launch_mismatch'); + return { schema: 'paperclip.chat-completion-testops-launch/v1', ...context, launchId: launched.id, + planId: binding.planId, jobId: binding.jobId, caseIds: binding.cases.map(row => row.caseId), + launchUrl: `${endpoint}/launch/${launched.id}`, status: 'REQUESTED', actualResult: null }; +} + +export async function verifyPilotUpload(api, selection, context, exported, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { + const selected = validateSelection(selection, context, selection.allureJobRunId); + if (!Array.isArray(exported) || exported.length !== selected.length || new Set(exported.map(row => row.caseId)).size !== selected.length) reject('export_mismatch'); + for (const row of exported) { + const found = selected.find(value => value.caseId === row.caseId); + if (!found || row.executionId !== found.executionId || !['passed', 'failed'].includes(row.status) + || !['passed', 'failed', 'not_started'].includes(row.cleanup) || row.status === 'passed' && row.cleanup !== 'passed' + || !Array.isArray(row.attachments) || row.attachments.length !== 2 + || row.attachments.filter(value => value.name === 'Execution provenance and checks' && value.contentType === 'application/json').length !== 1 + || row.attachments.filter(value => value.name === 'Private evidence receipt' && value.contentType === 'application/json').length !== 1 + || row.attachments.some(value => !['Execution provenance and checks', 'Private evidence receipt'].includes(value.name) + || !Number.isSafeInteger(value.contentLength) || value.contentLength <= 0 || value.contentLength > 12 * 1024 * 1024 + || value.contentType !== 'application/json')) reject('export_mismatch'); + } + const deadline = now() + 120_000; + const get = route => { + const remaining = deadline - now(); if (remaining < 1) reject('upload_unverified'); + return api(route, 'GET', undefined, Math.min(30_000, remaining)); + }; + const signature = rows => rows.map(row => JSON.stringify([row.name, row.contentType, row.contentLength])).sort(); + for (let poll = 0; poll < 12 && now() < deadline; poll++) { + const run = await get(`/api/jobrun/${selection.allureJobRunId}`); + const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`; + if (run.id !== Number(selection.allureJobRunId) || run.job?.id !== selection.jobId || run.job?.type !== 'github' + || !['in_progress', 'finished'].includes(run.stage) + || run.launchId !== selection.launchId || run.url !== runUrl) reject('jobrun_mismatch'); + const launch = await get(`/api/launch/${selection.launchId}`); + if (launch.id !== selection.launchId || launch.projectId !== 1) reject('launch_mismatch'); + const page = await get(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`); + const listed = pageRows(page); + if (!Number.isSafeInteger(page.totalElements) || page.totalElements !== listed.length || listed.length > selected.length + || new Set(listed.map(row => row.id)).size !== listed.length || new Set(listed.map(row => row.testCaseId)).size !== listed.length) reject('result_page'); + let ready = listed.length === selected.length; const verified = []; + for (const listedRow of listed) { + const expected = selected.find(value => value.caseId === listedRow.testCaseId); + if (!expected || !positive(listedRow.id)) reject('foreign_result'); + const wanted = exported.find(value => value.caseId === expected.caseId); + const result = await get(`/api/testresult/${listedRow.id}`); + if (result.id !== listedRow.id || result.projectId !== 1 || result.launchId !== selection.launchId || result.testCaseId !== expected.caseId + || result.fullName !== expected.fullName || result.jobRun?.id !== Number(selection.allureJobRunId) + || result.jobRun?.launchId !== selection.launchId || result.jobRun?.url !== runUrl + || result.jobRun.job != null && (result.jobRun.job.id !== selection.jobId + || result.jobRun.job.type != null && result.jobRun.job.type !== 'github' + || result.jobRun.job.url != null && !isWorkflowUrl(result.jobRun.job.url, context.workflowId)) + || !Array.isArray(result.links) || result.links.some(link => typeof link.url === 'string' + && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl + || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== `https://github.com/paperclipai/paperclip/commit/${targetSha}`)) + || !['unknown', wanted.status].includes(result.status)) reject('result_mismatch'); + const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(result.id), page: '0', size: '100' })}`); + const attachments = pageRows(attached); + if (attached.totalElements !== attachments.length || attachments.length > 10 || new Set(attachments.map(row => row.id)).size !== attachments.length + || attachments.some(row => !positive(row.id) || row.fromTestCase === true)) reject('attachment_mismatch'); + // TestOps may publish result metadata before its attachment processing is + // ready. Keep every final requirement; wait within the existing bound. + const complete = result.status === wanted.status && attachments.every(row => row.missed === false && row.fromTestCase === false) + && same(signature(attachments), signature(wanted.attachments)); + ready &&= complete; + verified.push({ executionId: expected.executionId, caseId: expected.caseId, resultId: result.id, + status: result.status, cleanup: wanted.cleanup, attachmentsVerified: complete, resultUrl: `${endpoint}/testresult/${result.id}` }); + } + if (ready && now() < deadline) return { schema: 'paperclip.chat-completion-testops-verification/v1', ...context, + projectId: 1, planId: selection.planId, jobId: selection.jobId, launchId: selection.launchId, + allureJobRunId: selection.allureJobRunId, verification: 'VERIFIED', results: verified, launchUrl: `${endpoint}/launch/${selection.launchId}` }; + if (poll < 11 && now() + 5000 < deadline) await wait(5000); + } + reject('upload_unverified'); +} diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs new file mode 100644 index 00000000000..ac52c469549 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -0,0 +1,275 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { cases, targetSha, planName } from './chat-completion-testops-contract.mjs'; +import { validatePilotSelection, verifyPilotUpload, bootstrapPilotPlan, completePilotRegistration, launchPilotPlan } from './chat-completion-testops-api.mjs'; + +const selected = cases.map((row, i) => ({ ...row, caseId: 100 + i })); +const context = { runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha, operation: 'evaluate' }; +const binding = { planId: 10, jobId: 11, integrationId: 2, cases: selected }; +const env = { ALLURE_JOB_RUN_ID: '77', CHAT_COMPLETION_TESTOPS_BINDING: JSON.stringify(binding) }; +const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, + planId: 10, jobId: 11, allureJobRunId: '77', launchId: 12, cases: selected }; +const runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123'; +const plan = { id: 10, name: 'Chat completion pilot — OpenCode 1017', projectId: 1, testCasesCount: 3, + treeSelection: { inverted: false, leafsInclude: selected.map(row => row.caseId) } }; +const job = { id: 11, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-completion-testops.yml' }; +const jobrun = { id: 77, job, launchId: 12, stage: 'in_progress', url: runUrl }; +const attachment = { id: 900, name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 100, + missed: false, fromTestCase: false }; +const screenshot = { ...attachment, id: 901, name: 'Private evidence receipt', contentType: 'application/json', contentLength: 1000 }; +const exported = selected.map(row => ({ caseId: row.caseId, executionId: row.executionId, status: 'passed', cleanup: 'passed', + attachments: [attachment, screenshot].map(({ id, missed, fromTestCase, ...rest }) => rest) })); + +// Observed TestOps job-definition carrier: Branch/defaultValue. This is +// distinct from the id/value carrier accepted by plan runtime overrides. +const branchDefinition = { id: 34, name: 'Branch', defaultValue: 'refs/heads/master' }; +function mutationApi({ parameters = [branchDefinition], assignments = [], existingLaunch = false, uncertain = false } = {}) { + const calls = [], current = { ...job, parameters: structuredClone(parameters) }; + let assigned = structuredClone(assignments); + const api = async (route, method = 'GET', body) => { + calls.push({ route, method, body }); + if (route === '/api/jobrun/77') return { ...jobrun, stage: 'finished' }; + if (route === '/api/launch/12') return { id: 12, projectId: 1 }; + if (route === '/api/testplan/10') return plan; + if (route === '/api/job/11') { + if (method === 'PATCH') { + assert.deepEqual(body, { integrationId: 2, canRun: true }); + current.canRun = true; + } + return structuredClone(current); + } + if (route === '/api/testplan/10/job') return structuredClone(assigned); + if (route === '/api/testplan/10/assign') { + assert.equal(method, 'POST'); + assert.deepEqual(body, { jobId: 11, selection: { inverted: false, manual: false, leafsInclude: [100, 101, 102] } }); + assigned = [{ job: structuredClone(current), testCasesCount: 3 }]; return null; + } + const selectedCase = selected.find(row => route === `/api/testcase/${row.caseId}`); + if (selectedCase) return { ...selectedCase, id: selectedCase.caseId, projectId: 1, automated: true }; + if (route.startsWith('/api/launch?')) return { content: existingLaunch ? [{ id: 13 }] : [], totalPages: existingLaunch ? 1 : 0 }; + if (route === '/api/testplan/10/run') { + assert.equal(method, 'POST'); + if (uncertain) throw new Error('uncertain transport'); + return { id: 13, projectId: 1, name: body.launchName }; + } + assert.fail(route); + }; + return { api, calls, current }; +} +const prepared = { planId: 10, integrationId: 2, cases: selected }; +const official = { jobRunId: '77', launchId: 12 }; +const operatorEnv = { CHAT_COMPLETION_TESTOPS_BINDING: JSON.stringify(binding), GITHUB_RUN_NUMBER: '5', + CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: '5', CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: 'ab'.repeat(16) }; + +test('registration preserves the observed Branch definition and binds only the owned three-case job', async () => { + const f = mutationApi(), markers = []; + const receipt = await completePilotRegistration(f.api, prepared, official, context, 999, async stage => markers.push(stage)); + assert.deepEqual(receipt.binding.cases, selected); + assert.deepEqual(f.current.parameters, [branchDefinition]); + assert.deepEqual(markers, ['bind_owned_job', 'assign_plan']); + assert.deepEqual(f.calls.filter(row => row.method !== 'GET').map(row => row.route), ['/api/job/11', '/api/testplan/10/assign']); +}); + +for (const parameters of [[], [branchDefinition, { id: 35, name: 'operation', defaultValue: 'bootstrap' }], [{ id: 34, value: 'refs/heads/master' }]]) { + test(`registration rejects unreviewed parameter carrier/count ${JSON.stringify(parameters)} before writes`, async () => { + const f = mutationApi({ parameters }); + await assert.rejects(completePilotRegistration(f.api, prepared, official, context, 999, async () => assert.fail('no mutation marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); + }); +} + +for (const assignments of [[{ job: { id: 99 }, testCasesCount: 3 }], [{ job, testCasesCount: 2 }], [{ job, testCasesCount: 3 }, { job, testCasesCount: 3 }]]) { + test('registration rejects ambiguous assignments before writes', async () => { + const f = mutationApi({ assignments }); + await assert.rejects(completePilotRegistration(f.api, prepared, official, context, 999, async () => assert.fail('no mutation marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); + }); +} + +test('registration keeps an existing exact assignment without assigning it twice', async () => { + const f = mutationApi({ assignments: [{ job: { ...job, parameters: [branchDefinition] }, testCasesCount: 3 }] }); + await completePilotRegistration(f.api, prepared, official, context, 999, async () => {}); + assert.equal(f.calls.filter(row => row.route.endsWith('/assign')).length, 0); +}); + +test('bootstrap retains a closed observation before rejecting an unknown parameter shape', async () => { + const f = mutationApi({ parameters: [{ id: 34, name: 'PRIVATE_NAME', defaultValue: 'PRIVATE_DEFAULT', value: 'PRIVATE_VALUE' }] }); + let observed; + await assert.rejects(completePilotRegistration(f.api, prepared, official, context, 999, async () => assert.fail('no mutation marker'), async value => { observed = value; })); + assert.equal(observed.qualification, 'inspection_only'); assert.equal(observed.jobId, 11); + assert.equal(observed.parameters[0].runtimeValuePresent, true); + assert.ok(!JSON.stringify(observed).includes('PRIVATE_')); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); +}); + +test('the fenced launcher requests one exact three-case TestOps launch', async () => { + const f = mutationApi({ assignments: [{ job, testCasesCount: 3 }] }), markers = []; + const receipt = await launchPilotPlan(f.api, operatorEnv, context, 999, async stage => markers.push(stage)); + assert.deepEqual(receipt.caseIds, [100, 101, 102]); assert.deepEqual(markers, ['launch_plan']); + assert.deepEqual(f.calls.filter(row => row.method !== 'GET'), [{ route: '/api/testplan/10/run', method: 'POST', body: { + launchName: `${planName} · ${operatorEnv.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE}` } }]); +}); + +for (const changed of [{ GITHUB_RUN_NUMBER: '6' }, { CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: 'not-a-nonce' }, { CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: '' }]) { + test(`launcher rejects its invalid fence before requests: ${Object.keys(changed)[0]}`, async () => { + const f = mutationApi(); + await assert.rejects(launchPilotPlan(f.api, { ...operatorEnv, ...changed }, context, 999, async () => assert.fail('no marker'))); + assert.equal(f.calls.length, 0); + }); +} + +test('launcher reconciles an existing nonce without requesting a second launch', async () => { + const f = mutationApi({ assignments: [{ job, testCasesCount: 3 }], existingLaunch: true }); + await assert.rejects(launchPilotPlan(f.api, operatorEnv, context, 999, async () => assert.fail('no marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); +}); + +for (const assignments of [[], [{ job: { id: 99 }, testCasesCount: 3 }], [{ job, testCasesCount: 2 }]]) { + test('launcher rejects an unbound or mismatched assignment without requesting work', async () => { + const f = mutationApi({ assignments }); + await assert.rejects(launchPilotPlan(f.api, operatorEnv, context, 999, async () => assert.fail('no mutation marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); + }); +} + +test('launcher rejects a changed default branch before requesting work', async () => { + const f = mutationApi({ assignments: [{ job, testCasesCount: 3 }], parameters: [{ ...branchDefinition, defaultValue: 'refs/heads/candidate' }] }); + await assert.rejects(launchPilotPlan(f.api, operatorEnv, context, 999, async () => assert.fail('no mutation marker'))); + assert.equal(f.calls.filter(row => row.method !== 'GET').length, 0); +}); + +test('an uncertain launch response is never automatically resent', async () => { + const f = mutationApi({ assignments: [{ job, testCasesCount: 3 }], uncertain: true }), seen = new Set(); + const mark = async stage => { if (seen.has(stage)) throw new Error('already attempted'); seen.add(stage); }; + await assert.rejects(launchPilotPlan(f.api, operatorEnv, context, 999, mark), /uncertain transport/); + await assert.rejects(launchPilotPlan(f.api, operatorEnv, context, 999, mark), /already attempted/); + assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); +}); + +test('registration creates only the three missing cases and the dedicated plan after once markers', async () => { + const calls = []; let nextCase = 100; + const get = async (route, method = 'GET', body) => { + calls.push({ route, method, body }); + if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; + if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; + if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; + if (route === '/api/testcase' && method === 'POST') return { ...body, id: nextCase++ }; + if (route === '/api/testplan' && method === 'POST') return { ...plan, name: planName }; + if (route === '/api/testplan/10') return plan; + assert.fail(route); + }; + const value = await bootstrapPilotPlan(get, 2, async stage => calls.push({ marker: stage })); + assert.equal(value.cases.length, 3); assert.equal(value.planId, 10); + assert.equal(calls.filter(row => row.method === 'POST').length, 4); + for (const [i, row] of calls.entries()) if (row.method === 'POST') assert.ok(calls[i - 1].marker); +}); + +test('uncertain case creation is never automatically resent', async () => { + let writes = 0, markers = 0; + const get = async (route, method = 'GET') => { + if (method === 'POST') { writes++; throw new Error('uncertain transport'); } + if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; + if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; + if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; + assert.fail(route); + }; + await assert.rejects(bootstrapPilotPlan(get, 2, async () => { markers++; }), /uncertain transport/); + assert.equal(writes, 1); assert.equal(markers, 1); +}); + +function api(replace = {}) { + const fixtures = { + '/api/testplan/10': plan, '/api/testplan/10/job': [{ job, testCasesCount: 3 }], '/api/job/11': job, + '/api/jobrun/77': jobrun, '/api/launch/12': { id: 12, projectId: 1, closed: false }, + '/api/jobrun/77/plan?expected=true': selected.map(row => ({ id: row.caseId, selector: row.fullName })), + '/api/testresult?launchId=12&page=0&size=100': { content: selected.map((row, i) => ({ id: 200 + i, testCaseId: row.caseId })), totalElements: 3, totalPages: 1 }, + }; + for (const [i, row] of selected.entries()) { + fixtures[`/api/testcase/${row.caseId}`] = { id: row.caseId, fullName: row.fullName, projectId: 1, automated: true }; + fixtures[`/api/testresult/${200 + i}`] = { id: 200 + i, testCaseId: row.caseId, fullName: row.fullName, + projectId: 1, launchId: 12, status: 'passed', links: [], jobRun: { id: 77, launchId: 12, url: runUrl, job } }; + fixtures[`/api/testresult/attachment?testResultId=${200 + i}&page=0&size=100`] = { content: [attachment, screenshot], totalElements: 2, totalPages: 1 }; + } + return async (route, method = 'GET') => { + assert.equal(method, 'GET'); assert.ok(Object.hasOwn(fixtures, route), route); + return typeof replace[route] === 'function' ? replace[route]() : structuredClone(replace[route] ?? fixtures[route]); + }; +} + +test('validates actual registered plan, job run and all three selected case identities', async () => { + const value = await validatePilotSelection(api(), env, context, 999); + assert.deepEqual(value.cases.map(row => row.executionId), cases.map(row => row.executionId)); + assert.equal(value.targetSha, targetSha); +}); + +test('a genuine one-case selection executes only that case', async () => { + const value = await validatePilotSelection(api({ '/api/jobrun/77/plan?expected=true': [{ id: selected[2].caseId, selector: selected[2].fullName }] }), env, context, 999); + assert.deepEqual(value.cases.map(row => row.key), ['work']); +}); + +for (const kind of ['foreign-case', 'wrong-selector', 'duplicate-case', 'wrong-workflow', 'foreign-project', 'wrong-jobrun', 'wrong-plan-job']) { + test(`stops ${kind} before any provider work`, async () => { + const replacements = {}; + if (kind === 'foreign-case') replacements['/api/jobrun/77/plan?expected=true'] = [{ id: 999, selector: selected[0].fullName }]; + if (kind === 'wrong-selector') replacements['/api/jobrun/77/plan?expected=true'] = [{ id: 100, selector: selected[1].fullName }]; + if (kind === 'duplicate-case') replacements['/api/jobrun/77/plan?expected=true'] = [{ id: 100, selector: selected[0].fullName }, { id: 100, selector: selected[0].fullName }]; + if (kind === 'wrong-workflow') replacements['/api/job/11'] = { ...job, url: job.url.replace('chat-completion', 'foreign') }; + if (kind === 'foreign-project') replacements['/api/launch/12'] = { id: 12, projectId: 2 }; + if (kind === 'wrong-jobrun') replacements['/api/jobrun/77'] = { ...jobrun, url: runUrl + '4' }; + if (kind === 'wrong-plan-job') replacements['/api/testplan/10/job'] = [{ job: { id: 12 }, testCasesCount: 3 }]; + await assert.rejects(validatePilotSelection(api(replacements), env, context, 999)); + }); +} + +test('verifies exact results and attachment metadata for all three cases', async () => { + const value = await verifyPilotUpload(api(), selection, context, exported); + assert.equal(value.verification, 'VERIFIED'); assert.equal(value.results.length, 3); + assert.ok(value.results.every(row => row.attachmentsVerified && row.status === 'passed')); +}); + +test('accepts a finished TestOps job with its original complete results', async () => { + const value = await verifyPilotUpload(api({ '/api/jobrun/77': { ...jobrun, stage: 'finished' } }), selection, context, exported); + assert.equal(value.verification, 'VERIFIED'); +}); +for (const stage of ['cancelled', 'failed', 'scheduled']) { + test(`never qualifies a ${stage} TestOps job even with all results and attachments`, async () => { + await assert.rejects(verifyPilotUpload(api({ '/api/jobrun/77': { ...jobrun, stage } }), selection, context, exported)); + }); +} + +test('waits for an explicitly missing attachment and requires final complete metadata', async () => { + let polls = 0, slept = 0, clock = 0; + const get = api({ '/api/testresult/attachment?testResultId=200&page=0&size=100': () => { + polls++; + return { content: [{ ...attachment, missed: polls === 1 }, screenshot], totalElements: 2, totalPages: 1 }; + } }); + const value = await verifyPilotUpload(get, selection, context, exported, { now: () => clock, wait: async ms => { slept++; clock += ms; } }); + assert.equal(value.verification, 'VERIFIED'); assert.equal(polls, 2); assert.equal(slept, 1); +}); + +test('permanently missing or inherited attachments never qualify', async () => { + for (const field of ['missed', 'fromTestCase']) { + let clock = 0; + const get = api({ '/api/testresult/attachment?testResultId=200&page=0&size=100': { content: [{ ...attachment, [field]: true }, screenshot], totalElements: 2, totalPages: 1 } }); + await assert.rejects(verifyPilotUpload(get, selection, context, exported, { now: () => clock, wait: async ms => { clock += ms; } })); + assert.ok(clock < 120_000); + } +}); + +for (const kind of ['wrong-case', 'wrong-source-run', 'wrong-status', 'foreign-result', 'duplicate-result', 'extra-result', 'foreign-embedded-job', 'foreign-source-link']) { + test(`preserves failure for ${kind} during server result readback`, async () => { + const replacements = {}; + if (kind === 'wrong-case') replacements['/api/testresult/200'] = { id: 200, testCaseId: 999 }; + if (kind === 'wrong-source-run') replacements['/api/jobrun/77'] = { ...jobrun, url: runUrl + '4' }; + if (kind === 'wrong-status') replacements['/api/testresult/200'] = { id: 200, testCaseId: 100, fullName: selected[0].fullName, + projectId: 1, launchId: 12, status: 'failed', links: [], jobRun: { id: 77, launchId: 12, url: runUrl, job } }; + if (kind === 'foreign-embedded-job' || kind === 'foreign-source-link') replacements['/api/testresult/200'] = { + id: 200, testCaseId: 100, fullName: selected[0].fullName, projectId: 1, launchId: 12, status: 'passed', + links: kind === 'foreign-source-link' ? [{ url: 'https://github.com/paperclipai/paperclip/commit/' + 'b'.repeat(40) }] : [], + jobRun: { id: 77, launchId: 12, url: runUrl, job: kind === 'foreign-embedded-job' ? { ...job, type: 'foreign' } : job } }; + if (kind === 'foreign-result') replacements['/api/testresult?launchId=12&page=0&size=100'] = { content: [{ id: 999, testCaseId: 999 }], totalElements: 1, totalPages: 1 }; + if (kind === 'duplicate-result') replacements['/api/testresult?launchId=12&page=0&size=100'] = { content: [{ id: 200, testCaseId: 100 }, { id: 200, testCaseId: 100 }], totalElements: 2, totalPages: 1 }; + if (kind === 'extra-result') replacements['/api/testresult?launchId=12&page=0&size=100'] = { content: selected.map((row, i) => ({ id: 200 + i, testCaseId: row.caseId })).concat({ id: 999, testCaseId: 999 }), totalElements: 4, totalPages: 1 }; + await assert.rejects(verifyPilotUpload(api(replacements), selection, context, exported)); + }); +} diff --git a/tests/runner-e2e/chat-completion-testops-cli.mjs b/tests/runner-e2e/chat-completion-testops-cli.mjs new file mode 100644 index 00000000000..69e32c2391d --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-cli.mjs @@ -0,0 +1,130 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { appendFile, lstat, mkdir, open, readFile, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { admitDispatch, targetSha, workflow, hash, reject, validateSelection, evidencePolicy } from './chat-completion-testops-contract.mjs'; +import { testopsClient, bootstrapPilotPlan, completePilotRegistration, validatePilotSelection, launchPilotPlan, verifyPilotUpload } from './chat-completion-testops-api.mjs'; +import { parseOfficialBootstrapContext } from './testops-chat-reporter.mjs'; +import { exportPilotAllure, readPilotExport } from './chat-completion-testops-results.mjs'; + +const execute = promisify(execFile); +async function readOwned(file, root, limit) { + const absolute = path.resolve(file), base = await realpath(root); + if (!absolute.startsWith(`${base}${path.sep}`)) reject('file_path'); + let current = base; const parts = path.relative(base, absolute).split(path.sep); + for (const [index, part] of parts.entries()) { + if (!/^[A-Za-z0-9._-]+$/.test(part) || ['.', '..'].includes(part)) reject('file_path'); + current = path.join(current, part); const stat = await lstat(current); + if (stat.isSymbolicLink() || (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile() || stat.nlink !== 1 || stat.size > limit)) reject('file_path'); + } + return readFile(absolute); +} +export async function actualContext(env, exec = execute) { + if (!env.GITHUB_EVENT_PATH || !env.GH_TOKEN || !/^[1-9][0-9]{0,15}$/.test(env.GITHUB_RUN_ID ?? '')) reject('context'); + const stat = await lstat(env.GITHUB_EVENT_PATH); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 65_536) reject('event'); + const event = JSON.parse(await readFile(env.GITHUB_EVENT_PATH, 'utf8')); + const { stdout } = await exec('gh', ['api', `repos/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`], + { env, shell: false, timeout: 30_000, maxBuffer: 512_000, encoding: 'utf8' }); + const run = JSON.parse(stdout), context = admitDispatch(env, event, run.triggering_actor?.id); + if (run.id !== Number(context.runId) || run.run_attempt !== 1 || run.head_sha !== context.workflowSha || run.head_branch !== 'master' + || run.event !== 'workflow_dispatch' || run.path !== workflow || run.repository?.full_name !== 'paperclipai/paperclip' || run.repository.private !== false + || run.actor?.id !== context.actorId || run.run_number !== Number(env.GITHUB_RUN_NUMBER) + || !Number.isSafeInteger(run.workflow_id) || run.workflow_id <= 0) reject('actual_run'); + return { ...context, workflowId: run.workflow_id }; +} +async function readSelection(file, env, context) { + if (!/^[a-f0-9]{64}$/.test(env.EXPECTED_SELECTION_SHA256 ?? '')) reject('selection_digest'); + const bytes = await readOwned(file, env.GITHUB_WORKSPACE, 32_768); + if (hash(bytes) !== env.EXPECTED_SELECTION_SHA256) reject('selection_digest'); + const selected = JSON.parse(bytes); validateSelection(selected, context, env.ALLURE_JOB_RUN_ID); return selected; +} +async function output(env, key, value) { + if (!env.GITHUB_OUTPUT || /[\r\n]/.test(value)) reject('output'); + await appendFile(env.GITHUB_OUTPUT, `${key}=${value}\n`); +} +async function stateDirectory(env, command, existing = false) { + const root = path.join(await realpath(env.RUNNER_TEMP), `chat-completion-testops-${command}`); + if (!existing) await mkdir(root, { mode: 0o700 }); + const stat = await lstat(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.mode & 0o077 || process.getuid && stat.uid !== process.getuid()) reject('state_directory'); + return root; +} +const writeReceipt = (root, name, value) => writeFile(path.join(root, name), JSON.stringify(value, null, 2), { flag: 'wx', mode: 0o600 }); +function mutationMarker(root) { + return async stage => { + if (!/^[a-z_]{1,80}$/.test(stage)) reject('mutation_marker'); + const handle = await open(path.join(root, `once-${stage}.json`), 'wx', 0o600); + await handle.writeFile(JSON.stringify({ stage, status: 'STARTED_NO_AUTOMATIC_RESEND' })); await handle.close(); + }; +} + +export async function run(command, argument, env = process.env, { fetchImpl = fetch, exec = execute } = {}) { + let state; + try { + if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload'].includes(command)) reject('operation'); + const context = await actualContext(env, exec); + if (command === 'guard') { + await output(env, 'operation', context.operation); await output(env, 'target_sha', targetSha); + await output(env, 'workflow_id', String(context.workflowId)); return 0; + } + if (command === 'verify-selection') { + if (Object.hasOwn(env, 'ALLURE_TOKEN')) reject('worker_credential'); + await readSelection(argument, env, context); return 0; + } + if (['bootstrap', 'bootstrap-context'].includes(command) && context.operation !== 'bootstrap' + || command === 'launch-plan' && context.operation !== 'launch-plan' + || ['selection', 'upload', 'verify-upload'].includes(command) && context.operation !== 'evaluate') reject('operation'); + state = await stateDirectory(env, command === 'bootstrap-context' ? 'bootstrap' : command, command === 'bootstrap-context'); + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + if (command === 'bootstrap') { + const prepared = await bootstrapPilotPlan(api, Number(env.CHAT_TESTOPS_INTEGRATION_ID), mutationMarker(state)); + await writeReceipt(state, 'prepared.json', prepared); return 0; + } + if (command === 'bootstrap-context') { + const official = parseOfficialBootstrapContext((await exec('allurectl', ['job-run', 'env'], + { env, shell: false, timeout: 30_000, maxBuffer: 16_384, encoding: 'utf8' })).stdout); + const prepared = JSON.parse(await readOwned(path.join(state, 'prepared.json'), env.RUNNER_TEMP, 32_768)); + const receipt = await completePilotRegistration(api, prepared, official, context, context.workflowId, mutationMarker(state), + observation => writeReceipt(state, 'registration-observed.json', observation)); + await writeReceipt(state, 'registration.json', receipt); return 0; + } + if (command === 'launch-plan') { + const receipt = await launchPilotPlan(api, env, context, context.workflowId, mutationMarker(state)); + await writeReceipt(state, 'launch.json', receipt); return 0; + } + if (command === 'selection') { + const selection = await validatePilotSelection(api, env, context, context.workflowId); + await writeReceipt(state, 'selection.json', selection); + const bytes = Buffer.from(JSON.stringify(selection, null, 2)); + await writeFile(path.join(env.GITHUB_WORKSPACE, 'testops-pilot-selection.json'), bytes, { flag: 'wx', mode: 0o600 }); + await output(env, 'selection_sha256', hash(bytes)); + await output(env, 'execution_ids', JSON.stringify(selection.cases.map(row => row.executionId))); + await output(env, 'matrix', JSON.stringify({ include: selection.cases.map(row => ({ executionId: row.executionId, key: row.key })) })); + await output(env, 'policy', JSON.stringify(evidencePolicy(selection, context, env.ALLURE_JOB_RUN_ID))); return 0; + } + const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context); + if (command === 'upload') { + const exported = await exportPilotAllure(path.resolve(argument), env.ALLURE_RESULTS, context, selected); + await writeReceipt(state, 'exported.json', exported); return exported.some(row => row.status !== 'passed') ? 1 : 0; + } + const prior = path.join(env.RUNNER_TEMP, 'chat-completion-testops-upload', 'exported.json'); + const exported = JSON.parse(await readOwned(prior, env.RUNNER_TEMP, 32_768)); + const { campaign } = await readPilotExport(path.resolve(argument), context, selected); + if (exported.some(row => { + const original = campaign.results.find(value => value.executionId === row.executionId); + return !original || row.status !== (original.status === 'passed' && original.cleanup === 'passed' ? 'passed' : 'failed') || row.cleanup !== original.cleanup; + })) reject('upload_receipt'); + const verification = await verifyPilotUpload(api, selected, context, exported); + await writeReceipt(state, 'verification.json', verification); return 0; + } catch (error) { + const code = /^(chat_completion_testops_[a-z_]{1,80}|testops_(?:api|auth)_http(?:_[1-5][0-9]{2})?)$/.test(error?.message ?? '') ? error.message : 'chat_completion_testops_unknown'; + const receipt = { command, code, httpStatus: Number.isInteger(error?.status) ? error.status : null }; + if (state) await writeReceipt(state, 'failure.json', receipt).catch(() => {}); + process.stderr.write(`Chat completion TestOps stopped: ${JSON.stringify(receipt)}\n`); return 1; + } +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + run(process.argv[2], process.argv[3]).then(code => { process.exitCode = code; }); +} diff --git a/tests/runner-e2e/chat-completion-testops-contract.d.mts b/tests/runner-e2e/chat-completion-testops-contract.d.mts new file mode 100644 index 00000000000..5c034f52e85 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-contract.d.mts @@ -0,0 +1,28 @@ +export interface PilotCase { + key: "control" | "defaults" | "work"; + scenario: string; + name: string; + checks: number; + minimumRuns: number; + maximumRuns: number; + executionId: string; + fullName: string; +} +export const cases: readonly Readonly[]; +export const targetSha: string; +export const launchSha256: string; +export const workflow: string; +export const planName: string; +export const model: string; +export function hash(value: string | Uint8Array): string; +export function validRunCount(ids: readonly string[] | undefined, descriptor: PilotCase): boolean; +export interface PilotContext { runId: string; attempt: string; workflowSha: string; targetSha: string } +export interface PilotSelection extends PilotContext { + schema: string; projectId: number; planId: number; jobId: number; launchId: number; + allureJobRunId: string; cases: (PilotCase & { caseId: number })[]; +} +export function validateSelection(selection: PilotSelection, context: PilotContext, jobRunId: string): (PilotCase & { caseId: number })[]; +export function evidencePolicy(selection: PilotSelection, context: PilotContext, jobRunId: string): { + version: number; approvals: { workflowSha: string; targetSha: string; runId: string; attempt: string; + executionIds: string[]; protocol: string; launchSha256: string }[]; +}; diff --git a/tests/runner-e2e/chat-completion-testops-contract.mjs b/tests/runner-e2e/chat-completion-testops-contract.mjs new file mode 100644 index 00000000000..39a1b3727ef --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-contract.mjs @@ -0,0 +1,73 @@ +import { createHash } from 'node:crypto'; + +// The catalog remains on its preservation/integration branch. Updating this +// immutable source pin requires a reviewed source change. +export const targetSha = 'de0941835cc8f0d01c982574ee258e110babdbeb'; +export const launchSha256 = 'f2e69957c350b220b06edca49194a4bd07d70bc882f80838eeb7203184c0bd0d'; +export const workflow = '.github/workflows/chat-completion-testops.yml'; +export const planName = 'Chat completion pilot — OpenCode 1017'; +export const model = 'openrouter/deepseek/deepseek-v4.1-flash-1017'; +export const cases = Object.freeze([ + { key: 'control', scenario: 'instance-operations.restart-service', name: 'Restart an instance and retain the conversation', checks: 9, minimumRuns: 1, maximumRuns: 1 }, + { key: 'defaults', scenario: 'projects.project-defaults', name: 'Save project defaults for future work', checks: 11, minimumRuns: 3, maximumRuns: 3 }, + { key: 'work', scenario: 'projects.project-work', name: 'Create and delegate project work through chat', checks: 15, minimumRuns: 1, maximumRuns: 3 }, +].map(row => Object.freeze({ ...row, executionId: `chat-features.runner-opencode.local.${row.scenario}`, + fullName: `paperclip.Product E2E.chat-features.${row.scenario}` }))); +export const positive = value => Number.isSafeInteger(value) && value > 0; +export const hash = value => createHash('sha256').update(value).digest('hex'); +// These bounds are copied from the pinned source, not inferred from one +// measurement. The real-source control checks them before provider access. +export const validRunCount = (ids, descriptor) => Array.isArray(ids) && ids.length >= descriptor.minimumRuns + && ids.length <= descriptor.maximumRuns && new Set(ids).size === ids.length; +const numeric = value => /^[1-9][0-9]{0,15}$/.test(value ?? '') && positive(Number(value)); +export function reject(code) { throw new Error(`chat_completion_testops_${code}`); } + +export function admitDispatch(env, event, triggeringActorId) { + let allowed; + try { allowed = JSON.parse(env.TESTOPS_ALLOWED_ACTOR_IDS); } catch { reject('actor'); } + const branch = event.repository?.default_branch; + const operation = event.inputs?.operation ?? 'evaluate'; + if (env.GITHUB_ACTIONS !== 'true' || env.GITHUB_EVENT_NAME !== 'workflow_dispatch' + || env.GITHUB_REPOSITORY !== 'paperclipai/paperclip' || event.repository?.full_name !== 'paperclipai/paperclip' + || event.repository?.private !== false || branch !== 'master' || env.GITHUB_REF !== 'refs/heads/master' + || event.ref !== env.GITHUB_REF || env.GITHUB_WORKFLOW_REF !== `paperclipai/paperclip/${workflow}@refs/heads/master` + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? '') || env.GITHUB_RUN_ATTEMPT !== '1' + || !numeric(env.GITHUB_RUN_ID) || !numeric(env.GITHUB_ACTOR_ID) || event.sender?.id !== Number(env.GITHUB_ACTOR_ID) + || !Array.isArray(allowed) || !allowed.length || allowed.some(v => !positive(v)) + || !allowed.includes(Number(env.GITHUB_ACTOR_ID)) || !allowed.includes(triggeringActorId) + || !event.inputs || Object.keys(event.inputs).some(k => !['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME'].includes(k)) + || !['bootstrap', 'evaluate', 'launch-plan'].includes(operation) + || (operation === 'evaluate' ? !numeric(event.inputs.ALLURE_JOB_RUN_ID) : (event.inputs.ALLURE_JOB_RUN_ID ?? '') !== '')) reject('dispatch'); + return Object.freeze({ runId: env.GITHUB_RUN_ID, attempt: '1', workflowSha: env.GITHUB_SHA, targetSha, operation, + actorId: Number(env.GITHUB_ACTOR_ID), triggeringActorId }); +} + +export function selectedCases(rows) { + if (!Array.isArray(rows) || !rows.length || rows.length > cases.length || new Set(rows.map(row => row.executionId)).size !== rows.length) reject('selection'); + const selected = cases.filter(row => rows.some(value => value.executionId === row.executionId)); + if (selected.length !== rows.length) reject('selection'); + for (const row of rows) { + const expected = selected.find(value => value.executionId === row.executionId); + if (!positive(row.caseId) || row.fullName !== expected.fullName || row.key !== expected.key || row.scenario !== expected.scenario) reject('selection'); + } + if (new Set(rows.map(row => row.caseId)).size !== rows.length) reject('selection'); + return selected.map(expected => ({ ...expected, caseId: rows.find(row => row.executionId === expected.executionId).caseId })); +} + +export function validateSelection(selection, context, jobRunId) { + if (selection.schema !== 'paperclip.chat-completion-testops-selection/v1' || selection.projectId !== 1 + || selection.runId !== context.runId || selection.attempt !== '1' || selection.workflowSha !== context.workflowSha + || selection.targetSha !== targetSha || selection.allureJobRunId !== jobRunId || !numeric(jobRunId) + || ['planId', 'jobId', 'launchId'].some(key => !positive(selection[key]))) reject('selection_binding'); + return selectedCases(selection.cases); +} + +// This authority is emitted only by the trusted main-only selector after an +// authenticated TestOps selection. The existing evidence validator, protocol, +// scanners, byte limits and exact-key S3 policies remain the enforcement owner. +export function evidencePolicy(selection, context, jobRunId) { + const selected = validateSelection(selection, context, jobRunId); + return { version: 1, approvals: [{ workflowSha: context.workflowSha, targetSha, + runId: context.runId, attempt: '1', executionIds: selected.map(row => row.executionId), + protocol: 'paperclip.manual-private-evidence/v1', launchSha256 }] }; +} diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs new file mode 100644 index 00000000000..c77d89031a8 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -0,0 +1,151 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { parse } from 'yaml'; +import { admitDispatch, cases, targetSha, validateSelection, evidencePolicy } from './chat-completion-testops-contract.mjs'; +import { actualContext } from './chat-completion-testops-cli.mjs'; + +function fixture() { + const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: 'paperclipai/paperclip', + GITHUB_REF: 'refs/heads/master', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-completion-testops.yml@refs/heads/master', + GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', GITHUB_ACTOR_ID: '42', TESTOPS_ALLOWED_ACTOR_IDS: '[42]' }; + const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, + ref: env.GITHUB_REF, sender: { id: 42 }, inputs: { ALLURE_JOB_RUN_ID: '77' } }; + const context = admitDispatch(env, event, 42); + const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, + allureJobRunId: '77', planId: 10, jobId: 11, launchId: 12, cases: cases.map((row, index) => ({ ...row, caseId: 100 + index })) }; + return { env, event, context, selection }; +} + +test('actual TestOps selection can execute the three exact cases or a genuine subset', () => { + const f = fixture(); assert.equal(validateSelection(f.selection, f.context, '77').length, 3); + f.selection.cases = [f.selection.cases[2]]; + const policy = evidencePolicy(f.selection, f.context, '77'); + assert.deepEqual(policy.approvals[0].executionIds, [cases[2].executionId]); + assert.equal(policy.approvals[0].targetSha, targetSha); +}); + +for (const kind of ['foreign-workflow', 'candidate-ref', 'replay', 'unlisted-trigger', 'foreign-repo', 'extra-source-input', 'missing-jobrun']) { + test(`rejects ${kind} before any service or provider credential`, () => { + const f = fixture(); let trigger = 42; + if (kind === 'foreign-workflow') f.env.GITHUB_WORKFLOW_REF = f.env.GITHUB_WORKFLOW_REF.replace('chat-completion', 'chat-feature'); + if (kind === 'candidate-ref') f.env.GITHUB_REF = 'refs/heads/codex/candidate'; + if (kind === 'replay') f.env.GITHUB_RUN_ATTEMPT = '2'; + if (kind === 'unlisted-trigger') trigger = 99; + if (kind === 'foreign-repo') f.event.repository.full_name = 'foreign/paperclip'; + if (kind === 'extra-source-input') f.event.inputs.targetSha = 'b'.repeat(40); + if (kind === 'missing-jobrun') delete f.event.inputs.ALLURE_JOB_RUN_ID; + assert.throws(() => admitDispatch(f.env, f.event, trigger)); + }); +} + +for (const kind of ['foreign-case', 'duplicate', 'wrong-fullname', 'foreign-source', 'wrong-run', 'wrong-jobrun', 'duplicate-case-id']) { + test(`refuses evidence authority for ${kind}`, () => { + const f = fixture(); + if (kind === 'foreign-case') f.selection.cases[0].executionId = 'chat-features.runner-opencode.local.tasks.delete'; + if (kind === 'duplicate') f.selection.cases.push(f.selection.cases[0]); + if (kind === 'wrong-fullname') f.selection.cases[0].fullName = cases[1].fullName; + if (kind === 'foreign-source') f.selection.targetSha = 'b'.repeat(40); + if (kind === 'wrong-run') f.selection.runId = '456'; + if (kind === 'wrong-jobrun') f.selection.allureJobRunId = '78'; + if (kind === 'duplicate-case-id') f.selection.cases[1].caseId = f.selection.cases[0].caseId; + assert.throws(() => evidencePolicy(f.selection, f.context, '77')); + }); +} + +test('bootstrap and the fenced plan launcher cannot carry a TestOps execution identity', () => { + const f = fixture(); f.event.inputs = { operation: 'bootstrap' }; + assert.equal(admitDispatch(f.env, f.event, 42).operation, 'bootstrap'); + f.event.inputs.operation = 'launch-plan'; assert.equal(admitDispatch(f.env, f.event, 42).operation, 'launch-plan'); + f.event.inputs.ALLURE_JOB_RUN_ID = '77'; assert.throws(() => admitDispatch(f.env, f.event, 42)); +}); + +for (const value of ['00077', '9007199254740992', '12345678901234567']) { + test(`dispatch rejects unsafe job identity ${value}`, () => { + const f = fixture(); f.event.inputs.ALLURE_JOB_RUN_ID = value; assert.throws(() => admitDispatch(f.env, f.event, 42)); + }); +} +test('dispatch preserves the maximum safe TestOps job identity', () => { + const f = fixture(); f.event.inputs.ALLURE_JOB_RUN_ID = '9007199254740991'; + assert.equal(admitDispatch(f.env, f.event, 42).operation, 'evaluate'); +}); + +test('the executable guard binds the actual original GitHub run before service or provider access', async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-context-'))); + try { + const f = fixture(), eventPath = path.join(root, 'event.json'); await writeFile(eventPath, JSON.stringify(f.event)); + const env = { ...f.env, GITHUB_EVENT_PATH: eventPath, GH_TOKEN: 'synthetic-gh-token', GITHUB_RUN_NUMBER: '5' }; + const run = { id: 123, run_attempt: 1, run_number: 5, workflow_id: 99, head_sha: env.GITHUB_SHA, head_branch: 'master', + event: 'workflow_dispatch', path: '.github/workflows/chat-completion-testops.yml', repository: { full_name: 'paperclipai/paperclip', private: false }, + actor: { id: 42 }, triggering_actor: { id: 42 } }; + let calls = 0; + const exec = async (program, args) => { + calls++; assert.equal(program, 'gh'); assert.deepEqual(args, ['api', 'repos/paperclipai/paperclip/actions/runs/123']); + return { stdout: JSON.stringify(run) }; + }; + assert.equal((await actualContext(env, exec)).workflowId, 99); assert.equal(calls, 1); + for (const field of ['head_sha', 'path', 'head_branch', 'run_attempt', 'run_number', 'workflow_id']) { + const other = { ...run, [field]: typeof run[field] === 'number' ? run[field] + 1 : 'wrong' }; + if (field === 'workflow_id') other[field] = 0; + await assert.rejects(actualContext(env, async () => ({ stdout: JSON.stringify(other) }))); + } + await assert.rejects(actualContext({ ...env, GITHUB_RUN_ID: '../other' }, () => assert.fail('no request before numeric admission'))); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +test('the real workflow separates metadata authority, provider execution and result reporting', async () => { + const text = await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8'); + const w = parse(text); + assert.deepEqual(Object.keys(w.on.workflow_dispatch.inputs).sort(), ['ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'operation']); + assert.equal(w.concurrency['cancel-in-progress'], false); + assert.equal(w.jobs.chat.strategy['max-parallel'], 1); assert.equal(w.jobs.chat.strategy['fail-fast'], false); + assert.equal(w.jobs.select.environment, 'runner-e2e-testops-reporting'); + assert.equal(w.jobs.chat.environment, 'runner-e2e-private-paid'); + assert.equal(w.jobs.report.environment, 'runner-e2e-private-history'); + assert.equal(w.jobs.upload.environment, 'runner-e2e-testops-reporting'); + for (const [name, job] of Object.entries(w.jobs)) { + assert.ok(job.steps.slice(0, 3).some(step => step.run?.endsWith('chat-completion-testops-cli.mjs guard')), name); + if (name !== 'chat') assert.ok(!JSON.stringify(job).includes('secrets.OPENROUTER_API_KEY'), name); + if (!['select', 'upload'].includes(name)) assert.ok(!JSON.stringify(job).includes('secrets.ALLURE_TOKEN'), name); + } + assert.ok(w.jobs.upload.if.includes("needs.report.result == 'success'")); + assert.ok(w.jobs.report.needs.includes('build')); + assert.ok(w.jobs.report.if.includes("needs.build.result == 'success'")); + assert.ok(w.jobs.report.if.includes("needs.chat.result == 'failure'")); + assert.ok(!text.includes('<<:')); assert.ok(!text.includes('target_branch:')); + assert.equal(w.jobs.build.steps.find(step => step.name?.startsWith('Resolve the fixed')).run.includes('--ignore-scripts'), true); + const paid = w.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); + assert.equal(paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, 'v1'); + assert.equal(paid.env.ALLURE_JOB_RUN_ID, '${{ inputs.ALLURE_JOB_RUN_ID }}'); + assert.ok(paid.run.includes('--max-automatic-retries 0')); + assert.ok(w.jobs.build.steps.some(step => step.run?.includes('chat-completion-testops-host-control.mjs'))); + assert.ok(w.jobs.chat.steps.indexOf(paid) > w.jobs.chat.steps.findIndex(step => step.name?.startsWith('Verify source-bound'))); +}); + +test('target checkout cannot populate or overwrite the trusted helper path', async () => { + const definition = parse(await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8')); + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); + try { + const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); + git('init'); await writeFile(path.join(root, 'fixture.txt'), 'normal target'); git('add', '.'); + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'normal target'); + for (const job of [definition.jobs.build, definition.jobs.chat]) { + const target = job.steps.findIndex(step => step.with?.ref === '${{ needs.authorize.outputs.target_sha }}'); + const boundary = job.steps.findIndex(step => step.id === 'target_boundary'); + const trusted = job.steps.findIndex(step => step.with?.path === '.testops-trusted'); + assert.ok(target >= 0 && boundary > target && trusted > boundary, 'Reject reserved target paths, then check out trusted main last'); + assert.notEqual(job.steps[target].with.clean, false); + execFileSync('bash', ['-euo', 'pipefail', '-c', job.steps[boundary].run], { cwd: root, stdio: 'pipe' }); + } + await mkdir(path.join(root, '.testops-trusted')); + await writeFile(path.join(root, '.testops-trusted', 'helper.js'), 'candidate-controlled helper'); git('add', '.testops-trusted'); + git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'reserved path in target'); + for (const job of [definition.jobs.build, definition.jobs.chat]) { + const guard = job.steps.find(step => step.id === 'target_boundary'); + assert.throws(() => execFileSync('bash', ['-euo', 'pipefail', '-c', guard.run], { cwd: root, stdio: 'pipe' })); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/chat-completion-testops-export.test.ts b/tests/runner-e2e/chat-completion-testops-export.test.ts new file mode 100644 index 00000000000..fbf86aea432 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-export.test.ts @@ -0,0 +1,143 @@ +import { mkdtemp, mkdir, readFile, readdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { exportCompletionPilot } from "./chat-completion-testops-export.js"; +import { exportPilotAllure } from "./chat-completion-testops-results.mjs"; +import { cases, targetSha, evidencePolicy, hash } from "./chat-completion-testops-contract.mjs"; +import { authorizeManualPrivateEvidence } from "./manual-private-retention.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/l9sAAAAASUVORK5CYII=", "base64"); +const binding = { runId: "123456", attempt: "1", workflowSha: "a".repeat(40), targetSha }; +async function fixture() { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "chat-completion-export-"))); roots.push(root); + const report = path.join(root, "report"), output = path.join(root, "projection"), allure = path.join(root, "allure"); + const results = cases.map(row => ({ + schema: "paperclip.runner-e2e.result/v2", executionId: row.executionId, suiteId: "chat-features", suiteDefinitionHash: "c".repeat(64), + source: { sha: targetSha, ref: "refs/heads/codex/chat-catalog-master-integration", workflowRunUrl: `https://github.com/paperclipai/paperclip/actions/runs/${binding.runId}` }, + attempt: 1, status: "passed", cleanup: "passed", evidenceValid: true, + runIds: Array.from({ length: row.maximumRuns }, (_, i) => `synthetic-${row.key}-${i}`), + profileId: "runner-opencode", environmentId: "local", caseId: row.scenario, provider: "opencode", + model: "openrouter/deepseek/deepseek-v4.1-flash-1017", runtimeMode: "native", + startedAt: "2026-11-01T00:00:00Z", finishedAt: "2026-11-01T00:00:01Z", durationMs: 1000, + matcherResults: Array.from({ length: row.checks }, (_, i) => ({ matcher: { kind: "json_path", path: `fixture.check-${i}`, expected: true }, passed: true, detail: "OPAQUE_PROVIDER_BODY" })), + screenshots: [], usage: { hiddenReasoning: "PRIVATE_REASONING_MUST_NOT_EXPORT" }, unknownField: "PRIVATE_PROVIDER_EXTRA", + })); + for (const row of cases) { + const file = path.join(report, "evidence", row.executionId, "attempt-1", "final-state.png"); + await mkdir(path.dirname(file), { recursive: true }); await writeFile(file, png); + } + const campaign = { schema: "paperclip.runner-e2e.campaign/v2", campaignId: `gha-${binding.runId}-1`, + generatedAt: results[0]!.finishedAt, expected: cases.map(row => row.executionId), results, retries: 0 }; + const selected = { schema: "paperclip.chat-completion-testops-selection/v1", ...binding, projectId: 1, + planId: 10, jobId: 11, launchId: 12, allureJobRunId: "77", cases: cases.map((row, i) => ({ ...row, caseId: 100 + i })) }; + const save = () => writeFile(path.join(report, "normalized-results.json"), JSON.stringify(campaign)); + await save(); return { root, report, output, allure, campaign, results, selected, save }; +} + +it("converts three original cases into three exact TestOps outcomes with all 35 checks and private-evidence receipts", async () => { + const f = await fixture(); await writeFile(path.join(f.report, "private-provider.log"), "PRIVATE_LOG_MUST_NOT_EXPORT"); + await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + const projected = await readFile(path.join(f.output, "normalized-results.json"), "utf8"); + expect(projected).not.toMatch(/PRIVATE_|OPAQUE_PROVIDER_BODY|publication/); + expect((await readdir(f.output)).sort()).toEqual(["binding.json", "junit.xml", "normalized-results.json"]); + const receipt = JSON.parse(await readFile(path.join(f.output, "binding.json"), "utf8")); + expect(receipt.privateEvidence.every((row: { finalScreenshot: { sha256: string } }) => /^[a-f0-9]{64}$/.test(row.finalScreenshot.sha256))).toBe(true); + const exported = await exportPilotAllure(f.output, f.allure, binding, f.selected); + expect(exported.map(row => row.caseId)).toEqual([100, 101, 102]); + expect(exported.every(row => row.status === "passed" && row.attachments.length === 2)).toBe(true); + const names = (await readdir(f.allure)).filter(name => name.endsWith("-result.json")); expect(names).toHaveLength(3); + const rendered = await Promise.all(names.map(async name => JSON.parse(await readFile(path.join(f.allure, name), "utf8")))); + expect(rendered.reduce((sum, row) => sum + row.steps.length, 0)).toBe(35); + expect(rendered.map(row => row.fullName).sort()).toEqual(cases.map(row => row.fullName).sort()); + expect(rendered.every(row => row.start === Date.parse(f.results[0]!.startedAt))).toBe(true); + expect(rendered.flatMap(row => row.labels.filter((label: { name: string }) => label.name === "AS_ID").map((label: { value: string }) => label.value)).sort()).toEqual(["100", "101", "102"]); +}); + +it("uses the unchanged evidence authority owner for the authenticated exact selection", async () => { + const f = await fixture(); + const policy = evidencePolicy(f.selected, binding, "77"); + const identity = { ...binding, executionIds: f.campaign.expected }; + const admitted = authorizeManualPrivateEvidence(identity, JSON.stringify(policy)); + expect(admitted).toMatchObject({ targetSha, executionIds: f.campaign.expected, protocol: "paperclip.manual-private-evidence/v1" }); + expect(() => authorizeManualPrivateEvidence({ ...identity, executionIds: ["chat-features.runner-opencode.local.foreign"] }, JSON.stringify(policy))).toThrow(); + expect(() => authorizeManualPrivateEvidence({ ...identity, targetSha: "b".repeat(40) }, JSON.stringify(policy))).toThrow(); +}); + +it.each([1, 2, 3])("accepts a genuine project-work pass with %i declared provider runs through both export gates", async count => { + const f = await fixture(); + f.results[2]!.runIds = Array.from({ length: count }, (_, i) => `project-work-${i}`); await f.save(); + await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + expect((await exportPilotAllure(f.output, f.allure, binding, f.selected))[2]!.status).toBe("passed"); +}); + +it.each([0, 4])("rejects project-work outside its declared range: %i runs", async count => { + const f = await fixture(); + f.results[2]!.runIds = Array.from({ length: count }, (_, i) => `project-work-${i}`); await f.save(); + await expect(exportCompletionPilot(f.report, f.output, binding, f.campaign.expected)).rejects.toThrow(); +}); + +it.each([0, 4])("the upload consumer independently rejects %i project-work runs", async count => { + const f = await fixture(); await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + const file = path.join(f.output, "normalized-results.json"); + const campaign = JSON.parse(await readFile(file, "utf8")); + campaign.results[2].runIds = Array.from({ length: count }, (_, i) => `project-work-${i}`); + const bytes = Buffer.from(JSON.stringify(campaign)); await writeFile(file, bytes); + const receiptFile = path.join(f.output, "binding.json"), receipt = JSON.parse(await readFile(receiptFile, "utf8")); + Object.assign(receipt.files.find((row: { path: string }) => row.path === "normalized-results.json"), { bytes: bytes.length, sha256: hash(bytes) }); + await writeFile(receiptFile, JSON.stringify(receipt)); + await expect(exportPilotAllure(f.output, f.allure, binding, f.selected)).rejects.toThrow(); +}); + +it("retains failed grading and failed cleanup as a failed TestOps result", async () => { + const f = await fixture(); f.results[1]!.status = "failed"; f.results[1]!.cleanup = "failed"; + f.results[1]!.matcherResults[0]!.passed = false; await f.save(); + await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + const exported = await exportPilotAllure(f.output, f.allure, binding, f.selected); + expect(exported.map(row => row.status)).toEqual(["passed", "failed", "passed"]); + expect(await readFile(path.join(f.output, "junit.xml"), "utf8")).toContain('failures="1"'); +}); + +it("records a failed case with unavailable private evidence without inventing an image", async () => { + const f = await fixture(); f.results[0]!.status = "failed"; f.results[0]!.evidenceValid = false; + await rm(path.join(f.report, "evidence", f.results[0]!.executionId, "attempt-1", "final-state.png")); + await f.save(); await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + const receipt = JSON.parse(await readFile(path.join(f.output, "binding.json"), "utf8")); + expect(receipt.privateEvidence[0]).toMatchObject({ requiredEvidenceComplete: false, finalScreenshot: null }); + expect((await exportPilotAllure(f.output, f.allure, binding, f.selected))[0]!.status).toBe("failed"); +}); + +it.each(["unexpected.png", "media"])("rejects private media added to the public artifact: %s", async name => { + const f = await fixture(); await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + if (name === "media") await mkdir(path.join(f.output, name)); + else await writeFile(path.join(f.output, name), png); + await expect(exportPilotAllure(f.output, f.allure, binding, f.selected)).rejects.toThrow(); +}); + +it.each(["wrong-source", "wrong-workflow-run", "wrong-model", "retry", "missing-check", "failed-check", "cleanup", "run-count", "missing-image", "symlink"])("rejects %s before exporting a pass", async kind => { + const f = await fixture(), first = f.results[0]!; + if (kind === "wrong-source") first.source.sha = "b".repeat(40); + if (kind === "wrong-workflow-run") first.source.workflowRunUrl += "9"; + if (kind === "wrong-model") first.model = "other/model"; + if (kind === "retry") first.attempt = 2; + if (kind === "missing-check") first.matcherResults.pop(); + if (kind === "failed-check") first.matcherResults[0]!.passed = false; + if (kind === "cleanup") first.cleanup = "failed"; + if (kind === "run-count") first.runIds.push("unexpected-run"); + const file = path.join(f.report, "evidence", first.executionId, "attempt-1", "final-state.png"); + if (kind === "missing-image") await rm(file); + if (kind === "symlink") { await rm(file); const other = path.join(f.root, "unowned.png"); await writeFile(other, png); await symlink(other, file); } + await f.save(); await expect(exportCompletionPilot(f.report, f.output, binding, f.campaign.expected)).rejects.toThrow(); + await expect(readFile(path.join(f.output, "binding.json"))).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("refuses foreign selection and corrupt uploaded projection bytes", async () => { + const f = await fixture(); await exportCompletionPilot(f.report, f.output, binding, f.campaign.expected); + f.selected.cases[0]!.caseId = 999; f.selected.cases[0]!.fullName = "foreign.Case"; + await expect(exportPilotAllure(f.output, f.allure, binding, f.selected)).rejects.toThrow(); + f.selected.cases[0] = { ...cases[0]!, caseId: 100 }; + await writeFile(path.join(f.output, "normalized-results.json"), "{}"); + await expect(exportPilotAllure(f.output, f.allure, binding, f.selected)).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/chat-completion-testops-export.ts b/tests/runner-e2e/chat-completion-testops-export.ts new file mode 100644 index 00000000000..c3ddb3f80d3 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-export.ts @@ -0,0 +1,107 @@ +import { lstat, mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { buildRunnerCampaign } from "./history.js"; +import { assertSecretFree, findSecretLeakInJsonValues } from "./redaction.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import { publicBilling } from "./testops-chat-export.js"; +import { cases, targetSha, model, hash, validRunCount } from "./chat-completion-testops-contract.mjs"; +import type { RunnerE2EResult } from "./types.js"; + +const pngMagic = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +interface Binding { runId: string; attempt: string; workflowSha: string; targetSha: string } + +async function ownedRead(root: string, relative: string, limit: number) { + if (!path.isAbsolute(root) || await realpath(root) !== root || relative.split("/").some(part => !/^[A-Za-z0-9._-]+$/.test(part) || [".", ".."].includes(part))) throw new Error("Invalid pilot evidence path"); + let current = root; + const rootStat = await lstat(root); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error("Invalid pilot evidence root"); + const parts = relative.split("/"); + for (const [index, part] of parts.entries()) { + current = path.join(current, part); const stat = await lstat(current); + if (stat.isSymbolicLink() || (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile() || stat.nlink !== 1 || stat.size > limit)) throw new Error("Invalid pilot evidence file"); + } + const before = await lstat(current), bytes = await readFile(current), after = await lstat(current); + if (bytes.length > limit || before.ino !== after.ino || before.dev !== after.dev || before.size !== after.size + || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("Pilot evidence changed"); + return bytes; +} + +/** Public-safe metadata projection for the protected TestOps reporter. + * Original screenshots stay in private AWS storage. The exact final PNG is + * verified here, but only its checksum and availability receipt cross jobs. + */ +export async function exportCompletionPilot(root: string, output: string, binding: Binding, expectedIds: string[]) { + if (!/^[1-9][0-9]{0,15}$/.test(binding.runId) || binding.attempt !== "1" || !/^[a-f0-9]{40}$/.test(binding.workflowSha) + || binding.targetSha !== targetSha || !Array.isArray(expectedIds) || !expectedIds.length || expectedIds.length > 3 + || new Set(expectedIds).size !== expectedIds.length || expectedIds.some(id => !cases.some(row => row.executionId === id))) throw new Error("Invalid pilot binding"); + const campaignId = `gha-${binding.runId}-1`, runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${binding.runId}`; + const raw = JSON.parse((await ownedRead(root, "normalized-results.json", 8 * 1024 * 1024)).toString("utf8")); + if (raw.schema !== "paperclip.runner-e2e.campaign/v2" || raw.campaignId !== campaignId || raw.retries !== 0 + || !Array.isArray(raw.expected) || JSON.stringify([...raw.expected].sort()) !== JSON.stringify([...expectedIds].sort()) + || !Array.isArray(raw.results) || raw.results.length !== expectedIds.length || new Set(raw.results.map((row: RunnerE2EResult) => row.executionId)).size !== expectedIds.length) throw new Error("Foreign pilot campaign"); + const files = new Map(), results: RunnerE2EResult[] = []; + const privateEvidence: { executionId: string; requiredEvidenceComplete: boolean | null; finalScreenshot: { sha256: string; bytes: number } | null }[] = []; + for (const id of expectedIds) { + const descriptor = cases.find(row => row.executionId === id)!; + const original = raw.results.find((row: RunnerE2EResult) => row.executionId === id) as RunnerE2EResult & { evidenceValid?: boolean }; + if (!original) throw new Error("Missing selected pilot result"); + validateRetainedRunnerResult(original); + if (original.suiteId !== "chat-features" || original.profileId !== "runner-opencode" || original.environmentId !== "local" + || original.caseId !== descriptor.scenario || original.provider !== "opencode" || original.model !== model || original.runtimeMode !== "native" + || original.source?.sha !== targetSha || original.source?.workflowRunUrl !== runUrl || original.attempt !== 1 + || !["passed", "failed"].includes(original.status)) throw new Error("Foreign pilot result"); + const checks = original.matcherResults ?? []; + if (checks.length > 200 || checks.some(check => check.matcher.kind !== "json_path" || !/^[A-Za-z0-9._-]{1,150}$/.test(check.matcher.path) + || check.matcher.expected !== true || typeof check.passed !== "boolean")) throw new Error("Invalid pilot checks"); + if (original.status === "passed" && (original.cleanup !== "passed" || original.evidenceValid !== true + || checks.length !== descriptor.checks || checks.some(check => !check.passed) || !validRunCount(original.runIds, descriptor))) throw new Error("Incomplete pilot pass"); + let image: Buffer | undefined; + const relative = `evidence/${id}/attempt-1/final-state.png`; + try { image = await ownedRead(root, relative, 12 * 1024 * 1024); } + catch (error) { if (original.status === "passed" || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (image) { + if (!image.subarray(0, 8).equals(pngMagic) || image.subarray(12, 16).toString("ascii") !== "IHDR") throw new Error("Invalid pilot fixture PNG"); + assertSecretFree(image, [], "Private synthetic pilot PNG", { includeShapes: true }); + } + privateEvidence.push({ executionId: id, requiredEvidenceComplete: typeof original.evidenceValid === "boolean" ? original.evidenceValid : null, + finalScreenshot: image ? { sha256: hash(image), bytes: image.length } : null }); + const result: RunnerE2EResult = { + schema: original.schema, executionId: id, suiteId: "chat-features", suiteDefinitionHash: original.suiteDefinitionHash, + source: { sha: targetSha, ref: null, workflowRunUrl: runUrl }, attempt: 1, status: original.status, + ...(original.failureClass ? { failureClass: original.failureClass } : {}), + ...(original.status === "failed" ? { error: "Original pilot evaluation failed; diagnostics remain in protected evidence" } : {}), + profileId: "runner-opencode", environmentId: "local", caseId: descriptor.scenario, provider: "opencode", model, runtimeMode: "native", + runIds: original.runIds?.map(value => { if (!/^[A-Za-z0-9._-]{1,100}$/.test(value)) throw new Error("Invalid pilot run identity"); return value; }), + startedAt: original.startedAt, finishedAt: original.finishedAt, durationMs: original.durationMs, cleanup: original.cleanup, + ...(original.billing ? { billing: publicBilling(original.billing) } : {}), + matcherResults: checks.map(check => ({ matcher: { kind: "json_path", path: (check.matcher as { path: string }).path, expected: true }, + passed: check.passed, detail: "Retained independent Product E2E oracle check" })), + screenshots: [], + }; + validateRetainedRunnerResult(result); results.push(result); + } + const campaign = buildRunnerCampaign({ campaignId, generatedAt: raw.generatedAt, expected: expectedIds, results }); + if (findSecretLeakInJsonValues(campaign, [], { includeShapes: true })) throw new Error("Unsafe pilot projection"); + files.set("normalized-results.json", Buffer.from(`${JSON.stringify(campaign, null, 2)}\n`)); + const failed = results.filter(row => row.status !== "passed" || row.cleanup !== "passed").length; + files.set("junit.xml", Buffer.from(`${results.map(row => `${row.status !== "passed" || row.cleanup !== "passed" ? '' : ""}`).join("")}\n`)); + const manifest = [...files].sort(([a], [b]) => a.localeCompare(b)).map(([name, bytes]) => ({ path: name, sha256: hash(bytes), bytes: bytes.length })); + if (manifest.reduce((total, row) => total + row.bytes, 0) > 32 * 1024 * 1024) throw new Error("Pilot projection exceeds allowance"); + const receipt = { schema: "paperclip.chat-completion-testops-export/v1", ...binding, campaignId, executionIds: expectedIds, privateEvidence, files: manifest }; + await mkdir(output, { mode: 0o700 }); + for (const [relative, bytes] of files) { + const destination = path.join(output, relative); await mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + await writeFile(destination, bytes, { flag: "wx", mode: 0o600 }); + } + await writeFile(path.join(output, "binding.json"), `${JSON.stringify(receipt, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + return receipt; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + exportCompletionPilot(path.resolve(process.env.TESTOPS_PILOT_REPORT_ROOT ?? "runner-e2e-merged-report/normalized"), + path.resolve(process.env.TESTOPS_PILOT_EXPORT_ROOT ?? "runner-e2e-testops"), { + runId: process.env.GITHUB_RUN_ID ?? "", attempt: process.env.GITHUB_RUN_ATTEMPT ?? "", workflowSha: process.env.GITHUB_SHA ?? "", targetSha, + }, JSON.parse(process.env.MANUAL_PRIVATE_IDS ?? "null")) + .catch(() => { process.stderr.write("TestOps pilot projection rejected; originals remain protected\n"); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/chat-completion-testops-host-control.mjs b/tests/runner-e2e/chat-completion-testops-host-control.mjs new file mode 100644 index 00000000000..bf0a169c424 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-host-control.mjs @@ -0,0 +1,60 @@ +import { execFileSync } from 'node:child_process'; +import { mkdtemp, readFile, realpath, rm } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { parse } from 'yaml'; +import { cases, model, targetSha, workflow } from './chat-completion-testops-contract.mjs'; + +/** A credential-free caller/source control with synthetic IMDS responses. + * It does not qualify an actual AWS host or make a provider call. + */ +export async function verifyHostContract(workspace, temporaryRoot, env = process.env) { + if (env.ALLURE_TOKEN || env.OPENROUTER_API_KEY || !path.isAbsolute(workspace)) throw new Error('Credential-free source control required'); + const source = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: workspace, encoding: 'utf8' }).trim(); + if (source !== targetSha) throw new Error('Source control pin mismatch'); + const definition = parse(await readFile(new URL(`../../${workflow}`, import.meta.url), 'utf8')); + const paid = definition.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); + if (paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS !== 'v1' || paid.env.ALLURE_JOB_RUN_ID !== '${{ inputs.ALLURE_JOB_RUN_ID }}' + || !paid.run.includes('--max-automatic-retries 0') || definition.jobs.chat.strategy['max-parallel'] !== 1) throw new Error('Workflow host contract mismatch'); + const { runnerMatrix } = await import(pathToFileURL(path.join(workspace, 'tests/runner-e2e/catalog.ts')).href); + const { prepareChatRemoteHost } = await import(pathToFileURL(path.join(workspace, 'tests/runner-e2e/chat-remote-host.ts')).href); + const root = await realpath(await mkdtemp(path.join(temporaryRoot, 'chat-completion-host-control-'))); + try { + for (const descriptor of cases) { + const execution = runnerMatrix.find(row => row.id === descriptor.executionId); + if (!execution || execution.task.id !== descriptor.scenario || execution.profile.model !== model + || execution.profile.id !== 'runner-opencode' || execution.environment.id !== 'local' + || execution.task.expectedRunCount !== descriptor.maximumRuns + || (execution.task.minimumExpectedRunCount ?? execution.task.expectedRunCount) !== descriptor.minimumRuns) throw new Error('Pinned source selection mismatch'); + const controlEnv = { GITHUB_ACTIONS: 'true', GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', + GITHUB_WORKFLOW_REF: `paperclipai/paperclip/${workflow}@refs/heads/master`, GITHUB_RUN_ID: '1', GITHUB_RUN_ATTEMPT: '1', + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: targetSha, PAPERCLIP_CHAT_COMPLETION_TESTOPS: paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, + ALLURE_JOB_RUN_ID: '1' }; + let calls = 0; + const dependencies = { platform: 'linux', arch: 'x64', sourceRevision: async () => source, + fetch: async url => { + calls++; + if (String(url) === 'http://169.254.169.254/latest/api/token') return new Response('synthetic-imds-control'); + if (String(url) === 'http://169.254.169.254/latest/meta-data/instance-life-cycle') return new Response('spot'); + if (String(url) === 'http://169.254.169.254/latest/dynamic/instance-identity/document') return Response.json({ + instanceId: 'i-1234567890abcdef0', imageId: 'ami-1234567890abcdef0', region: 'us-east-1', architecture: 'x86_64' }); + throw new Error('Unexpected control request'); + } }; + const input = { executions: [execution], env: controlEnv, repositoryRoot: workspace, outputDirectory: root, dependencies }; + await prepareChatRemoteHost(input); + const receipt = JSON.parse(await readFile(path.join(root, 'chat-remote-host.json'), 'utf8')); + if (calls !== 3 || receipt.workflowRef !== controlEnv.GITHUB_WORKFLOW_REF || receipt.testOps?.allureJobRunId !== '1' + || receipt.testOps.admission !== 'v1' || receipt.evaluationAdmitted !== false) throw new Error('Host audit contract mismatch'); + let rejected = false; + try { await prepareChatRemoteHost({ ...input, executions: [{ ...execution, task: { ...execution.task, id: 'foreign' } }] }); } + catch { rejected = true; } + if (!rejected || calls !== 3) throw new Error('Host task identity contract mismatch'); + } + return { fixtureOnly: true, liveHostQualified: false, providerCalls: 0, source, cases: cases.length }; + } finally { await rm(root, { recursive: true, force: true }); } +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + verifyHostContract(process.env.GITHUB_WORKSPACE, process.env.RUNNER_TEMP) + .then(value => process.stdout.write(`${JSON.stringify(value)}\n`)) + .catch(() => { process.stderr.write('Workflow/source host control failed before provider access\n'); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/chat-completion-testops-results.d.mts b/tests/runner-e2e/chat-completion-testops-results.d.mts new file mode 100644 index 00000000000..fa0e560c29a --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-results.d.mts @@ -0,0 +1,17 @@ +import type { RunnerE2EResult } from "./types.js"; +import type { PilotCase } from "./chat-completion-testops-contract.mjs"; +interface Context { runId: string; attempt: string; workflowSha: string; targetSha: string } +interface Selection extends Context { + schema: string; projectId: number; planId: number; jobId: number; launchId: number; + allureJobRunId: string; cases: (PilotCase & { caseId: number })[]; +} +export interface PilotOutcome { + caseId: number; executionId: string; status: "passed" | "failed"; cleanup: string; + attachments: { name: string; contentType: string; contentLength: number }[]; +} +export function readPilotExport(root: string, context: Context, selection: Selection): Promise<{ + campaign: { campaignId: string; results: RunnerE2EResult[] }; + selected: (PilotCase & { caseId: number })[]; retained: Map; + privateEvidence: { executionId: string; requiredEvidenceComplete: boolean | null; finalScreenshot: { sha256: string; bytes: number } | null }[]; +}>; +export function exportPilotAllure(root: string, output: string, context: Context, selection: Selection): Promise; diff --git a/tests/runner-e2e/chat-completion-testops-results.mjs b/tests/runner-e2e/chat-completion-testops-results.mjs new file mode 100644 index 00000000000..426f1928773 --- /dev/null +++ b/tests/runner-e2e/chat-completion-testops-results.mjs @@ -0,0 +1,125 @@ +import { randomUUID } from 'node:crypto'; +import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { targetSha, model, hash, validateSelection, validRunCount, reject } from './chat-completion-testops-contract.mjs'; + +async function inventory(root, relative = '') { + const found = []; const entries = await readdir(path.join(root, relative)); + if (entries.length > 12) reject('export_inventory'); + for (const name of entries) { + if (!/^[A-Za-z0-9._-]+$/.test(name) || ['.', '..'].includes(name)) reject('export_path'); + const file = relative ? `${relative}/${name}` : name, stat = await lstat(path.join(root, file)); + if (stat.isSymbolicLink() || stat.nlink !== 1 && !stat.isDirectory()) reject('export_path'); + if (stat.isDirectory()) { + reject('export_path'); + } else if (stat.isFile() && stat.size <= 12 * 1024 * 1024) found.push(file); + else reject('export_path'); + if (found.length > 8) reject('export_inventory'); + } + return found; +} + +export async function readPilotExport(root, context, selection) { + const selected = validateSelection(selection, context, selection.allureJobRunId); + const stat = await lstat(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(root) !== path.resolve(root)) reject('export_path'); + const files = await inventory(root); + if (!files.includes('binding.json')) reject('export_binding'); + const bindingBytes = await readFile(path.join(root, 'binding.json')); + if (bindingBytes.length > 32_768) reject('export_binding'); + const binding = JSON.parse(bindingBytes); + if (binding.schema !== 'paperclip.chat-completion-testops-export/v1' || binding.runId !== context.runId || binding.attempt !== '1' + || binding.workflowSha !== context.workflowSha || binding.targetSha !== targetSha || binding.campaignId !== `gha-${context.runId}-1` + || JSON.stringify(binding.executionIds) !== JSON.stringify(selected.map(row => row.executionId)) || !Array.isArray(binding.files) + || binding.files.length !== 2 || !Array.isArray(binding.privateEvidence) || binding.privateEvidence.length !== selected.length + || new Set(binding.privateEvidence.map(row => row.executionId)).size !== selected.length) reject('export_binding'); + const names = binding.files.map(row => row.path); + const allowed = ['normalized-results.json', 'junit.xml']; + if (new Set(names).size !== names.length || names.some(name => !allowed.includes(name)) || !names.includes('normalized-results.json') + || !names.includes('junit.xml') || JSON.stringify([...names, 'binding.json'].sort()) !== JSON.stringify(files.sort())) reject('export_inventory'); + const retained = new Map(); let total = 0; + for (const row of binding.files) { + if (!/^[a-f0-9]{64}$/.test(row.sha256 ?? '') || !Number.isSafeInteger(row.bytes) || row.bytes < 0) reject('export_inventory'); + const before = await lstat(path.join(root, row.path)), bytes = await readFile(path.join(root, row.path)), after = await lstat(path.join(root, row.path)); + total += bytes.length; + if (total > 32 * 1024 * 1024 || bytes.length !== row.bytes || hash(bytes) !== row.sha256 || before.ino !== after.ino + || before.dev !== after.dev || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) reject('export_changed'); + retained.set(row.path, bytes); + } + const normalized = retained.get('normalized-results.json'); + if (normalized.length > 2_000_000) reject('export_inventory'); + const campaign = JSON.parse(normalized); + if (campaign.schema !== 'paperclip.runner-e2e.campaign/v2' || campaign.campaignId !== binding.campaignId || campaign.retries !== 0 + || !Array.isArray(campaign.results) || campaign.results.length !== selected.length + || new Set(campaign.results.map(row => row.executionId)).size !== selected.length) reject('export_results'); + for (const row of campaign.results) { + const expected = selected.find(value => value.executionId === row.executionId); + if (!expected || row.suiteId !== 'chat-features' || row.caseId !== expected.scenario || row.profileId !== 'runner-opencode' + || row.provider !== 'opencode' || row.environmentId !== 'local' || row.model !== model || row.runtimeMode !== 'native' + || row.attempt !== 1 || row.source?.sha !== targetSha || row.source?.workflowRunUrl !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` + || !['passed', 'failed'].includes(row.status) || !['passed', 'failed', 'not_started'].includes(row.cleanup) + || !Array.isArray(row.matcherResults) || row.matcherResults.length > 200 || !Array.isArray(row.screenshots) || row.screenshots.length !== 0) reject('export_result'); + if (row.matcherResults.some(check => check.matcher?.kind !== 'json_path' || !/^[A-Za-z0-9._-]{1,150}$/.test(check.matcher.path) + || check.matcher.expected !== true || typeof check.passed !== 'boolean')) reject('export_checks'); + const evidence = binding.privateEvidence.find(value => value.executionId === row.executionId); + if (!evidence || ![true, false, null].includes(evidence.requiredEvidenceComplete) + || evidence.finalScreenshot !== null && (!/^[a-f0-9]{64}$/.test(evidence.finalScreenshot?.sha256 ?? '') + || !Number.isSafeInteger(evidence.finalScreenshot.bytes) || evidence.finalScreenshot.bytes < 24 || evidence.finalScreenshot.bytes > 12 * 1024 * 1024)) reject('private_evidence_receipt'); + if (row.status === 'passed' && (row.cleanup !== 'passed' || row.matcherResults.length !== expected.checks + || row.matcherResults.some(check => !check.passed) || !validRunCount(row.runIds, expected) + || evidence.requiredEvidenceComplete !== true || evidence.finalScreenshot === null)) reject('export_incomplete_pass'); + } + return { campaign, selected, retained, privateEvidence: binding.privateEvidence }; +} + +export async function exportPilotAllure(root, output, context, selection) { + const { campaign, selected, privateEvidence } = await readPilotExport(root, context, selection); + await mkdir(output, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST') throw error; }); + const stat = await lstat(output); + if (!stat.isDirectory() || stat.isSymbolicLink() || process.getuid && stat.uid !== process.getuid() || (await readdir(output)).length) reject('allure_output'); + const exported = []; + for (const selectedCase of selected) { + const result = campaign.results.find(row => row.executionId === selectedCase.executionId); + const status = result.status === 'passed' && result.cleanup === 'passed' ? 'passed' : 'failed'; + const attachments = [], metadata = []; + const attach = async (name, type, bytes, suffix) => { + const source = `${randomUUID()}-attachment.${suffix}`; + await writeFile(path.join(output, source), bytes, { flag: 'wx', mode: 0o600 }); + attachments.push({ name, source, type }); metadata.push({ name, contentType: type, contentLength: bytes.length }); + }; + const numeric = value => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null; + const provenance = { schema: 'paperclip.chat-completion-testops-provenance/v1', campaignId: campaign.campaignId, + runId: context.runId, attempt: 1, workflowSha: context.workflowSha, sourceSha: targetSha, + executionId: selectedCase.executionId, scenario: selectedCase.scenario, status: result.status, cleanup: result.cleanup, + definitionHash: /^[a-f0-9]{64}$/.test(result.suiteDefinitionHash ?? '') ? result.suiteDefinitionHash : null, + model, runtime: 'native', environment: 'local on GitHub Actions / AWS', durationMs: numeric(result.durationMs), + billing: { complete: typeof result.billing?.complete === 'boolean' ? result.billing.complete : null, + reportedCostUsd: numeric(result.billing?.reportedCostUsd), estimatedRuntimeCostUsd: numeric(result.billing?.estimatedRuntimeCostUsd), + observedAndEstimatedCostUsd: numeric(result.billing?.observedAndEstimatedCostUsd) }, + checks: result.matcherResults.map(row => ({ path: row.matcher.path, passed: row.passed })), + note: 'Original case clocks, one attempt and no eval retries. Setup usage and AWS accounting may be incomplete. Private original evidence remains separate.' }; + await attach('Execution provenance and checks', 'application/json', Buffer.from(JSON.stringify(provenance, null, 2)), 'json'); + const evidence = privateEvidence.find(row => row.executionId === result.executionId); + const evidenceReceipt = { schema: 'paperclip.chat-completion-testops-private-evidence/v1', runId: context.runId, attempt: 1, + executionId: result.executionId, ...evidence, storage: 'existing protected Runner E2E history', + archiveKind: `manual-cell-${hash(result.executionId)}`, + note: 'Only this availability receipt and checksum are attached. Original screenshots and diagnostics require authenticated private-history retrieval.' }; + await attach('Private evidence receipt', 'application/json', Buffer.from(JSON.stringify(evidenceReceipt, null, 2)), 'json'); + const value = { uuid: randomUUID(), name: selectedCase.name, fullName: selectedCase.fullName, + testCaseId: hash(`${JSON.stringify(selectedCase.fullName)}\n`), + historyId: hash(JSON.stringify([selectedCase.fullName, 'runner-opencode', model, 'local'])), status, stage: 'finished', + statusDetails: { message: status === 'passed' ? 'Real Product E2E assertions and cleanup passed' : 'Original evaluation or cleanup failed' }, + labels: [{ name: 'AS_ID', value: String(selectedCase.caseId) }, { name: 'parentSuite', value: 'Product E2E' }, + { name: 'suite', value: 'chat-features' }, { name: 'area', value: 'Chat and collaboration' }, + { name: 'framework', value: 'Paperclip Evals' }, { name: 'tag', value: 'chat-completion-testops' }], + parameters: [{ name: 'canonicalScenario', value: selectedCase.scenario }, { name: 'model', value: model }, + { name: 'environment', value: 'GitHub Actions / AWS' }], + links: [{ name: 'GitHub Actions run', type: 'custom', url: `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` }, + { name: 'Evaluated source', type: 'custom', url: `https://github.com/paperclipai/paperclip/commit/${targetSha}` }], + steps: result.matcherResults.map(row => ({ name: row.matcher.path, status: row.passed ? 'passed' : 'failed', stage: 'finished' })), attachments }; + for (const [key, time] of [['start', result.startedAt], ['stop', result.finishedAt]]) if (Number.isFinite(Date.parse(time))) value[key] = Date.parse(time); + await writeFile(path.join(output, `${value.uuid}-result.json`), JSON.stringify(value), { flag: 'wx', mode: 0o600 }); + exported.push({ caseId: selectedCase.caseId, executionId: selectedCase.executionId, status, cleanup: result.cleanup, attachments: metadata }); + } + return exported; +} diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index 0bce376c69b..678ccfa40a4 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -22,7 +22,8 @@ for (const [name, command] of [ ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", - "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs"]], + "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", + "tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs"]], ]) { const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); if (result.error || result.signal) { diff --git a/tests/runner-e2e/testops-chat-export.ts b/tests/runner-e2e/testops-chat-export.ts index cc67cff2e75..f9841f5d2a9 100644 --- a/tests/runner-e2e/testops-chat-export.ts +++ b/tests/runner-e2e/testops-chat-export.ts @@ -15,7 +15,7 @@ const sha = /^[a-f0-9]{40}$/; const pngMagic = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); export interface TestOpsChatBinding { runId: string; attempt: string; workflowSha: string; targetSha: string } -function publicBilling(value: NonNullable) { +export function publicBilling(value: NonNullable) { const pick = (source: object, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(source, key)).map(key => [key, (source as Record)[key]])); const numeric = (source: object, keys: string[]) => { const selected = pick(source, keys); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index b77babcd5a8..d0bc59611ec 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -7,7 +7,7 @@ export default defineConfig({ globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"], + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs"], testTimeout: 30_000, }, }); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 7d18453e59c..91c0e3e9f29 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -606,6 +606,8 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", + "chat-feature-testops.yml", + "chat-completion-testops.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -891,6 +893,24 @@ describe("public repository paid workflow security", () => { for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); + if (["chat-feature-testops.yml", "chat-completion-testops.yml"].includes(filename)) { + const guard = filename === "chat-feature-testops.yml" ? "testops-chat-dispatch.ts" : "chat-completion-testops-cli.mjs guard"; + expect(other).toContain(`run: node tests/runner-e2e/${guard}`); + expect(other).toContain("ref: ${{ github.sha }}"); + expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); + const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; + expect(chat).toContain("environment: runner-e2e-private-paid"); + expect(chat).toContain("needs: [authorize, select, build]"); + expect(chat).toContain(" - *guard"); + expect(chat).toContain("verify-selection"); + expect(chat).not.toContain("ALLURE_TOKEN"); + expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); + const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; + expect(report).toContain("environment: runner-e2e-private-history"); + expect(report).toContain("manual-private-transfer.ts report-session"); + expect(report).not.toContain("ALLURE_TOKEN"); + continue; + } for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8");