diff --git a/.github/workflows/chat-catalog-testops-case.yml b/.github/workflows/chat-catalog-testops-case.yml
new file mode 100644
index 00000000000..38229c9f070
--- /dev/null
+++ b/.github/workflows/chat-catalog-testops-case.yml
@@ -0,0 +1,318 @@
+name: Chat catalog TestOps case
+
+on:
+ workflow_call:
+ inputs:
+ protocol:
+ type: string
+ required: true
+ cell:
+ type: string
+ required: true
+ target_sha:
+ type: string
+ required: true
+ runner:
+ type: string
+ required: true
+ selection_artifact_id:
+ type: string
+ required: true
+ selection_sha256:
+ type: string
+ required: true
+ build_artifact_id:
+ type: string
+ required: true
+ ALLURE_JOB_RUN_ID:
+ type: string
+ required: true
+ secrets:
+ OPENROUTER_API_KEY:
+ required: false
+ ANTHROPIC_API_KEY:
+ required: false
+ ALLURE_TOKEN:
+ required: false
+
+permissions:
+ contents: read
+ actions: read
+
+env:
+ NODE_OPTIONS: ''
+ NODE_PATH: ''
+ AWS_EC2_METADATA_DISABLED: 'true'
+
+jobs:
+ chat:
+ name: Execute the original selected case
+ runs-on: ${{ inputs.runner }}
+ timeout-minutes: ${{ fromJSON(inputs.cell).timeout_minutes }}
+ environment: runner-e2e-private-paid
+ permissions:
+ contents: read
+ actions: read
+ id-token: write
+ env: &retention
+ TARGET_SHA: ${{ inputs.target_sha }}
+ MANUAL_PRIVATE_IDS: ${{ fromJSON(inputs.cell).private_ids }}
+ MANUAL_PRIVATE_APPROVALS_JSON: ${{ fromJSON(inputs.cell).private_policy }}
+ RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
+ steps:
+ - &checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ persist-credentials: false
+ - &node
+ uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+ with:
+ node-version: 24
+ - &guard
+ name: Verify the exact reusable protocol and main-only caller
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ CHAT_TESTOPS_PROTOCOL: ${{ inputs.protocol }}
+ run: |
+ test "$CHAT_TESTOPS_PROTOCOL" = streaming-v1
+ node tests/runner-e2e/chat-completion-testops-cli.mjs guard
+ - &pnpm
+ uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
+ env:
+ NPM_CONFIG_AUDIT: 'false'
+ NPM_CONFIG_FUND: 'false'
+ NPM_CONFIG_UPDATE_NOTIFIER: 'false'
+ with:
+ version: 9.15.4
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ inputs.target_sha }}
+ persist-credentials: false
+ - name: Reject target-owned trusted helper paths
+ id: target_boundary
+ run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ ref: ${{ github.sha }}
+ path: .testops-trusted
+ persist-credentials: false
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ inputs.build_artifact_id }}
+ path: chat-completion-build
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ inputs.selection_artifact_id }}
+ path: chat-completion-testops-selection
+ - name: Verify source-bound build and authenticated selection before credentials
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ run: |
+ set -euo pipefail
+ test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
+ node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
+ (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
+ tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
+ pnpm install --frozen-lockfile --ignore-scripts
+ (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
+ - name: Qualify the pinned provider runtime and preinstalled browser
+ run: |
+ node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
+ node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
+ google-chrome --version
+ pnpm exec playwright install ffmpeg
+ - name: Bind exact private evidence under the authenticated TestOps authority
+ id: private
+ env:
+ TARGET_ROOT: ${{ github.workspace }}
+ SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
+ run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
+ - name: Obtain only this cell's exact-key storage session
+ id: cell_credentials
+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
+ with:
+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ inline-session-policy: ${{ steps.private.outputs.session_policy }}
+ output-env-credentials: false
+ output-credentials: true
+ disable-retry: true
+ action-timeout-s: 60
+ - name: Verify private custody before delivering the model key
+ env: &cell_session
+ SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
+ PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
+ AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
+ - name: Execute only the selected original case with zero retries
+ env:
+ OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
+ ANTHROPIC_API_KEY: ${{ contains(fromJSON(inputs.cell).required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}
+ PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
+ PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
+ PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).executionId }}
+ PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
+ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }}
+ PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
+ PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
+ run: pnpm test:e2e:runner -- --id "${{ fromJSON(inputs.cell).executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
+ - name: Retain original evidence after success or failure
+ if: always() && steps.cell_credentials.outcome == 'success'
+ env: *cell_session
+ run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
+
+ report:
+ name: Retain private originals and build the exact safe TestOps projection
+ needs: chat
+ if: always() && !cancelled() && (needs.chat.result == 'success' || needs.chat.result == 'failure')
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ environment: runner-e2e-private-history
+ permissions:
+ contents: read
+ actions: read
+ id-token: write
+ env: *retention
+ outputs:
+ bundle_id: ${{ steps.bundle.outputs.artifact-id }}
+ steps:
+ - *checkout
+ - *node
+ - *guard
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ inputs.selection_artifact_id }}
+ path: chat-completion-testops-selection
+ - *pnpm
+ - name: Install trusted reporting dependencies with lifecycle scripts disabled
+ run: |
+ node tests/runner-e2e/prepare-report-lock.mjs
+ pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
+ - name: Prepare unchanged exact-key report and cell policies
+ id: session
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
+ - name: Obtain exact first-cell reader
+ id: reader_0
+ uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
+ with:
+ role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
+ aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
+ output-env-credentials: false
+ output-credentials: true
+ disable-retry: true
+ action-timeout-s: 60
+ - name: Download the first exact original
+ env:
+ PRIVATE_REPORT_ROOT: ${{ github.workspace }}
+ SELECTED_CELL_INDEX: '0'
+ AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
+ AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
+ AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
+ AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
+ - name: Verify every selected original and its completeness receipt
+ env:
+ PRIVATE_REPORT_ROOT: ${{ github.workspace }}
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
+ - name: Build the existing report while preserving failed source grades
+ id: normalize
+ continue-on-error: true
+ env:
+ PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
+ PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
+ PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ fromJSON(inputs.cell).private_ids }}
+ PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
+ PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }}
+ PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
+ run: |
+ mkdir -p runner-e2e-merged-report
+ pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
+ - name: Export only exact reviewed synthetic fixture results
+ id: export
+ if: always() && steps.reader_0.outcome == 'success'
+ run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
+ - name: Require a complete safe projection even after a failed normalization
+ if: always()
+ env:
+ EXPORT_OUTCOME: ${{ steps.export.outcome }}
+ run: test "$EXPORT_OUTCOME" = success
+ - name: Freeze one safe Allure bundle before any TestOps transmission
+ id: prepare
+ if: always() && !cancelled() && steps.export.outcome == 'success'
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
+ CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
+ ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure
+ run: |
+ mkdir -m 700 "$RUNNER_TEMP/chat-catalog-case-bundle"
+ cp -R runner-e2e-testops "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
+ node tests/runner-e2e/chat-completion-testops-cli.mjs prepare-case-upload "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
+ cp "$RUNNER_TEMP/chat-completion-testops-prepare-case-upload/prepared.json" "$RUNNER_TEMP/chat-catalog-case-bundle/prepared.json"
+ - name: Retain the immutable metadata-only case bundle
+ id: bundle
+ if: always() && !cancelled()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: chat-catalog-case-bundle-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }}
+ path: ${{ runner.temp }}/chat-catalog-case-bundle/
+ retention-days: 30
+ if-no-files-found: error
+
+ upload:
+ name: Publish only this original case without closing the shared job run
+ needs: report
+ if: always() && !cancelled() && (needs.report.result == 'success' || needs.report.result == 'failure')
+ runs-on: ubuntu-latest
+ timeout-minutes: 10
+ environment: runner-e2e-testops-reporting
+ steps:
+ - *checkout
+ - *node
+ - *guard
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ inputs.selection_artifact_id }}
+ path: chat-completion-testops-selection
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ artifact-ids: ${{ needs.report.outputs.bundle_id }}
+ path: ${{ runner.temp }}/chat-catalog-case-bundle
+ - name: Admit the frozen singleton and submit it at most once
+ id: publication
+ env:
+ GH_TOKEN: ${{ github.token }}
+ TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
+ ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
+ ALLURE_ENDPOINT: https://paperclip.testops.cloud
+ ALLURE_PROJECT_ID: '1'
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }}
+ CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }}
+ CHAT_TESTOPS_PREPARED_FILE: ${{ runner.temp }}/chat-catalog-case-bundle/prepared.json
+ ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure
+ run: node tests/runner-e2e/chat-completion-testops-cli.mjs upload-case "$RUNNER_TEMP/chat-catalog-case-bundle/projection"
+ - name: Retain only the closed singleton publication receipt
+ if: always() && !cancelled()
+ uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+ with:
+ name: chat-catalog-case-publication-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }}
+ path: |
+ ${{ runner.temp }}/chat-completion-testops-upload-case/publication.json
+ ${{ runner.temp }}/chat-completion-testops-upload-case/failure.json
+ ${{ runner.temp }}/chat-completion-testops-upload-case/once-upload_case.json
+ retention-days: 30
+ if-no-files-found: error
diff --git a/.github/workflows/chat-catalog-testops.yml b/.github/workflows/chat-catalog-testops.yml
index db340509033..52360e6af47 100644
--- a/.github/workflows/chat-catalog-testops.yml
+++ b/.github/workflows/chat-catalog-testops.yml
@@ -23,7 +23,7 @@ permissions:
actions: read
concurrency:
- group: chat-catalog-testops
+ group: chat-catalog-testops-streaming-v1
cancel-in-progress: false
env:
@@ -78,8 +78,7 @@ jobs:
- *checkout
- *node
- *guard
- - &allure
- if: needs.authorize.outputs.operation != 'launch-plan'
+ - if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
@@ -204,107 +203,24 @@ jobs:
fail-fast: false
max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}
matrix: ${{ fromJSON(needs.select.outputs.native_matrix) }}
- runs-on: ${{ needs.authorize.outputs.runner }}
- timeout-minutes: ${{ matrix.timeout_minutes }}
- environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
- env: &retention
- TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
- MANUAL_PRIVATE_IDS: ${{ matrix.private_ids }}
- MANUAL_PRIVATE_APPROVALS_JSON: ${{ matrix.private_policy }}
- RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
- steps: &catalog_worker_steps
- - *checkout
- - *node
- - *guard
- - *pnpm
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- ref: ${{ needs.authorize.outputs.target_sha }}
- persist-credentials: false
- - name: Reject target-owned trusted helper paths
- id: target_boundary
- run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- with:
- ref: ${{ github.sha }}
- path: .testops-trusted
- persist-credentials: false
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- with:
- artifact-ids: ${{ needs.build.outputs.artifact_id }}
- path: chat-completion-build
- - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
- with:
- artifact-ids: ${{ needs.select.outputs.artifact_id }}
- path: chat-completion-testops-selection
- - name: Verify source-bound build and authenticated selection before credentials
- env:
- GH_TOKEN: ${{ github.token }}
- TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
- EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
- ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
- run: |
- set -euo pipefail
- test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
- node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
- (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
- tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
- pnpm install --frozen-lockfile --ignore-scripts
- (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
- - name: Qualify the pinned provider runtime and preinstalled browser
- run: |
- node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
- node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
- google-chrome --version
- pnpm exec playwright install ffmpeg
- - name: Bind exact private evidence under the authenticated TestOps authority
- id: private
- env:
- TARGET_ROOT: ${{ github.workspace }}
- SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
- run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- - name: Obtain only this cell's exact-key storage session
- id: cell_credentials
- uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
- with:
- role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
- aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
- inline-session-policy: ${{ steps.private.outputs.session_policy }}
- output-env-credentials: false
- output-credentials: true
- disable-retry: true
- action-timeout-s: 60
- - name: Verify private custody before delivering the model key
- env: &cell_session
- SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
- PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
- AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
- AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
- AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
- AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
- run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- - name: Execute only the selected original case with zero retries
- env:
- OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
- ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}
- PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
- ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
- PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
- PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
- PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }}
- PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
- PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
- PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
- PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
- run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- - name: Retain original evidence after success or failure
- if: always() && steps.cell_credentials.outcome == 'success'
- env: *cell_session
- run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
+ uses: ./.github/workflows/chat-catalog-testops-case.yml
+ with:
+ protocol: streaming-v1
+ cell: ${{ toJSON(matrix) }}
+ target_sha: ${{ needs.authorize.outputs.target_sha }}
+ runner: ${{ needs.authorize.outputs.runner }}
+ selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
+ selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
+ build_artifact_id: ${{ needs.build.outputs.artifact_id }}
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ secrets:
+ OPENROUTER_API_KEY: ''
+ ANTHROPIC_API_KEY: ''
+ ALLURE_TOKEN: ''
chat_external:
name: ${{ matrix.executionId }}
@@ -314,102 +230,29 @@ jobs:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}
- runs-on: ${{ needs.authorize.outputs.runner }}
- timeout-minutes: ${{ matrix.timeout_minutes }}
- environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
- env: *retention
- steps: *catalog_worker_steps
-
- report:
- name: Retain private originals and build the exact safe TestOps projection
- needs: [authorize, select, build, chat, chat_external]
- if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure')
- runs-on: ubuntu-latest
- timeout-minutes: 20
- environment: runner-e2e-private-history
- permissions:
- contents: read
- actions: read
- id-token: write
- env: *retention
- strategy:
- fail-fast: false
- max-parallel: 4
- matrix: ${{ fromJSON(needs.select.outputs.matrix) }}
- steps:
- - *checkout
- - *node
- - *guard
- - *pnpm
- - name: Install trusted reporting dependencies with lifecycle scripts disabled
- run: |
- node tests/runner-e2e/prepare-report-lock.mjs
- pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
- - name: Prepare unchanged exact-key report and cell policies
- id: session
- run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- - name: Obtain exact first-cell reader
- id: reader_0
- uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
- with:
- role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
- aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
- inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
- output-env-credentials: false
- output-credentials: true
- disable-retry: true
- action-timeout-s: 60
- - name: Download the first exact original
- env:
- PRIVATE_REPORT_ROOT: ${{ github.workspace }}
- SELECTED_CELL_INDEX: '0'
- AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
- AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
- AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
- AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
- run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- - name: Verify every selected original and its completeness receipt
- env:
- PRIVATE_REPORT_ROOT: ${{ github.workspace }}
- run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- - name: Build the existing report while preserving failed source grades
- id: normalize
- continue-on-error: true
- env:
- PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
- PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
- PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ matrix.private_ids }}
- PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
- PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
- PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
- run: |
- mkdir -p runner-e2e-merged-report
- pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- - name: Export only exact reviewed synthetic fixture results
- id: export
- if: always() && steps.reader_0.outcome == 'success'
- run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
- - name: Require a complete safe projection even after a failed normalization
- if: always()
- env:
- EXPORT_OUTCOME: ${{ steps.export.outcome }}
- run: test "$EXPORT_OUTCOME" = success
- - name: Retain the metadata-only TestOps projection
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
- with:
- name: chat-catalog-testops-cell-${{ github.run_id }}-${{ matrix.key }}
- path: runner-e2e-testops/
- retention-days: 30
- if-no-files-found: error
+ uses: ./.github/workflows/chat-catalog-testops-case.yml
+ with:
+ protocol: streaming-v1
+ cell: ${{ toJSON(matrix) }}
+ target_sha: ${{ needs.authorize.outputs.target_sha }}
+ runner: ${{ needs.authorize.outputs.runner }}
+ selection_artifact_id: ${{ needs.select.outputs.artifact_id }}
+ selection_sha256: ${{ needs.select.outputs.selection_sha256 }}
+ build_artifact_id: ${{ needs.build.outputs.artifact_id }}
+ ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
+ secrets:
+ OPENROUTER_API_KEY: ''
+ ANTHROPIC_API_KEY: ''
+ ALLURE_TOKEN: ''
aggregate:
name: Validate and merge every selected metadata-only projection
- needs: [authorize, select, report]
- if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && (needs.report.result == 'success' || needs.report.result == 'failure')
+ needs: [authorize, select, build, chat, chat_external]
+ if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
@@ -429,8 +272,15 @@ jobs:
path: chat-completion-testops-selection
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
- pattern: chat-catalog-testops-cell-${{ github.run_id }}-*
- path: catalog-projections
+ pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
+ path: catalog-bundles
+ - name: Gather only safe singleton projections from immutable bundles
+ run: |
+ mkdir -m 700 catalog-projections
+ for bundle in catalog-bundles/*; do
+ test -d "$bundle/projection"
+ cp -R "$bundle/projection" "catalog-projections/${bundle##*/}"
+ done
- name: Authenticate selection and validate all singleton exports before aggregation
env:
GH_TOKEN: ${{ github.token }}
@@ -449,9 +299,9 @@ jobs:
retention-days: 30
if-no-files-found: error
- upload:
- name: Record and independently verify each TestOps result
- needs: [authorize, select, chat, chat_external, report, aggregate]
+ verify:
+ name: Verify final coverage without uploading any result again
+ needs: [authorize, select, aggregate]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.aggregate.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -468,10 +318,17 @@ jobs:
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- - *allure
- - name: Upload original outcomes, checks and private-evidence receipts
- id: allure_upload
- env: &upload_env
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-*
+ path: catalog-bundles
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+ with:
+ pattern: chat-catalog-case-publication-${{ github.run_id }}-1-*
+ path: catalog-publications
+ - name: Authenticate every frozen publication and verify the exact original selection
+ id: verification
+ env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
@@ -479,20 +336,14 @@ jobs:
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
- ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results
- run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops
- - name: Verify actual selected results and complete result-owned attachments
- id: verification
- if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
- env: *upload_env
- run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops
- - name: Retain closed TestOps verification receipts
- if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure')
+ CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles
+ CHAT_TESTOPS_PUBLICATIONS: ${{ github.workspace }}/catalog-publications
+ run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-stream runner-e2e-testops
+ - name: Retain closed final coverage verification
+ if: always() && !cancelled()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-verification-${{ github.run_id }}-1
- path: |
- ${{ runner.temp }}/chat-completion-testops-verify-upload/
- ${{ runner.temp }}/chat-completion-testops-upload/exported.json
+ path: ${{ runner.temp }}/chat-completion-testops-verify-stream/
retention-days: 30
if-no-files-found: error
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index 12dbe906011..c4a430c95c4 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -580,6 +580,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
readWorkflow("chat-catalog-testops.yml"),
+ readWorkflow("chat-catalog-testops-case.yml"),
];
for (const workflow of actionPinWorkflows) {
@@ -620,6 +621,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
"chat-catalog-testops.yml",
+ "chat-catalog-testops-case.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
const providerSecretReference =
@@ -638,6 +640,69 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
for (const name of paidWorkflowNames) {
const workflow = readWorkflow(name);
const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n"));
+ if (name === "chat-catalog-testops.yml") {
+ assert.match(triggerHeader, /^ workflow_dispatch:/m);
+ assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_call|workflow_run):/m);
+ assert.doesNotMatch(workflow, providerSecretReference);
+ assert.match(workflow, /group: chat-catalog-testops-streaming-v1/);
+ for (const id of ["chat", "chat_external"]) {
+ const block = workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0];
+ const assertCaller = block => {
+ assert.match(block, /if: needs\.authorize\.outputs\.operation == 'evaluate'/);
+ assert.ok(block.includes(`needs.select.outputs.${id === "chat" ? "native" : "external"}_count != '0'`));
+ assert.match(block, /needs: \[authorize, select, build\]/);
+ assert.match(block, /uses: \.\/\.github\/workflows\/chat-catalog-testops-case\.yml/);
+ assert.match(block, /protocol: streaming-v1/);
+ assert.match(block, /secrets:\n OPENROUTER_API_KEY: ''\n ANTHROPIC_API_KEY: ''\n ALLURE_TOKEN: ''/);
+ assert.doesNotMatch(block, /secrets: inherit|secrets\./);
+ assert.match(block, /fail-fast: false/);
+ };
+ assertCaller(block);
+ const lane = id === "chat" ? "native" : "external";
+ for (const mutant of [block.replace("needs.authorize.outputs.operation == 'evaluate' && ", ""),
+ block.replace(`needs.select.outputs.${lane}_count != '0'`, 'true')]) assert.throws(() => assertCaller(mutant));
+ }
+ const final = workflow.split(" verify:\n")[1];
+ assert.match(final, /chat-completion-testops-cli\.mjs verify-stream/);
+ assert.doesNotMatch(final, /allurectl|cli\.mjs upload(?: |$)/);
+ continue;
+ }
+ if (name === "chat-catalog-testops-case.yml") {
+ assert.match(triggerHeader, /^ workflow_call:/m);
+ assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_dispatch|workflow_run):/m);
+ const chat = workflow.split(" chat:\n")[1].split("\n report:\n")[0];
+ const report = workflow.split(" report:\n")[1].split("\n upload:\n")[0];
+ const upload = workflow.split(" upload:\n")[1];
+ const boundary = value => {
+ assert.match(value, /environment: runner-e2e-private-paid/);
+ assert.match(value, /CHAT_TESTOPS_PROTOCOL: \$\{\{ inputs\.protocol \}\}/);
+ assert.match(value, /test "\$CHAT_TESTOPS_PROTOCOL" = streaming-v1/);
+ assert.match(value, /chat-completion-testops-cli\.mjs guard/);
+ assert.match(value, /chat-completion-testops-cli\.mjs verify-selection/);
+ assert.match(value, /--max-parallel 1 --max-automatic-retries 0/);
+ assert.match(value, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/);
+ assert.doesNotMatch(value, /ALLURE_TOKEN/);
+ assert.equal([...value.matchAll(providerSecretReference)].length, 2);
+ assert.ok(value.indexOf("chat-completion-testops-cli.mjs verify-selection") < value.search(providerSecretReference));
+ };
+ boundary(chat);
+ for (const mutant of [chat.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
+ chat.replace("cli.mjs verify-selection", "cli.mjs unverified-selection"), chat + "\n ALLURE_TOKEN: leaked"]) {
+ assert.throws(() => boundary(mutant));
+ }
+ assert.match(report, /needs: chat/); assert.match(report, /always\(\) && !cancelled\(\)/);
+ assert.match(report, /environment: runner-e2e-private-history/);
+ assert.doesNotMatch(report, /secrets\./);
+ assert.match(report, /manual-private-transfer\.ts download-cell/);
+ assert.match(report, /cli\.mjs prepare-case-upload/);
+ assert.match(upload, /needs: report/); assert.match(upload, /always\(\) && !cancelled\(\)/);
+ assert.match(upload, /environment: runner-e2e-testops-reporting/);
+ assert.doesNotMatch(upload, /configure-aws-credentials|manual-private-transfer|secrets\.(?:OPENROUTER|ANTHROPIC)/);
+ assert.match(upload, /ALLURE_TOKEN: \$\{\{ secrets\.ALLURE_TOKEN \}\}/);
+ assert.match(upload, /cli\.mjs upload-case/);
+ assert.doesNotMatch(upload, /allurectl/);
+ continue;
+ }
assert.doesNotMatch(
triggerHeader,
/^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m,
@@ -652,56 +717,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
(block) => [...block.matchAll(providerSecretReference)].length > 0,
);
assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`);
- if (name === "chat-completion-testops.yml" || name === "chat-catalog-testops.yml") {
- assert.equal(providerJobs.length, 1);
- const expectedKeys = name === "chat-catalog-testops.yml" ? 2 : 1;
- const assertBoundary = (block) => {
- assert.match(block, /^ chat:\n/);
- assert.match(block, /\n needs: \[authorize, select, build\]\n/);
- if (expectedKeys === 2) assert.ok(block.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.native_count != '0'"));
- else assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
- assert.match(block, /\n environment: runner-e2e-private-paid\n/);
- assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
- assert.match(block, /chat-completion-testops-cli\.mjs verify-selection/);
- assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
- assert.match(block, /--id "\$\{\{ matrix\.executionId \}\}" --max-parallel 1 --max-automatic-retries 0/);
- assert.match(block, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/);
- if (expectedKeys === 2) assert.ok(block.includes('max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}'));
- else assert.match(block, /max-parallel: 1/);
- assert.doesNotMatch(block, /ALLURE_TOKEN/);
- assert.equal([...block.matchAll(providerSecretReference)].length, expectedKeys);
- if (expectedKeys === 2) assert.ok(block.includes("ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}"));
- assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
- assert.ok(block.indexOf("chat-completion-testops-cli.mjs verify-selection") < block.search(providerSecretReference));
- };
- const block = providerJobs[0]; assertBoundary(block);
- assert.equal([...workflow.matchAll(providerSecretReference)].length, expectedKeys);
- for (const mutant of [
- block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
- block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
- block.replace(" - *guard\n", ""),
- block.replace("chat-completion-testops-cli.mjs verify-selection", "chat-completion-testops-cli.mjs unverified-selection"),
- block.replace('--id "${{ matrix.executionId }}"', "--all"),
- block.replace("PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1", "PAPERCLIP_CHAT_COMPLETION_TESTOPS: unknown"),
- ]) assert.throws(() => assertBoundary(mutant));
- if (expectedKeys === 2) assert.throws(() => assertBoundary(block.replace("contains(matrix.required_credentials, 'ANTHROPIC_API_KEY')", "true")));
- if (expectedKeys === 2) {
- const external = jobBlocks.find(value => value.startsWith(' chat_external:\n'));
- const assertExternal = value => {
- assert.match(value, /needs: \[authorize, select, build\]/);
- assert.ok(value.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.external_count != '0'"));
- assert.match(value, /environment: runner-e2e-private-paid/);
- assert.match(value, /max-parallel: 1/);
- assert.ok(value.includes('matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}'));
- assert.match(value, /env: \*retention/);
- assert.match(value, /steps: \*catalog_worker_steps/);
- };
- assertExternal(external);
- for (const mutant of [external.replace('*catalog_worker_steps', '*unguarded'), external.replace('max-parallel: 1', 'max-parallel: 8'), external.replace('environment: runner-e2e-private-paid', 'environment: unprotected')]) assert.throws(() => assertExternal(mutant));
- }
- continue;
- }
-
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
diff --git a/server/src/services/remote-workspace-snapshot-storage.test.ts b/server/src/services/remote-workspace-snapshot-storage.test.ts
index 8700af9346d..e1c0511da9c 100644
--- a/server/src/services/remote-workspace-snapshot-storage.test.ts
+++ b/server/src/services/remote-workspace-snapshot-storage.test.ts
@@ -86,7 +86,12 @@ suite("remote workspace snapshot storage", () => {
it("rejects object corruption and leaves the requested plaintext path unpublished", async () => {
const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload);
const objectPath = path.join(root, "objects", ref.objectKey), object = await fs.open(objectPath, "r+");
- try { await object.write(Buffer.from("X"), 0, 1, 15); } finally { await object.close(); }
+ try {
+ const byte = Buffer.alloc(1);
+ expect((await object.read(byte, 0, 1, 15)).bytesRead).toBe(1);
+ byte[0] = byte[0]! ^ 1;
+ await object.write(byte, 0, 1, 15);
+ } finally { await object.close(); }
const targetPath = path.join(root, "corrupted.pcws");
await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toThrow("object_hash_mismatch");
await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" });
diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md
index 01d7c354b23..a2267a42242 100644
--- a/tests/runner-e2e/CHAT-FEATURES.md
+++ b/tests/runner-e2e/CHAT-FEATURES.md
@@ -250,19 +250,32 @@ Initial setup uses the existing reporting environment:
evaluation after an uncertain request. Initial route verification uses a small
selection and does not qualify all 244 cases.
-Each worker executes one case. Selections of up to three cases run serially.
-Larger selections allow eight independent cases and one external-input case at
-once. External-input cases remain serial because they can share QA resources.
-A full run can take hours. Its elapsed time depends on setup and fleet capacity;
-the declared case clocks are not a completion-time guarantee. Each worker and
-report reader has a singleton private-evidence approval and exact-key AWS
-session; the existing eight-cell
-authority limit is unchanged. Reports wait for all selected workers. A
-credential-free job then validates and merges their metadata projections. The
-protected reporter uploads original outcomes and verifies every selected result,
-including paginated selections over 100 cases. Failed originals and failed
-cleanup remain failed beside passed results. Missing originals cannot become
-fabricated measurements or a full-selection pass.
+Each worker executes one case. Selections of up to three cases allow two cases
+at once. Larger selections allow eight independent cases and one external-input
+case at once. External-input cases remain serial because they can share QA
+resources. A full run can take hours. Its elapsed time depends on setup and fleet
+capacity; the declared case clocks are not a completion-time guarantee.
+
+Each finished worker starts its own protected reader and safe exporter. A
+separate reporting job uploads that case and verifies its result and two JSON
+attachments while other workers continue. The reader has only an exact-key AWS
+session. The worker has only its declared model keys. Only the uploader has the
+TestOps key. The existing eight-cell private authority limit is unchanged.
+
+The exporter freezes each UUID-bearing Allure bundle once. An uncertain upload
+response triggers readback, never an automatic resend. The final job verifies
+complete selected-case coverage and the original statuses without uploading the
+results again. Failed originals and failed cleanup remain failed beside passed
+results. Missing originals cannot become fabricated measurements or a
+full-selection pass. Backend ingestion does not guarantee that an open TestOps
+page refreshes without user action.
+
+The caller and reusable case workflow must both be admitted at their exact
+`refs/heads/master` paths in the existing runner group. Called jobs use their own
+protected environments; the caller passes empty named secret placeholders and
+never inherits or forwards actual model or reporting keys. Streaming runs use
+the fixed `chat-catalog-testops-streaming-v1` serialization group. Runs started
+before this change retain their original batch-reporting flow.
Raw evidence and final PNGs remain in protected storage. Public GitHub artifacts
and TestOps attachments contain only allowlisted provenance, checks, accounting
diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.mjs
new file mode 100644
index 00000000000..6d27e7d30ae
--- /dev/null
+++ b/tests/runner-e2e/chat-catalog-testops-streaming.mjs
@@ -0,0 +1,134 @@
+import { lstat, readFile, readdir, realpath } from 'node:fs/promises';
+import path from 'node:path';
+import { hash, validateSelection, reject } from './chat-completion-testops-contract.mjs';
+import { readPilotExport, exportPilotAllure } from './chat-completion-testops-results.mjs';
+import { inspectPilotCaseUpload, verifyPilotCaseUpload } from './chat-completion-testops-api.mjs';
+
+export function caseSelection(selection, context, executionId) {
+ if (context.catalog !== 'full') reject('stream_scope');
+ const authorized = validateSelection(selection, context, selection.allureJobRunId);
+ const row = authorized.find(value => value.executionId === executionId);
+ if (!row) reject('stream_scope');
+ return { ...selection, cases: [row] };
+}
+const identity = (selection, context, executionId, selectionSha256) => {
+ if (!/^[a-f0-9]{64}$/.test(selectionSha256)) reject('stream_binding');
+ const scoped = caseSelection(selection, context, executionId), row = scoped.cases[0];
+ return { schema: 'paperclip.chat-catalog-testops-case-bundle/v1', ...context,
+ projectId: 1, planId: selection.planId, jobId: selection.jobId, launchId: selection.launchId,
+ allureJobRunId: selection.allureJobRunId, executionId, caseId: row.caseId,
+ fullSelectionSha256: selectionSha256, fullSelectionCount: selection.cases.length };
+};
+async function filesAt(root, wanted) {
+ if (await realpath(root) !== path.resolve(root)) reject('stream_bundle');
+ const names = (await readdir(root)).sort();
+ if (names.length !== 3 || names.some(name => !/^[a-f0-9-]{36}-(?:attachment|result)\.json$/.test(name))
+ || names.filter(name => name.endsWith('-result.json')).length !== 1) reject('stream_bundle');
+ const files = []; let total = 0;
+ for (const name of names) {
+ const file = path.join(root, name), before = await lstat(file);
+ if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > 12 * 1024 * 1024) reject('stream_bundle');
+ const bytes = await readFile(file), after = await lstat(file); total += bytes.length;
+ if (total > 32 * 1024 * 1024 || before.ino !== after.ino || before.dev !== after.dev || before.size !== bytes.length
+ || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) reject('stream_bundle');
+ files.push({ name, bytes, sha256: hash(bytes) });
+ }
+ const metadata = files.map(file => ({ name: file.name, bytes: file.bytes.length, sha256: file.sha256 }));
+ if (wanted && JSON.stringify(metadata) !== JSON.stringify(wanted)) reject('stream_bundle');
+ return { files, metadata };
+}
+
+// The existing exporter owns grades and the public allowlist. Materialize once;
+// preserve these exact UUID-bearing bytes through every later readback.
+export async function prepareCaseBundle(root, output, context, selection, executionId, selectionSha256) {
+ const bound = identity(selection, context, executionId, selectionSha256);
+ const scoped = caseSelection(selection, context, executionId);
+ const exported = await exportPilotAllure(root, output, context, scoped);
+ const { metadata: files } = await filesAt(output);
+ const prepared = { ...bound,
+ projectionBindingSha256: hash(await readFile(path.join(root, 'binding.json'))), exported, files };
+ return { ...prepared, payloadSha256: hash(JSON.stringify(prepared)) };
+}
+
+export async function readCaseBundle(root, output, prepared, context, selection, executionId, selectionSha256) {
+ const expected = identity(selection, context, executionId, selectionSha256);
+ if (!prepared || Object.entries(expected).some(([key, value]) => prepared[key] !== value)) reject('stream_binding');
+ const { payloadSha256, ...unsigned } = prepared;
+ if (!/^[a-f0-9]{64}$/.test(payloadSha256) || hash(JSON.stringify(unsigned)) !== payloadSha256) reject('stream_bundle');
+ const scoped = caseSelection(selection, context, executionId);
+ const validated = await readPilotExport(root, context, scoped), original = validated.campaign.results[0];
+ if (hash(await readFile(path.join(root, 'binding.json'))) !== prepared.projectionBindingSha256
+ || !Array.isArray(prepared.exported) || prepared.exported.length !== 1
+ || prepared.exported[0].caseId !== expected.caseId || prepared.exported[0].executionId !== executionId
+ || prepared.exported[0].status !== (original.status === 'passed' && original.cleanup === 'passed' ? 'passed' : 'failed')
+ || prepared.exported[0].cleanup !== original.cleanup) reject('stream_binding');
+ return filesAt(output, prepared.files);
+}
+
+export async function publishCaseBundle(api, root, output, prepared, context, selection, executionId, selectionSha256, markOnce, options) {
+ const { files } = await readCaseBundle(root, output, prepared, context, selection, executionId, selectionSha256);
+ const before = await inspectPilotCaseUpload(api, selection, context, executionId, options);
+ let response, uncertain = null;
+ if (!before.present) {
+ // Once markers are retained before this call. An uncertain reply is reconciled
+ // by GET only. Neither this function nor the finalizer repeats the POST.
+ await markOnce({ executionId, caseId: prepared.caseId, payloadSha256: prepared.payloadSha256 });
+ const body = new FormData();
+ for (const file of files) body.append('file', new Blob([file.bytes], { type: 'application/json' }), file.name);
+ try { response = await api(`/api/jobrun/${selection.allureJobRunId}/upload`, 'POST', body); }
+ catch (error) { uncertain = { kind: ['testops_api_http', 'testops_response_invalid', 'testops_response_too_large'].includes(error?.code)
+ ? error.code : 'transport_or_response_failure', httpStatus: Number.isInteger(error?.status) ? error.status : null }; }
+ }
+ let verification;
+ try { verification = await verifyPilotCaseUpload(api, selection, context, executionId, prepared.exported, options); }
+ catch (error) {
+ if (uncertain) { const stopped = new Error('chat_completion_testops_stream_upload_uncertain'); stopped.uploadFailure = uncertain; throw stopped; }
+ throw error;
+ }
+ const after = await api(`/api/jobrun/${selection.allureJobRunId}`);
+ if (after.id !== Number(selection.allureJobRunId) || after.job?.id !== selection.jobId || after.launchId !== selection.launchId
+ || after.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` || !['in_progress', 'finished'].includes(after.stage)) reject('jobrun_mismatch');
+ return { ...identity(selection, context, executionId, selectionSha256), schema: 'paperclip.chat-catalog-testops-case-publication/v1',
+ payloadSha256: prepared.payloadSha256, projectionBindingSha256: prepared.projectionBindingSha256, exported: prepared.exported[0],
+ uploadSent: !before.present, uncertainUpload: uncertain, jobRunStageBefore: before.jobRunStage, jobRunStageAfter: after.stage,
+ responseFilesCount: Number.isSafeInteger(response?.filesCount) && response.filesCount >= 0 ? response.filesCount : null,
+ verification: verification.verification, results: verification.results };
+}
+
+// The finalizer authenticates the already-emitted bytes and their original
+// grades. It never calls the random-UUID exporter or the upload endpoint.
+export async function publicationSet(bundlesRoot, publicationsRoot, context, selection, selectionSha256) {
+ const rows = validateSelection(selection, context, selection.allureJobRunId);
+ const roots = [bundlesRoot, publicationsRoot];
+ for (const [index, root] of roots.entries()) {
+ if (await realpath(root) !== path.resolve(root)) reject('stream_publications');
+ const prefix = `chat-catalog-case-${index === 0 ? 'bundle' : 'publication'}-${context.runId}-1-`;
+ const names = await readdir(root);
+ if (names.length !== rows.length || names.some(name => !rows.some((_, i) => name === `${prefix}${i}`))) reject('stream_publications');
+ }
+ const exported = [];
+ for (const [index, row] of rows.entries()) {
+ const bundle = path.join(bundlesRoot, `chat-catalog-case-bundle-${context.runId}-1-${index}`);
+ const publication = path.join(publicationsRoot, `chat-catalog-case-publication-${context.runId}-1-${index}`);
+ for (const directory of [bundle, publication]) {
+ const stat = await lstat(directory);
+ if (!stat.isDirectory() || stat.isSymbolicLink()) reject('stream_publications');
+ }
+ const read = async (directory, name) => {
+ const stat = await lstat(path.join(directory, name));
+ if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 65_536) reject('stream_publications');
+ return JSON.parse(await readFile(path.join(directory, name), 'utf8'));
+ };
+ const prepared = await read(bundle, 'prepared.json');
+ await readCaseBundle(path.join(bundle, 'projection'), path.join(bundle, 'allure'), prepared, context, selection, row.executionId, selectionSha256);
+ const value = await read(publication, 'publication.json');
+ const expected = { ...identity(selection, context, row.executionId, selectionSha256), schema: 'paperclip.chat-catalog-testops-case-publication/v1' };
+ if (Object.entries(expected).some(([key, wanted]) => value[key] !== wanted)
+ || value.payloadSha256 !== prepared.payloadSha256 || value.projectionBindingSha256 !== prepared.projectionBindingSha256
+ || JSON.stringify(value.exported) !== JSON.stringify(prepared.exported[0]) || value.verification !== 'VERIFIED'
+ || !Array.isArray(value.results) || value.results.length !== 1 || value.results[0].caseId !== row.caseId
+ || value.results[0].executionId !== row.executionId || value.results[0].attachmentsVerified !== true) reject('stream_publications');
+ exported.push(value.exported);
+ }
+ return exported;
+}
diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs
new file mode 100644
index 00000000000..525b77f65ba
--- /dev/null
+++ b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs
@@ -0,0 +1,280 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, mkdir, readFile, readdir, writeFile, rm, realpath, cp } from 'node:fs/promises';
+import path from 'node:path';
+import { tmpdir } from 'node:os';
+import { fullCatalogCases, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs';
+import { hash, model } from './chat-completion-testops-contract.mjs';
+import { run } from './chat-completion-testops-cli.mjs';
+import { prepareCaseBundle, readCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs';
+const context = { catalog: 'full', runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: catalogTargetSha };
+const caseIds = { 'tasks.create-backlog': 388, 'documents-artifacts.write-plan': 26, 'projects.project-defaults': 100 };
+const cases = fullCatalogCases.filter(row => Object.hasOwn(caseIds, row.key)).map(row => ({ ...row, caseId: caseIds[row.key] }));
+const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1,
+ planId: 102, jobId: 100, launchId: 12, allureJobRunId: '77', cases };
+const digest = hash(JSON.stringify(selection)), current = cases.find(row => row.caseId === 388), runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123';
+async function fixture(t, failed = false, owned = current, runtimeContext = context) {
+ const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-test-')));
+ t.after(() => rm(base, { recursive: true, force: true }));
+ const root = path.join(base, 'projection'), output = path.join(base, 'allure'); await mkdir(root);
+ const original = { executionId: owned.executionId, suiteId: 'chat-features', caseId: owned.scenario,
+ profileId: 'runner-opencode', provider: 'opencode', environmentId: 'local', model, runtimeMode: 'native', attempt: 1,
+ source: { sha: catalogTargetSha, workflowRunUrl: runUrl }, status: failed ? 'failed' : 'passed', cleanup: failed ? 'failed' : 'passed',
+ runIds: failed ? [] : Array.from({ length: owned.minimumRuns }, (_, i) => `synthetic-owned-run-${i}`), matcherResults: failed ? [] : Array.from({ length: owned.checks ?? 2 }, (_, i) => ({
+ matcher: { kind: 'json_path', path: `owned.check${i}`, expected: true }, passed: true })), screenshots: [] };
+ const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2',
+ campaignId: 'gha-123-1', retries: 0, results: [original] }))], ['junit.xml', Buffer.from('')]]);
+ const binding = { schema: 'paperclip.chat-completion-testops-export/v1', ...context, campaignId: 'gha-123-1',
+ executionIds: [owned.executionId], files: [...files].map(([name, bytes]) => ({ path: name, bytes: bytes.length, sha256: hash(bytes) })),
+ privateEvidence: [{ executionId: owned.executionId, requiredEvidenceComplete: !failed,
+ finalScreenshot: failed ? null : { sha256: 'e'.repeat(64), bytes: 24 } }] };
+ for (const [name, bytes] of files) await writeFile(path.join(root, name), bytes);
+ await writeFile(path.join(root, 'binding.json'), JSON.stringify(binding));
+ const prepared = await prepareCaseBundle(root, output, runtimeContext, selection, owned.executionId, digest);
+ return { root, output, prepared };
+}
+function server(prepared, { existing = false, uncertain = false, apply = true } = {}) {
+ let present = existing, posts = 0; const calls = [];
+ const api = async (route, method = 'GET', body) => {
+ calls.push({ route, method });
+ if (method === 'POST') {
+ posts++; assert.equal(route, '/api/jobrun/77/upload');
+ assert.deepEqual([...body.keys()], ['file','file','file']);
+ const result = [...body.values()].find(value => value.name.endsWith('-result.json'));
+ const parsed = JSON.parse(await result.text());
+ assert.equal(parsed.status, prepared.exported[0].status); assert.equal(parsed.labels[0].value, '388');
+ if (apply) present = true;
+ if (uncertain) throw new Error('SYNTHETIC_PRIVATE_RESPONSE_MUST_NOT_ESCAPE');
+ return { filesCount: 3 };
+ }
+ if (route === '/api/jobrun/77') return { id: 77, job: { id: 100, type: 'github' }, stage: 'finished', launchId: 12, url: runUrl };
+ if (route === '/api/launch/12') return { id: 12, projectId: 1, closed: false };
+ if (route.startsWith('/api/testresult?')) return { content: [...(present ? [{ id: 501, testCaseId: 388 }] : []),
+ { id: 502, testCaseId: 26 }], totalElements: present ? 2 : 1, totalPages: 1, number: 0 };
+ if (route === '/api/testresult/501') return { id: 501, testCaseId: 388, projectId: 1, launchId: 12,
+ fullName: current.fullName, status: prepared.exported[0].status, jobRun: { id: 77, launchId: 12, url: runUrl }, links: [] };
+ if (route.startsWith('/api/testresult/attachment?')) return { content: prepared.exported[0].attachments.map((row, index) => ({
+ ...row, id: 700 + index, missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 };
+ throw new Error(`Unexpected test route ${route}`);
+ };
+ return { api, calls, get posts() { return posts; } };
+}
+const immediate = { now: () => 0, wait: async () => {} };
+async function publish(f, s, mark = async () => {}) {
+ return publishCaseBundle(s.api, f.root, f.output, f.prepared, context, selection, current.executionId, digest, mark, immediate);
+}
+test('one frozen singleton bundle publishes into the bound shared launch beside siblings', async t => {
+ const f = await fixture(t), s = server(f.prepared); let markers = 0;
+ const names = await readdir(f.output), value = await publish(f, s, async () => { markers++; });
+ assert.equal(value.verification, 'VERIFIED'); assert.equal(markers, 1); assert.equal(s.posts, 1);
+ assert.equal(value.jobRunStageBefore, 'finished'); assert.equal(value.jobRunStageAfter, 'finished');
+ assert.deepEqual(await readdir(f.output), names);
+ await assert.rejects(prepareCaseBundle(f.root, f.output, context, selection, current.executionId, digest), /allure_output/);
+});
+test('an existing result is verified without another mutation or a regenerated UUID', async t => {
+ const f = await fixture(t), s = server(f.prepared, { existing: true });
+ const value = await publish(f, s, async () => assert.fail('must not mark another upload'));
+ assert.equal(value.uploadSent, false); assert.equal(s.posts, 0); assert.equal(value.verification, 'VERIFIED');
+});
+test('an uncertain applied upload reconciles by GET while an absent upload stops without resend', async t => {
+ for (const apply of [true, false]) {
+ const f = await fixture(t), s = server(f.prepared, { uncertain: true, apply });
+ if (apply) {
+ const value = await publish(f, s); assert.equal(value.verification, 'VERIFIED');
+ assert.equal(value.uncertainUpload.kind, 'transport_or_response_failure');
+ assert.ok(!JSON.stringify(value).includes('SYNTHETIC_PRIVATE'));
+ } else await assert.rejects(publish(f, s), error => error.message === 'chat_completion_testops_stream_upload_uncertain'
+ && !JSON.stringify(error).includes('SYNTHETIC_PRIVATE'));
+ assert.equal(s.posts, 1);
+ }
+});
+test('failed originals remain failed, including zero-run setup or cleanup failure', async t => {
+ const f = await fixture(t, true), s = server(f.prepared), value = await publish(f, s);
+ assert.equal(value.exported.status, 'failed'); assert.equal(value.exported.cleanup, 'failed');
+ assert.equal(value.verification, 'VERIFIED');
+});
+test('changed bundle bytes, source, selection digest and foreign case refuse before transport', async t => {
+ const f = await fixture(t); let calls = 0;
+ const noApi = async () => { calls++; assert.fail('untrusted bundle must not reach API'); };
+ await assert.rejects(publishCaseBundle(noApi, f.root, f.output, f.prepared, { ...context, workflowSha: 'b'.repeat(40) }, selection,
+ current.executionId, digest, async () => {}), /selection_binding/);
+ await assert.rejects(readCaseBundle(f.root, f.output, f.prepared, context, selection, current.executionId, 'b'.repeat(64)), /stream_binding/);
+ await assert.rejects(readCaseBundle(f.root, f.output, f.prepared, context, selection, 'foreign', digest), /stream_scope/);
+ await writeFile(path.join(f.output, f.prepared.files[0].name), 'SYNTHETIC_PRIVATE_TAMPER');
+ await assert.rejects(publishCaseBundle(noApi, f.root, f.output, f.prepared, context, selection, current.executionId, digest, async () => {}), /stream_bundle/);
+ assert.equal(calls, 0);
+});
+
+ test('final coverage authenticates existing per-case bytes and keeps mixed original grades without re-export', async t => {
+ const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-final-')));
+ t.after(() => rm(base, { recursive: true, force: true }));
+ const bundles = path.join(base, 'bundles'), publications = path.join(base, 'publications');
+ await mkdir(bundles); await mkdir(publications);
+ for (const [index, row] of cases.entries()) {
+ const f = await fixture(t, index === 1, row);
+ const bundle = path.join(bundles, `chat-catalog-case-bundle-123-1-${index}`);
+ const receipt = path.join(publications, `chat-catalog-case-publication-123-1-${index}`);
+ await mkdir(bundle); await mkdir(receipt);
+ await cp(f.root, path.join(bundle, 'projection'), { recursive: true });
+ await cp(f.output, path.join(bundle, 'allure'), { recursive: true });
+ await writeFile(path.join(bundle, 'prepared.json'), JSON.stringify(f.prepared));
+ await writeFile(path.join(receipt, 'publication.json'), JSON.stringify({ ...f.prepared,
+ schema: 'paperclip.chat-catalog-testops-case-publication/v1', exported: f.prepared.exported[0],
+ verification: 'VERIFIED', results: [{ executionId: row.executionId, caseId: row.caseId, attachmentsVerified: true }] }));
+ }
+ const rows = await publicationSet(bundles, publications, context, selection, digest);
+ assert.deepEqual(rows.map(row => row.status), ['passed','failed','passed']);
+ const receiptPath = path.join(publications, 'chat-catalog-case-publication-123-1-1', 'publication.json');
+ const bad = JSON.parse(await readFile(receiptPath)); bad.exported.status = 'passed';
+ await writeFile(receiptPath, JSON.stringify(bad));
+ await assert.rejects(publicationSet(bundles, publications, context, selection, digest), /stream_publications/);
+ await rm(path.dirname(receiptPath), { recursive: true });
+ await assert.rejects(publicationSet(bundles, publications, context, selection, digest), /stream_publications/);
+});
+
+test('the real CLI prepares without TestOps credentials and retains closed failed or uncertain publication witnesses', async t => {
+ for (const uncertain of [false, true]) {
+ const f = await fixture(t, true), base = path.dirname(f.root);
+ await rm(f.output, { recursive: true });
+ const selectionRoot = path.join(base, 'chat-completion-testops-selection'); await mkdir(selectionRoot);
+ await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selection));
+ const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' },
+ ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } };
+ const eventPath = path.join(base, 'event.json'); await writeFile(eventPath, JSON.stringify(event));
+ const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name,
+ GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master',
+ GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42',
+ TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath,
+ GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: base, GITHUB_WORKSPACE: base, ALLURE_JOB_RUN_ID: '77',
+ ALLURE_RESULTS: f.output, CHAT_TESTOPS_EXECUTION_ID: current.executionId, EXPECTED_SELECTION_SHA256: digest };
+ const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99,
+ head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml',
+ repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) });
+ assert.equal(await run('prepare-case-upload', f.root, env, { exec, fetchImpl: () => assert.fail('reader must not authenticate to TestOps') }), 0);
+ const preparedPath = path.join(base, 'chat-completion-testops-prepare-case-upload', 'prepared.json');
+ const prepared = JSON.parse(await readFile(preparedPath)), s = server(prepared, { uncertain, apply: !uncertain });
+ const fetchImpl = async (url, options) => {
+ if (url.endsWith('/api/uaa/oauth/token')) return Response.json({ access_token: 'SYNTHETIC_ACCESS_TOKEN' });
+ if (uncertain && s.posts && new URL(url).pathname === '/api/jobrun/77') return Response.json({ id: 78 });
+ return Response.json(await s.api(new URL(url).pathname + new URL(url).search, options.method, options.body));
+ };
+ let captured = ''; const capture = t.mock.method(process.stderr, 'write', value => { captured += value; return true; });
+ assert.equal(await run('upload-case', f.root, { ...env, ALLURE_TOKEN: 'SYNTHETIC_TESTOPS_SECRET',
+ CHAT_TESTOPS_PREPARED_FILE: preparedPath }, { exec, fetchImpl }), 1);
+ capture.mock.restore();
+ const state = path.join(base, 'chat-completion-testops-upload-case');
+ assert.equal(s.posts, 1);
+ const receipt = JSON.parse(await readFile(path.join(state, uncertain ? 'failure.json' : 'publication.json')));
+ if (uncertain) {
+ assert.equal(receipt.code, 'chat_completion_testops_stream_upload_uncertain');
+ assert.equal(receipt.uploadFailure.kind, 'transport_or_response_failure');
+ await assert.rejects(readFile(path.join(state, 'publication.json')), { code: 'ENOENT' });
+ } else { assert.equal(receipt.verification, 'VERIFIED'); assert.equal(receipt.exported.status, 'failed'); }
+ assert.deepEqual(JSON.parse(await readFile(path.join(state, 'once-upload_case.json'))),
+ { stage: 'upload_case', status: 'STARTED_NO_AUTOMATIC_RESEND' });
+ assert.doesNotMatch(captured + JSON.stringify(receipt), /SYNTHETIC_|ACCESS_TOKEN/);
+ const yaml = await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8');
+ for (const filename of ['publication.json', 'failure.json', 'once-upload_case.json']) {
+ assert.ok(yaml.includes(`/chat-completion-testops-upload-case/${filename}`));
+ }
+ }
+});
+
+test('source host admission follows only the fixed same-commit callee with empty caller secret placeholders', async () => {
+ const { parse } = await import('yaml');
+ const { paidWorker } = await import('./chat-completion-testops-host-control.mjs');
+ const main = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8'));
+ const callee = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8'));
+ assert.equal(paidWorker(main, callee).environment, 'runner-e2e-private-paid');
+ for (const change of [
+ value => { value.jobs.chat.uses = 'foreign/repository/workflow.yml@master'; },
+ value => { value.jobs.chat.secrets = 'inherit'; },
+ value => { value.jobs.chat_external.secrets.ALLURE_TOKEN = '${{ secrets.ALLURE_TOKEN }}'; },
+ value => { value.jobs.chat.with.selection_sha256 = 'arbitrary'; },
+ ]) { const mutant = structuredClone(main); change(mutant); assert.throws(() => paidWorker(mutant, callee), /boundary/); }
+ const wrong = structuredClone(callee); wrong.jobs.chat.environment = 'runner-e2e-testops-reporting';
+ assert.throws(() => paidWorker(main, wrong), /boundary/);
+});
+
+
+test('the final CLI verifies all authenticated mixed-grade publications without re-export or upload and refuses a contradictory merged grade', async t => {
+ for (const contradiction of [false, true]) {
+ const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-cli-final-')));
+ t.after(() => rm(base, { recursive: true, force: true }));
+ const bundles = path.join(base, 'bundles'), publications = path.join(base, 'publications'), merged = path.join(base, 'merged');
+ for (const root of [bundles, publications, merged]) await mkdir(root);
+ const actual = { ...context, operation: 'evaluate', actorId: 42, triggeringActorId: 42, workflowId: 99 };
+ const originals = [], privateEvidence = [], exported = [];
+ for (const [index, row] of cases.entries()) {
+ const f = await fixture(t, index === 1, row, actual);
+ const bundle = path.join(bundles, `chat-catalog-case-bundle-123-1-${index}`);
+ const publication = path.join(publications, `chat-catalog-case-publication-123-1-${index}`);
+ await mkdir(bundle); await mkdir(publication);
+ await cp(f.root, path.join(bundle, 'projection'), { recursive: true });
+ await cp(f.output, path.join(bundle, 'allure'), { recursive: true });
+ await writeFile(path.join(bundle, 'prepared.json'), JSON.stringify(f.prepared));
+ await writeFile(path.join(publication, 'publication.json'), JSON.stringify({ ...f.prepared,
+ schema: 'paperclip.chat-catalog-testops-case-publication/v1', exported: f.prepared.exported[0], verification: 'VERIFIED',
+ results: [{ executionId: row.executionId, caseId: row.caseId, attachmentsVerified: true }] }));
+ originals.push(JSON.parse(await readFile(path.join(f.root, 'normalized-results.json'))).results[0]);
+ privateEvidence.push(...JSON.parse(await readFile(path.join(f.root, 'binding.json'))).privateEvidence);
+ exported.push(f.prepared.exported[0]);
+ }
+ if (contradiction) Object.assign(originals[0], { status: 'failed', cleanup: 'failed', runIds: [], matcherResults: [] });
+ const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2',
+ campaignId: 'gha-123-1', retries: 0, results: originals }))], ['junit.xml', Buffer.from('')]]);
+ for (const [name, bytes] of files) await writeFile(path.join(merged, name), bytes);
+ await writeFile(path.join(merged, 'binding.json'), JSON.stringify({ schema: 'paperclip.chat-completion-testops-export/v1',
+ ...context, campaignId: 'gha-123-1', executionIds: cases.map(row => row.executionId), privateEvidence,
+ files: [...files].map(([name, bytes]) => ({ path: name, bytes: bytes.length, sha256: hash(bytes) })) }));
+ const selectionRoot = path.join(base, 'chat-completion-testops-selection'); await mkdir(selectionRoot);
+ await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selection));
+ const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' },
+ ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } };
+ const eventPath = path.join(base, 'event.json'); await writeFile(eventPath, JSON.stringify(event));
+ const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name,
+ GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master',
+ GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42',
+ TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath,
+ GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: base, GITHUB_WORKSPACE: base, ALLURE_JOB_RUN_ID: '77',
+ ALLURE_TOKEN: 'SYNTHETIC_REPORTING_SECRET', EXPECTED_SELECTION_SHA256: digest,
+ CHAT_TESTOPS_BUNDLES: bundles, CHAT_TESTOPS_PUBLICATIONS: publications, ALLURE_RESULTS: path.join(base, 'must-not-export') };
+ const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99,
+ head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml',
+ repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) });
+ const details = [];
+ const fetchImpl = async (url, options) => {
+ const u = new URL(url), route = u.pathname + u.search;
+ if (u.pathname === '/api/uaa/oauth/token') return Response.json({ access_token: 'SYNTHETIC_ACCESS_TOKEN' });
+ assert.equal(options.method, 'GET', 'final verification must not upload or change TestOps');
+ if (route === '/api/jobrun/77') return Response.json({ id: 77, job: { id: 100, type: 'github' }, stage: 'finished', launchId: 12, url: runUrl });
+ if (route === '/api/launch/12') return Response.json({ id: 12, projectId: 1 });
+ if (u.pathname === '/api/testresult') return Response.json({ content: cases.map((row, i) => ({ id: 501 + i, testCaseId: row.caseId })), totalElements: 3, totalPages: 1, number: 0 });
+ const match = u.pathname.match(/^\/api\/testresult\/(\d+)$/);
+ if (match) {
+ const i = Number(match[1]) - 501; details.push(cases[i].caseId);
+ return Response.json({ id: Number(match[1]), projectId: 1, launchId: 12, testCaseId: cases[i].caseId, fullName: cases[i].fullName,
+ jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], status: exported[i].status });
+ }
+ assert.equal(u.pathname, '/api/testresult/attachment');
+ const i = Number(u.searchParams.get('testResultId')) - 501;
+ return Response.json({ content: exported[i].attachments.map((row, index) => ({ ...row, id: 700 + index, missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 });
+ };
+ const before = await Promise.all((await readdir(bundles)).map(async name => [name, hash(await readFile(path.join(bundles, name, 'prepared.json')))]));
+ let stderr = ''; const capture = t.mock.method(process.stderr, 'write', value => { stderr += value; return true; });
+ assert.equal(await run('verify-stream', merged, env, { exec, fetchImpl }), 1);
+ capture.mock.restore();
+ const state = path.join(base, 'chat-completion-testops-verify-stream');
+ if (contradiction) {
+ assert.equal(JSON.parse(await readFile(path.join(state, 'failure.json'))).code, 'chat_completion_testops_upload_receipt');
+ assert.deepEqual(details, []);
+ } else {
+ const verified = JSON.parse(await readFile(path.join(state, 'verification.json')));
+ assert.equal(verified.verification, 'VERIFIED'); assert.deepEqual(details.sort((a,b) => a-b), [26,100,388]);
+ assert.deepEqual(verified.results.map(row => row.status), ['passed','failed','passed']);
+ }
+ await assert.rejects(readFile(env.ALLURE_RESULTS), { code: 'ENOENT' });
+ assert.deepEqual(await Promise.all((await readdir(bundles)).map(async name => [name, hash(await readFile(path.join(bundles, name, 'prepared.json')))])), before);
+ assert.doesNotMatch(stderr, /SYNTHETIC_/);
+ }
+});
diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs
index 7d02341af3e..75f85726bef 100644
--- a/tests/runner-e2e/chat-completion-testops-api.mjs
+++ b/tests/runner-e2e/chat-completion-testops-api.mjs
@@ -215,9 +215,22 @@ export async function launchPilotPlan(api, env, context, workflowId, markOnce) {
launchUrl: `${endpoint}/launch/${launched.id}`, status: 'REQUESTED', actualResult: null };
}
-export async function verifyPilotUpload(api, selection, context, exported, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) {
+async function uploadBinding(get, selection, context) {
+ const run = await get(`/api/jobrun/${selection.allureJobRunId}`);
+ const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`;
+ if (run.id !== Number(selection.allureJobRunId) || run.job?.id !== selection.jobId || run.job?.type !== 'github'
+ || !['in_progress', 'finished'].includes(run.stage)
+ || run.launchId !== selection.launchId || run.url !== runUrl) reject('jobrun_mismatch');
+ const launch = await get(`/api/launch/${selection.launchId}`);
+ if (launch.id !== selection.launchId || launch.projectId !== 1) reject('launch_mismatch');
+ return { run, launch, runUrl };
+}
+
+async function verifyUploadScope(api, selection, context, exported, executionId, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) {
const { cases, targetSha, planName } = catalogDefinition(context);
- const selected = validateSelection(selection, context, selection.allureJobRunId);
+ const authorized = validateSelection(selection, context, selection.allureJobRunId);
+ const selected = executionId === undefined ? authorized : authorized.filter(row => row.executionId === executionId);
+ if (executionId !== undefined && (context.catalog !== 'full' || selected.length !== 1)) reject('stream_scope');
if (!Array.isArray(exported) || exported.length !== selected.length || new Set(exported.map(row => row.caseId)).size !== selected.length) reject('export_mismatch');
for (const row of exported) {
const found = selected.find(value => value.caseId === row.caseId);
@@ -230,8 +243,8 @@ export async function verifyPilotUpload(api, selection, context, exported, { now
|| !Number.isSafeInteger(value.contentLength) || value.contentLength <= 0 || value.contentLength > 12 * 1024 * 1024
|| value.contentType !== 'application/json')) reject('export_mismatch');
}
- const metadataReadAllowance = context.catalog === 'full' && selected.length > 3
- ? (2 + Math.ceil(selected.length / 100) + 2 * Math.ceil(selected.length / 4)) * 2000 : 0;
+ const metadataReadAllowance = context.catalog === 'full' && authorized.length > 3
+ ? (2 + Math.ceil(authorized.length / 100) + 2 * Math.ceil(authorized.length / 4)) * 2000 : 0;
const deadline = now() + 120_000 + metadataReadAllowance;
const get = route => {
const remaining = deadline - now(); if (remaining < 1) reject('upload_unverified');
@@ -239,21 +252,17 @@ export async function verifyPilotUpload(api, selection, context, exported, { now
};
const signature = rows => rows.map(row => JSON.stringify([row.name, row.contentType, row.contentLength])).sort();
for (let poll = 0; poll < 12 && now() < deadline; poll++) {
- const run = await get(`/api/jobrun/${selection.allureJobRunId}`);
- const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`;
- if (run.id !== Number(selection.allureJobRunId) || run.job?.id !== selection.jobId || run.job?.type !== 'github'
- || !['in_progress', 'finished'].includes(run.stage)
- || run.launchId !== selection.launchId || run.url !== runUrl) reject('jobrun_mismatch');
- const launch = await get(`/api/launch/${selection.launchId}`);
- if (launch.id !== selection.launchId || launch.projectId !== 1) reject('launch_mismatch');
- const listed = await resultRows(get, selection.launchId, selected.length, context.catalog === 'full');
+ const { runUrl } = await uploadBinding(get, selection, context);
+ const listed = await resultRows(get, selection.launchId, authorized.length, context.catalog === 'full');
if (listed === null) {
if (poll < 11 && now() + 5000 < deadline) await wait(5000);
continue;
}
- if (listed.length > selected.length
+ if (listed.length > authorized.length
|| new Set(listed.map(row => row.id)).size !== listed.length || new Set(listed.map(row => row.testCaseId)).size !== listed.length) reject('result_page');
- let ready = listed.length === selected.length; const verified = [];
+ if (listed.some(row => !authorized.some(expected => expected.caseId === row.testCaseId))) reject('foreign_result');
+ const matching = executionId === undefined ? listed : listed.filter(row => row.testCaseId === selected[0].caseId);
+ let ready = matching.length === selected.length; const verified = [];
const verifyResult = async listedRow => {
const expected = selected.find(value => value.caseId === listedRow.testCaseId);
if (!expected || !positive(listedRow.id)) reject('foreign_result');
@@ -280,8 +289,8 @@ export async function verifyPilotUpload(api, selection, context, exported, { now
return { executionId: expected.executionId, caseId: expected.caseId, resultId: result.id,
status: result.status, cleanup: wanted.cleanup, attachmentsVerified: complete, resultUrl: `${endpoint}/testresult/${result.id}` };
};
- for (let offset = 0; offset < listed.length; offset += context.catalog === 'full' ? 4 : 1) {
- const batch = await Promise.all(listed.slice(offset, offset + (context.catalog === 'full' ? 4 : 1)).map(verifyResult));
+ for (let offset = 0; offset < matching.length; offset += context.catalog === 'full' ? 4 : 1) {
+ const batch = await Promise.all(matching.slice(offset, offset + (context.catalog === 'full' ? 4 : 1)).map(verifyResult));
ready &&= batch.every(row => row.attachmentsVerified); verified.push(...batch);
}
if (ready && now() < deadline) return { schema: 'paperclip.chat-completion-testops-verification/v1', ...context,
@@ -291,3 +300,40 @@ export async function verifyPilotUpload(api, selection, context, exported, { now
}
reject('upload_unverified');
}
+
+export async function verifyPilotUpload(api, selection, context, exported, options) {
+ return verifyUploadScope(api, selection, context, exported, undefined, options);
+}
+
+export async function verifyPilotCaseUpload(api, selection, context, executionId, exported, options) {
+ if (context.catalog !== 'full' || typeof executionId !== 'string') reject('stream_scope');
+ return verifyUploadScope(api, selection, context, exported, executionId, options);
+}
+
+// Inspect a stable current launch snapshot before the one allowed upload. Any
+// existing row for this case requires readback, never another blind POST.
+export async function inspectPilotCaseUpload(api, selection, context, executionId, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) {
+ if (context.catalog !== 'full') reject('stream_scope');
+ const authorized = validateSelection(selection, context, selection.allureJobRunId);
+ const row = authorized.find(value => value.executionId === executionId);
+ if (!row) reject('stream_scope');
+ const deadline = now() + 120_000 + (authorized.length > 3
+ ? (2 + Math.ceil(authorized.length / 100) + 2 * Math.ceil(authorized.length / 4)) * 2000 : 0);
+ const get = route => {
+ const remaining = deadline - now(); if (remaining < 1) reject('upload_unverified');
+ return api(route, 'GET', undefined, Math.min(30_000, remaining));
+ };
+ for (let poll = 0; poll < 12 && now() < deadline; poll++) {
+ const { run, launch } = await uploadBinding(get, selection, context);
+ if (launch.closed === true) reject('launch_mismatch');
+ const rows = await resultRows(get, selection.launchId, authorized.length, true);
+ if (rows !== null) {
+ if (new Set(rows.map(value => value.id)).size !== rows.length
+ || new Set(rows.map(value => value.testCaseId)).size !== rows.length) reject('result_page');
+ if (rows.some(value => !positive(value.id) || !authorized.some(expected => expected.caseId === value.testCaseId))) reject('foreign_result');
+ return { present: rows.some(value => value.testCaseId === row.caseId), jobRunStage: run.stage };
+ }
+ if (poll < 11 && now() + 5000 < deadline) await wait(5000);
+ }
+ reject('upload_unverified');
+}
diff --git a/tests/runner-e2e/chat-completion-testops-cli.mjs b/tests/runner-e2e/chat-completion-testops-cli.mjs
index 467700d7b62..3504ae87acc 100644
--- a/tests/runner-e2e/chat-completion-testops-cli.mjs
+++ b/tests/runner-e2e/chat-completion-testops-cli.mjs
@@ -7,6 +7,7 @@ import { admitDispatch, hash, reject, validateSelection, evidencePolicy, catalog
import { testopsClient, bootstrapPilotPlan, completePilotRegistration, validatePilotSelection, launchPilotPlan, verifyPilotUpload } from './chat-completion-testops-api.mjs';
import { parseOfficialBootstrapContext } from './testops-chat-reporter.mjs';
import { exportPilotAllure, readPilotExport } from './chat-completion-testops-results.mjs';
+import { prepareCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs';
const execute = promisify(execFile);
const fullCatalogFailureCodes = new Set([
@@ -31,12 +32,12 @@ export function executionPartition(selection, context, jobRunId) {
const matrix = executionMatrix(selection, context, jobRunId);
if (context.catalog !== 'full') reject('catalog');
const small = matrix.include.length <= 3;
- const native = { include: matrix.include.filter(row => small || !row.external) };
- const external = { include: matrix.include.filter(row => !small && row.external) };
+ const native = { include: matrix.include.filter(row => !row.external) };
+ const external = { include: matrix.include.filter(row => row.external) };
const size = [matrix, native, external, selection.cases.map(row => row.executionId)]
.reduce((sum, value) => sum + JSON.stringify(value).length * 2, 0);
if (size > 1_000_000) reject('matrix_size');
- return { matrix, native, external, parallelism: small ? '1' : '8' };
+ return { matrix, native, external, parallelism: small ? external.include.length ? '1' : '2' : '8' };
}
async function readOwned(file, root, limit) {
const absolute = path.resolve(file), base = await realpath(root);
@@ -92,7 +93,8 @@ function mutationMarker(root) {
export async function run(command, argument, env = process.env, { fetchImpl = fetch, exec = execute } = {}) {
let state;
try {
- if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload'].includes(command)) reject('operation');
+ if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload',
+ 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command)) reject('operation');
const context = await actualContext(env, exec);
if (command === 'guard') {
await output(env, 'operation', context.operation); await output(env, 'target_sha', context.targetSha);
@@ -104,8 +106,15 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe
}
if (['bootstrap', 'bootstrap-context'].includes(command) && context.operation !== 'bootstrap'
|| command === 'launch-plan' && context.operation !== 'launch-plan'
- || ['selection', 'upload', 'verify-upload'].includes(command) && context.operation !== 'evaluate') reject('operation');
+ || ['selection', 'upload', 'verify-upload', 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command) && context.operation !== 'evaluate') reject('operation');
state = await stateDirectory(env, command === 'bootstrap-context' ? 'bootstrap' : command, command === 'bootstrap-context');
+ if (command === 'prepare-case-upload') {
+ if (Object.hasOwn(env, 'ALLURE_TOKEN')) reject('worker_credential');
+ const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context);
+ const prepared = await prepareCaseBundle(path.resolve(argument), env.ALLURE_RESULTS, context, selected,
+ env.CHAT_TESTOPS_EXECUTION_ID, env.EXPECTED_SELECTION_SHA256);
+ await writeReceipt(state, 'prepared.json', prepared); return 0;
+ }
const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl);
if (command === 'bootstrap') {
const prepared = await bootstrapPilotPlan(api, Number(env.CHAT_TESTOPS_INTEGRATION_ID), mutationMarker(state), context);
@@ -144,6 +153,24 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe
await output(env, 'policy', JSON.stringify(context.catalog === 'full' ? null : evidencePolicy(selection, context, env.ALLURE_JOB_RUN_ID))); return 0;
}
const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context);
+ if (command === 'upload-case') {
+ const prepared = JSON.parse(await readOwned(env.CHAT_TESTOPS_PREPARED_FILE, env.RUNNER_TEMP, 65_536));
+ const publication = await publishCaseBundle(api, path.resolve(argument), env.ALLURE_RESULTS, prepared, context, selected,
+ env.CHAT_TESTOPS_EXECUTION_ID, env.EXPECTED_SELECTION_SHA256, () => mutationMarker(state)('upload_case'));
+ await writeReceipt(state, 'publication.json', publication);
+ return publication.exported.status === 'passed' ? 0 : 1;
+ }
+ if (command === 'verify-stream') {
+ const exported = await publicationSet(env.CHAT_TESTOPS_BUNDLES, env.CHAT_TESTOPS_PUBLICATIONS, context, selected, env.EXPECTED_SELECTION_SHA256);
+ const { campaign } = await readPilotExport(path.resolve(argument), context, selected);
+ if (exported.some(row => {
+ const original = campaign.results.find(value => value.executionId === row.executionId);
+ return !original || row.status !== (original.status === 'passed' && original.cleanup === 'passed' ? 'passed' : 'failed') || row.cleanup !== original.cleanup;
+ })) reject('upload_receipt');
+ const verification = await verifyPilotUpload(api, selected, context, exported);
+ await writeReceipt(state, 'verification.json', verification);
+ return exported.some(row => row.status !== 'passed') ? 1 : 0;
+ }
if (command === 'upload') {
const exported = await exportPilotAllure(path.resolve(argument), env.ALLURE_RESULTS, context, selected);
await writeReceipt(state, 'exported.json', exported); return exported.some(row => row.status !== 'passed') ? 1 : 0;
@@ -161,6 +188,11 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe
const code = fullCatalogFailureCodes.has(error?.message)
|| /^(chat_completion_testops_[a-z_]{1,80}|testops_(?:api|auth)_http(?:_[1-5][0-9]{2})?)$/.test(error?.message ?? '') ? error.message : 'chat_completion_testops_unknown';
const receipt = { command, code, httpStatus: Number.isInteger(error?.status) ? error.status : null };
+ if (code === 'chat_completion_testops_stream_upload_uncertain'
+ && ['testops_api_http', 'testops_response_invalid', 'testops_response_too_large', 'transport_or_response_failure'].includes(error.uploadFailure?.kind)) {
+ receipt.uploadFailure = { kind: error.uploadFailure.kind,
+ httpStatus: Number.isInteger(error.uploadFailure.httpStatus) ? error.uploadFailure.httpStatus : null };
+ }
if (state) await writeReceipt(state, 'failure.json', receipt).catch(() => {});
process.stderr.write(`Chat completion TestOps stopped: ${JSON.stringify(receipt)}\n`); return 1;
}
diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs
index c1199bbeb40..5902a391c89 100644
--- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs
+++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs
@@ -98,13 +98,15 @@ test('the executable guard binds the actual original GitHub run before service o
test('target checkout cannot populate or overwrite the trusted helper path', async () => {
const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8'));
+ const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8'));
+ const jobs = [definition.jobs.build, called.jobs.chat];
const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-')));
try {
const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' });
git('init'); await writeFile(path.join(root, 'fixture.txt'), 'normal target'); git('add', '.');
git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'normal target');
- for (const job of [definition.jobs.build, definition.jobs.chat]) {
- const target = job.steps.findIndex(step => step.with?.ref === '${{ needs.authorize.outputs.target_sha }}');
+ for (const job of jobs) {
+ const target = job.steps.findIndex(step => ['${{ needs.authorize.outputs.target_sha }}', '${{ inputs.target_sha }}'].includes(step.with?.ref));
const boundary = job.steps.findIndex(step => step.id === 'target_boundary');
const trusted = job.steps.findIndex(step => step.with?.path === '.testops-trusted');
assert.ok(target >= 0 && boundary > target && trusted > boundary, 'Reject reserved target paths, then check out trusted main last');
@@ -114,7 +116,7 @@ test('target checkout cannot populate or overwrite the trusted helper path', asy
await mkdir(path.join(root, '.testops-trusted'));
await writeFile(path.join(root, '.testops-trusted', 'helper.js'), 'candidate-controlled helper'); git('add', '.testops-trusted');
git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'reserved path in target');
- for (const job of [definition.jobs.build, definition.jobs.chat]) {
+ for (const job of jobs) {
const guard = job.steps.find(step => step.id === 'target_boundary');
assert.throws(() => execFileSync('bash', ['-euo', 'pipefail', '-c', guard.run], { cwd: root, stdio: 'pipe' }));
}
diff --git a/tests/runner-e2e/chat-completion-testops-host-control.mjs b/tests/runner-e2e/chat-completion-testops-host-control.mjs
index c72f7183aa9..bae0cdb5fe9 100644
--- a/tests/runner-e2e/chat-completion-testops-host-control.mjs
+++ b/tests/runner-e2e/chat-completion-testops-host-control.mjs
@@ -5,6 +5,25 @@ import { pathToFileURL } from 'node:url';
import { parse } from 'yaml';
import { model, catalogDefinition, catalogWorkflow } from './chat-completion-testops-contract.mjs';
+export function paidWorker(definition, called) {
+ const path = './.github/workflows/chat-catalog-testops-case.yml';
+ for (const name of ['chat', 'chat_external']) {
+ const job = definition.jobs?.[name];
+ if (job?.uses !== path || job.with?.protocol !== 'streaming-v1'
+ || job.with?.target_sha !== '${{ needs.authorize.outputs.target_sha }}'
+ || job.with?.selection_artifact_id !== '${{ needs.select.outputs.artifact_id }}'
+ || job.with?.selection_sha256 !== '${{ needs.select.outputs.selection_sha256 }}'
+ || JSON.stringify(job.secrets) !== JSON.stringify({ OPENROUTER_API_KEY: '', ANTHROPIC_API_KEY: '', ALLURE_TOKEN: '' })) {
+ throw new Error('Workflow caller boundary mismatch');
+ }
+ }
+ if (!called?.on?.workflow_call || Object.keys(called.on).length !== 1
+ || called.jobs?.chat?.environment !== 'runner-e2e-private-paid'
+ || called.jobs?.report?.environment !== 'runner-e2e-private-history'
+ || called.jobs?.upload?.environment !== 'runner-e2e-testops-reporting') throw new Error('Workflow environment boundary mismatch');
+ return called.jobs.chat;
+}
+
/** A credential-free caller/source control with synthetic IMDS responses.
* It does not qualify an actual AWS host or make a provider call.
*/
@@ -15,7 +34,10 @@ export async function verifyHostContract(workspace, temporaryRoot, env = process
const source = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: workspace, encoding: 'utf8' }).trim();
if (source !== targetSha) throw new Error('Source control pin mismatch');
const definition = parse(await readFile(new URL(`../../${workflow}`, import.meta.url), 'utf8'));
- const paid = definition.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY);
+ // Only the reviewed local callee at the same trusted checkout is read. A
+ // workflow input or another repository/ref cannot supply the worker body.
+ const worker = full ? paidWorker(definition, parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8'))) : definition.jobs.chat;
+ const paid = worker.steps.find(step => step.env?.OPENROUTER_API_KEY);
if (paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS !== 'v1' || paid.env.ALLURE_JOB_RUN_ID !== '${{ inputs.ALLURE_JOB_RUN_ID }}'
|| !paid.run.includes('--max-automatic-retries 0') || definition.jobs.chat.strategy['max-parallel'] !== (full ? '${{ fromJSON(needs.select.outputs.parallelism) }}' : 1)) throw new Error('Workflow host contract mismatch');
const { runnerMatrix } = await import(pathToFileURL(path.join(workspace, 'tests/runner-e2e/catalog.ts')).href);
diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs
index 47fc4939da8..813f6c868c2 100644
--- a/tests/runner-e2e/chat-feature-testops-api.mjs
+++ b/tests/runner-e2e/chat-feature-testops-api.mjs
@@ -85,9 +85,21 @@ export async function testopsClient(token, fetchImpl = fetch) {
return async (route, method = 'GET', body, timeoutMs = 30_000) => {
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30_000) fail('chat_testops_upload_unverified');
if (!route.startsWith('/api/') || route.includes('://')) fail('testops_route_invalid');
+ const multipart = body instanceof FormData;
+ if (multipart) {
+ const entries = [...body.entries()], files = entries.map(([, file]) => file);
+ if (method !== 'POST' || !/^\/api\/jobrun\/[1-9][0-9]{0,15}\/upload$/.test(route)
+ || !Number.isSafeInteger(Number(route.split('/')[3])) || entries.length !== 3
+ || entries.some(([name]) => name !== 'file')
+ || files.some(file => !(file instanceof Blob) || file.type !== 'application/json' || file.size > 12 * 1024 * 1024
+ || !/^[a-f0-9-]{36}-(?:attachment|result)\.json$/.test(file.name))
+ || files.filter(file => file.name.endsWith('-result.json')).length !== 1
+ || new Set(files.map(file => file.name)).size !== 3
+ || files.reduce((bytes, file) => bytes + file.size, 0) > 32 * 1024 * 1024) fail('testops_route_invalid');
+ }
const response = await fetchImpl(`${endpoint}${route}`, { method, redirect: 'error',
- headers: { Accept: 'application/json', Authorization: `Bearer ${value.access_token}`, ...(body ? { 'Content-Type': 'application/json' } : {}) },
- ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(timeoutMs) });
+ headers: { Accept: 'application/json', Authorization: `Bearer ${value.access_token}`, ...(body && !multipart ? { 'Content-Type': 'application/json' } : {}) },
+ ...(body ? { body: multipart ? body : JSON.stringify(body) } : {}), signal: AbortSignal.timeout(timeoutMs) });
if (!response.ok) fail(`testops_api_http_${response.status}`);
if (response.status === 204) return null;
const text = await response.text();
diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs
index 68f689beee6..6864aeb0ba4 100644
--- a/tests/runner-e2e/chat-feature-testops-api.test.mjs
+++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs
@@ -794,3 +794,31 @@ test('deadline crossing after a complete attachment GET keeps the exact readines
assert.equal(projected.verification.attachmentReadiness[0].withinDeadline, false); return true;
});
});
+
+test('existing-jobrun upload uses repeated literal file fields without JSON encoding or a lifecycle request', async () => {
+ const calls = [];
+ const api = await testopsClient('PRIVATE_KEY', async (url, options) => {
+ calls.push({ url, options });
+ return calls.length === 1 ? { ok: true, json: async () => ({ access_token: 'PRIVATE_BEARER' }) }
+ : { ok: true, status: 200, text: async () => '{"filesCount":3}' };
+ });
+ const data = new FormData();
+ for (const suffix of ['result', 'attachment', 'attachment']) data.append('file', new Blob(['{}'], { type: 'application/json' }), `${crypto.randomUUID()}-${suffix}.json`);
+ assert.deepEqual(await api('/api/jobrun/77/upload', 'POST', data), { filesCount: 3 });
+ assert.equal(calls.length, 2);
+ assert.equal(calls[1].url, `${endpoint}/api/jobrun/77/upload`);
+ assert.equal(calls[1].options.body, data);
+ assert.equal(calls[1].options.headers['Content-Type'], undefined);
+ assert.deepEqual([...calls[1].options.body.keys()], ['file', 'file', 'file']);
+ assert.equal(calls[1].options.body.getAll('file[]').length, 0);
+ assert.equal(calls[1].options.redirect, 'error');
+});
+
+test('multipart uploads cannot target another route or add undeclared parts', async () => {
+ let calls = 0;
+ const api = await testopsClient('PRIVATE_KEY', async () => { calls++; return { ok: true, json: async () => ({ access_token: 'PRIVATE_BEARER' }) }; });
+ const form = new FormData(); form.append('file[]', new Blob(['{}']), 'unknown.json');
+ await assert.rejects(api('/api/jobrun/77/upload', 'POST', form), /testops_route_invalid/);
+ await assert.rejects(api('/api/testcase', 'POST', form), /testops_route_invalid/);
+ assert.equal(calls, 1, 'only the existing authentication request');
+});
diff --git a/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs
index 9a6f803a7db..be23b411a6e 100644
--- a/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs
+++ b/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs
@@ -2,7 +2,7 @@ import test from 'node:test';
import assert from 'node:assert/strict';
import { fullCatalogCases, fullCatalogPlanName, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs';
import { admitDispatch, validateSelection, evidencePolicy, catalogWorkflow, catalogDefinition } from './chat-completion-testops-contract.mjs';
-import { bindingFromEnv, validatePilotSelection, verifyPilotUpload, completePilotRegistration } from './chat-completion-testops-api.mjs';
+import { bindingFromEnv, validatePilotSelection, verifyPilotUpload, verifyPilotCaseUpload, completePilotRegistration } from './chat-completion-testops-api.mjs';
import { executionMatrix, executionPartition, run } from './chat-completion-testops-cli.mjs';
import { mkdtemp, writeFile, readFile, readdir, rm, realpath } from 'node:fs/promises';
import os from 'node:os';
@@ -103,14 +103,15 @@ test('full catalog source admission is independent from the old three-case workf
assert.throws(() => validateSelection({ ...selection, targetSha: 'b'.repeat(40) }, context, '77'), /selection_binding/);
});
-test('native and external partitions preserve exact selections, serial small runs and the GitHub output bound', () => {
+test('native and external partitions preserve exact selections, bounded overlapping small runs and the GitHub output bound', () => {
for (const rows of [selected, selected.slice(0, 3), selected.filter(row => row.externalInput), [selected[0], selected.find(row => row.externalInput)]]) {
const input = { ...selection, cases: rows }, partition = executionPartition(input, context, '77');
const merged = [...partition.native.include, ...partition.external.include].map(row => row.executionId);
assert.equal(new Set(merged).size, rows.length);
assert.deepEqual(merged.sort(), rows.map(row => row.executionId).sort());
if (rows.length <= 3) {
- assert.equal(partition.parallelism, '1'); assert.equal(partition.external.include.length, 0);
+ assert.equal(partition.parallelism, rows.some(row => row.externalInput) ? '1' : '2');
+ assert.equal(partition.external.include.length, rows.filter(row => row.externalInput).length);
} else {
assert.equal(partition.parallelism, '8'); assert.ok(partition.external.include.every(row => row.external));
}
@@ -224,3 +225,36 @@ test('the real bootstrap facade preserves fixed reconciliation failures and disc
} finally { capture.mock.restore(); await rm(root, { recursive: true, force: true }); }
}
});
+
+test('a finished case verifies while known sibling rows coexist in the same launch', async () => {
+ const f = api();
+ const value = await verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]);
+ assert.equal(value.verification, 'VERIFIED');
+ assert.deepEqual(value.results.map(row => row.caseId), [selected[0].caseId]);
+ assert.equal(f.calls.filter(route => /^\/api\/testresult\/\d+$/.test(route)).length, 1);
+});
+
+test('partial verification rejects foreign sibling identities before qualifying the current case', async () => {
+ const base = api(), first = await base.get('/api/testresult?launchId=12&page=0&size=100');
+ first.content[1].testCaseId = 999999;
+ const f = api({ '/api/testresult?launchId=12&page=0&size=100': first });
+ await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /foreign_result/);
+});
+
+test('partial verification retains the exact current-result jobrun binding', async () => {
+ const base = api(), wrong = await base.get('/api/testresult/2000'); wrong.jobRun.id = 78;
+ const f = api({ '/api/testresult/2000': wrong });
+ await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /result_mismatch/);
+});
+
+test('small external selections remain in the serial external lane while native overlap stays bounded', () => {
+ const external = selected.filter(row => row.externalInput).slice(0, 2);
+ assert.equal(external.length, 2);
+ for (const rows of [external, [selected.find(row => !row.externalInput), ...external]]) {
+ const partition = executionPartition({ ...selection, cases: rows }, context, '77');
+ assert.deepEqual(partition.external.include.map(row => row.executionId), external.map(row => row.executionId));
+ assert.ok(partition.native.include.every(row => !row.external));
+ assert.equal(partition.parallelism, '1');
+ assert.equal(partition.native.include.length + partition.external.include.length, rows.length);
+ }
+});
diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs
index c45a1d8a69d..b0c55f687c6 100644
--- a/tests/runner-e2e/run-unit.mjs
+++ b/tests/runner-e2e/run-unit.mjs
@@ -24,7 +24,7 @@ for (const [name, command] of [
"tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs",
"tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs",
"tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs",
- "tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-reconcile.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs"]],
+ "tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-reconcile.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs", "tests/runner-e2e/chat-catalog-testops-streaming.test.mjs"]],
]) {
const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" });
if (result.error || result.signal) {
diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts
index 2cd918751d2..42707b008d3 100644
--- a/tests/runner-e2e/vitest.config.ts
+++ b/tests/runner-e2e/vitest.config.ts
@@ -7,7 +7,7 @@ export default defineConfig({
globalSetup: ["./verified-provider-bundles.setup.mjs"],
include: ["**/*.test.ts", "**/*.test.mjs"],
// These use node:test and run separately in test:e2e:runner:unit.
- exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs"],
+ exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"],
testTimeout: 30_000,
},
});