diff --git a/.github/infra/runner-e2e-private-history.yml b/.github/infra/runner-e2e-private-history.yml new file mode 100644 index 00000000000..0d837d24793 --- /dev/null +++ b/.github/infra/runner-e2e-private-history.yml @@ -0,0 +1,117 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Private bounded Paperclip eval evidence and reports; no CDN reader +Parameters: + BucketName: + Type: String + GitHubRepository: + Type: String + Default: paperclipai/paperclip + GitHubOidcProviderArn: + Type: String +Resources: + PrivateHistoryBucket: + Type: AWS::S3::Bucket + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + BucketName: !Ref BucketName + PublicAccessBlockConfiguration: + BlockPublicAcls: true + IgnorePublicAcls: true + BlockPublicPolicy: true + RestrictPublicBuckets: true + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + VersioningConfiguration: + Status: Enabled + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + LifecycleConfiguration: + Rules: + - Id: BoundedEvalRetentionV1 + Status: Enabled + Prefix: runner-e2e-private/bounded-v1/ + ExpirationInDays: 30 + NoncurrentVersionExpiration: + NoncurrentDays: 1 + AbortIncompleteMultipartUpload: + DaysAfterInitiation: 1 + - Id: BoundedEvalDeleteMarkersV1 + Status: Enabled + Prefix: runner-e2e-private/bounded-v1/ + ExpiredObjectDeleteMarker: true + PrivateHistoryBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref PrivateHistoryBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: DenyInsecureTransport + Effect: Deny + Principal: '*' + Action: s3:* + Resource: + - !GetAtt PrivateHistoryBucket.Arn + - !Sub '${PrivateHistoryBucket.Arn}/*' + Condition: + Bool: + aws:SecureTransport: 'false' + - Sid: DenyUnconditionalWrites + Effect: Deny + Principal: '*' + Action: s3:PutObject + Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*' + Condition: + 'Null': + s3:if-none-match: 'true' + PrivateHistoryRole: + Type: AWS::IAM::Role + Properties: + RoleName: paperclip-runner-e2e-private-history-github + MaxSessionDuration: 3600 + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Federated: !Ref GitHubOidcProviderArn + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + token.actions.githubusercontent.com:sub: + - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-paid' + - !Sub 'repo:${GitHubRepository}:environment:runner-e2e-private-history' + Policies: + - PolicyName: private-eval-artifacts + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: VerifyPrivateRetention + Effect: Allow + Action: + - s3:GetBucketVersioning + - s3:GetLifecycleConfiguration + - s3:GetBucketObjectLockConfiguration + - s3:GetReplicationConfiguration + - s3:GetBucketPublicAccessBlock + - s3:GetBucketOwnershipControls + - s3:GetBucketPolicy + Resource: !GetAtt PrivateHistoryBucket.Arn + - Sid: RetainAndReadPrivateArtifacts + Effect: Allow + Action: + - s3:GetObject + - s3:PutObject + Resource: !Sub '${PrivateHistoryBucket.Arn}/runner-e2e-private/bounded-v1/*' +Outputs: + Bucket: + Value: !Ref PrivateHistoryBucket + RoleArn: + Value: !GetAtt PrivateHistoryRole.Arn + Region: + Value: !Ref AWS::Region diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index c15a34085fa..a04db993e7c 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -119,7 +119,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 65eebf32ed3..99cdaf52326 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -1,10 +1,17 @@ name: Runner Full-Stack E2E +run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }} on: schedule: - - cron: "47 8 * * 0" + - cron: "47 8 * * *" + - cron: "47 9 * * 0" workflow_dispatch: inputs: + cadence: + description: "Bounded daily ($20) or weekly ($100) selection; manual keeps explicit selectors" + type: choice + options: [manual, daily, weekly] + default: manual target_branch: description: "Branch in paperclipai/paperclip to test; the trusted workflow still runs from master" type: string @@ -55,7 +62,7 @@ concurrency: jobs: authorize: name: Authorize paid campaign - if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') runs-on: ubuntu-latest timeout-minutes: 5 permissions: @@ -115,14 +122,25 @@ jobs: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + EVENT_NAME: ${{ github.event_name }} + EVAL_CADENCE: ${{ inputs.cadence || 'manual' }} run: | set -euo pipefail if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then echo "target_branch must name a branch in this repository without a refs/ prefix." >&2 exit 1 fi - encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')" - target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)" + if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then + if [ "$TARGET_BRANCH" != "$DEFAULT_BRANCH" ]; then + echo "Bounded campaigns require the workflow's default-branch commit." >&2 + exit 1 + fi + target_sha="$GITHUB_SHA" + else + encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')" + target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)" + fi if ! [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]]; then echo "The requested repository branch did not resolve to a commit." >&2 exit 1 @@ -135,12 +153,23 @@ jobs: id: runner env: AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} + EVENT_NAME: ${{ github.event_name }} + EVAL_CADENCE: ${{ inputs.cadence || 'manual' }} + REPOSITORY_PRIVATE: ${{ github.event.repository.private }} run: | set -euo pipefail github_runner='ubuntu-latest' aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' + bounded=false + if [ "$EVENT_NAME" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then + if [ "$REPOSITORY_PRIVATE" != false ]; then + echo 'Bounded campaigns require a public repository and standard GitHub-hosted compute.' >&2 + exit 1 + fi + bounded=true + fi - if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then + if [ "$AWS_PAID_RUNNER_ENABLED" = true ] && [ "$bounded" = false ]; then { echo "runner=$aws_runner" echo "max_parallel_default=100" @@ -155,7 +184,7 @@ jobs: echo "max_parallel_limit=57" echo "playwright_channel=" } >> "$GITHUB_OUTPUT" - echo '::notice title=Paid runner routing::RUNNER_E2E_AWS_ENABLED is not true; using the proven GitHub-hosted runner' + echo '::notice title=Paid runner routing::Using a standard GitHub-hosted runner' fi target_lock: @@ -200,6 +229,10 @@ jobs: fi echo "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" + - name: Bound scheduled lockfile upload + if: github.event_name == 'schedule' || (inputs.cadence != '' && inputs.cadence != 'manual') + run: node tests/runner-e2e/artifact-budget.mjs check lock pnpm-lock.yaml + - name: Upload resolved target lockfile id: upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -212,13 +245,15 @@ jobs: catalog: name: Validate catalog and select cells needs: [authorize, target_lock] - if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read outputs: matrix: ${{ steps.catalog.outputs.matrix }} + operations_cadence: ${{ steps.catalog.outputs.operations_cadence }} + operating_overhead: ${{ steps.catalog.outputs.operating_overhead }} needs_daytona: ${{ steps.catalog.outputs.needs_daytona }} needs_runner_typescript: ${{ steps.catalog.outputs.needs_runner_typescript }} needs_native_binaries: ${{ steps.catalog.outputs.needs_native_binaries }} @@ -281,6 +316,13 @@ jobs: id: catalog env: EVENT_NAME: ${{ github.event_name }} + EVAL_EVENT: ${{ github.event_name }} + EVAL_SCHEDULE: ${{ github.event.schedule }} + EVAL_CADENCE: ${{ inputs.cadence || 'manual' }} + EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }} + EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }} + EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} SELECT_ALL: ${{ inputs.all }} SELECT_SUITE: ${{ inputs.suite }} SELECT_GROUP: ${{ inputs.group }} @@ -293,6 +335,31 @@ jobs: REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }} run: | set -euo pipefail + if [ "$EVAL_EVENT" = schedule ] || [ "$EVAL_CADENCE" != manual ]; then + test "$TARGET_SHA" = "$GITHUB_SHA" + test -z "${SELECT_SUITE:-}${SELECT_GROUP:-}${SELECT_PROFILE:-}${SELECT_ENVIRONMENT:-}${SELECT_CASE:-}${SELECT_ID:-}" + catalog_json="$(node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts matrix)" + cadence="$EVAL_CADENCE" + if [ "$EVAL_EVENT" = schedule ]; then + case "$EVAL_SCHEDULE" in + '47 8 * * *') cadence=daily ;; + '47 9 * * 0') cadence=weekly ;; + *) exit 1 ;; + esac + fi + { + echo "operations_cadence=$cadence" + echo "operating_overhead=$(jq -c '.overhead' <<< "$catalog_json")" + echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" + echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" + echo "needs_daytona=false" + echo "needs_runner_typescript=true" + echo "needs_native_binaries=true" + echo "needs_remote_provider_pack=false" + echo "max_parallel=1" + } >> "$GITHUB_OUTPUT" + exit 0 + fi args=(--matrix-json) add_values() { local flag="$1" @@ -509,7 +576,7 @@ jobs: build_runner_artifacts: name: Build reusable runner campaign artifacts needs: [authorize, target_lock, catalog] - if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') # Compile native binaries on the same reviewed image used to execute them, # avoiding libc/architecture drift between GitHub-hosted and AWS lanes. runs-on: ${{ needs.authorize.outputs.test_runner }} @@ -618,6 +685,10 @@ jobs: TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} run: echo "name=runner-e2e-build-${TARGET_SHA}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT" + - name: Bound scheduled build upload + if: needs.catalog.outputs.operations_cadence != '' + run: node tests/runner-e2e/artifact-budget.mjs check build runner-e2e-build-bundle.tar.gz runner-e2e-build-bundle.tar.gz.sha256 + - name: Upload immutable shared campaign outputs uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -633,7 +704,7 @@ jobs: name: Build reusable remote provider pack needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts] - if: github.event_name != 'schedule' || vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' + if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -769,6 +840,7 @@ jobs: test: name: ${{ matrix.executionId }} + if: needs.catalog.outputs.operations_cadence == '' needs: [ authorize, @@ -790,7 +862,7 @@ jobs: fail-fast: false max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }} matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }} - steps: + steps: &paid_test_steps - name: Reauthorize paid execution before provider access env: GH_TOKEN: ${{ github.token }} @@ -962,8 +1034,8 @@ jobs: } NODE - - name: Install pinned legacy Claude CLI - if: matrix.profileId == 'legacy-claude' + - name: Install pinned Claude CLI for setup verification + if: matrix.profileId == 'legacy-claude' || matrix.profileId == 'runner-acpx-claude' run: | npm install --global --omit=dev --ignore-scripts @anthropic-ai/claude-code@2.1.277 launcher="$(npm root --global)/@anthropic-ai/claude-code/cli-wrapper.cjs" @@ -1012,7 +1084,7 @@ jobs: # This definition executes only from the authorized default-branch workflow. # Provision host policy before credentials reach target-controlled tests. - name: Provision Codex sandbox on the disposable trusted runner - if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription') + if: matrix.environmentId == 'local' && (needs.catalog.outputs.operations_cadence == 'weekly' || matrix.profileId == 'runner-codex' || matrix.profileId == 'runner-codex-mini' || matrix.profileId == 'runner-acpx-codex' || matrix.profileId == 'runner-codex-subscription') run: | node --input-type=module <<'NODE' import { execFileSync } from "node:child_process"; @@ -1052,7 +1124,60 @@ jobs: timeout 120s docker pull "$oracle_image" timeout 30s docker image inspect "$oracle_image" --format '{{.Id}}' + - name: Obtain scoped private history credentials + id: private_history_credentials + if: needs.catalog.outputs.operations_cadence != '' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Verify private evidence retention before paid execution + if: needs.catalog.outputs.operations_cadence != '' + env: + AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts verify evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence + + - name: Prepare bounded campaign before credentials + if: needs.catalog.outputs.operations_cadence != '' + env: + EVAL_EVENT: ${{ github.event_name }} + EVAL_SCHEDULE: ${{ github.event.schedule }} + EVAL_CADENCE: ${{ needs.catalog.outputs.operations_cadence }} + EVAL_OPERATIONS_ENABLED: ${{ vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED }} + EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }} + EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }} + PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-operations.ts prepare + + - name: Run bounded campaign and retain feedback review + if: needs.catalog.outputs.operations_cadence != '' + env: + BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} + PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: ${{ vars.RUNNER_E2E_BRAINTRUST_PROJECT_ID }} + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} + PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }} + run: | + set +e + args=(--max-parallel 1 --max-automatic-retries 0) + while IFS= read -r id; do args+=(--id "$id"); done < <(jq -r '.executionIds[]' "$PAPERCLIP_E2E_OPERATIONS_MANIFEST") + pnpm test:e2e:runner -- "${args[@]}" + run_status=$? + node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/scheduled-finalization.ts + finalization_status=$? + test "$run_status" = 0 && test "$finalization_status" = 0 + - name: Run paid cell + if: needs.catalog.outputs.operations_cadence == '' env: OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} @@ -1063,13 +1188,19 @@ jobs: PAPERCLIP_E2E_DAYTONA_IMAGE: ${{ needs.daytona_image.outputs.image }} PAPERCLIP_RUNNER_REMOTE_PROVIDER_PACK_PATH: ${{ github.workspace }}/packages/paperclip-runner/provider-pack PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} + PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }} run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" + - name: Stage bounded campaign evidence + id: stage_evidence + if: always() && needs.catalog.outputs.operations_cadence != '' + run: node tests/runner-e2e/artifact-budget.mjs stage evidence tests/runner-e2e/results bounded-runner-e2e-evidence + - name: Upload access-controlled packaged cell evidence - if: always() + if: always() && needs.catalog.outputs.operations_cadence == '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} @@ -1077,10 +1208,41 @@ jobs: retention-days: 30 if-no-files-found: error + - name: Retain bounded campaign evidence in private AWS storage + if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success' + env: + AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload evidence "$GITHUB_RUN_ID" bounded-runner-e2e-evidence + + - name: Enforce complete bounded evidence retention + if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_evidence.outputs.complete != 'true' + run: exit 1 + + test_recurring: + name: ${{ matrix.executionId }} + if: needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack] + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: ${{ matrix.timeoutMinutes }} + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-private-paid + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }} + steps: *paid_test_steps + report: name: Merge and enforce campaign result - if: always() && !cancelled() && needs.catalog.result == 'success' - needs: [authorize, catalog, daytona_image, test] + if: always() && !cancelled() && needs.catalog.result == 'success' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + needs: [authorize, catalog, daytona_image, test, test_recurring] outputs: history_source_ready: ${{ steps.history_source_ready.outputs.ready }} runs-on: ubuntu-latest @@ -1088,6 +1250,9 @@ jobs: permissions: actions: read contents: read + id-token: write + environment: + name: runner-e2e-private-history steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -1107,15 +1272,8 @@ jobs: with: version: 9.15.4 - - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - cache: pnpm - - # Resolve from this trusted checkout only. Never use the target branch's - # lockfile or install scripts in the publication job. - - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile - - run: pnpm install --frozen-lockfile + - name: Install pinned reporting dependencies + run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store" - name: Resolve workflow job attempts env: @@ -1138,6 +1296,7 @@ jobs: - name: Download cell evidence id: download_evidence + if: needs.catalog.outputs.operations_cadence == '' continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1146,15 +1305,37 @@ jobs: merge-multiple: false - name: Retry cell evidence download after transport failure - if: steps.download_evidence.outcome == 'failure' + if: needs.catalog.outputs.operations_cadence == '' && steps.download_evidence.outcome == 'failure' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: runner-e2e-${{ github.run_id }}-*-* path: downloaded-runner-e2e merge-multiple: false + - name: Obtain scoped private history credentials + id: private_history_credentials + if: needs.catalog.outputs.operations_cadence != '' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download private bounded campaign evidence + if: needs.catalog.outputs.operations_cadence != '' + env: + AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download evidence "$GITHUB_RUN_ID" "downloaded-runner-e2e/runner-e2e-$GITHUB_RUN_ID-1-scheduled-campaign" + - name: Select latest workflow attempt per cell - if: always() + if: always() && needs.catalog.outputs.operations_cadence == '' env: PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e @@ -1164,6 +1345,25 @@ jobs: PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/select-rerun-artifacts.ts + - name: Preserve bounded campaign evidence and journal + if: always() && needs.catalog.outputs.operations_cadence != '' + env: + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + artifact="downloaded-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign" + directory="$GITHUB_WORKSPACE/selected-runner-e2e/runner-e2e-$RUN_ID-1-scheduled-campaign/scheduled" + echo "PAPERCLIP_E2E_OPERATIONS_LEDGER=$directory/ledger.jsonl" >> "$GITHUB_ENV" + test -d "$artifact" + mkdir -p selected-runner-e2e + cp -R "$artifact" selected-runner-e2e/ + if [ -f "$directory/feedback-review.json" ]; then + echo "PAPERCLIP_E2E_FEEDBACK_REVIEW=$directory/feedback-review.json" >> "$GITHUB_ENV" + fi + if [ -f "$directory/experiment-receipt.json" ]; then + echo "PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT=$directory/experiment-receipt.json" >> "$GITHUB_ENV" + fi + - name: Collect blob reports run: | set -euo pipefail @@ -1188,21 +1388,34 @@ jobs: PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.catalog.outputs.execution_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-${{ github.run_attempt }} + PAPERCLIP_E2E_CAMPAIGN_ID: ${{ needs.catalog.outputs.operations_cadence != '' && format('gha-{0}-1-scheduled-campaign', github.run_id) || format('gha-{0}-{1}', github.run_id, github.run_attempt) }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} - PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }} PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }} + EVAL_OPERATIONS_CADENCE: ${{ needs.catalog.outputs.operations_cadence }} run: | set +e - pnpm test:e2e:runner:report + if [ -n "$EVAL_OPERATIONS_CADENCE" ]; then + mkdir -p runner-e2e-merged-report + pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 + else + pnpm test:e2e:runner:report + fi report_status=$? set -e - cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY" + if [ -z "$EVAL_OPERATIONS_CADENCE" ]; then + cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY" + fi exit "$report_status" + - name: Stage bounded merged report + id: stage_report + if: always() && needs.catalog.outputs.operations_cadence != '' + run: node tests/runner-e2e/artifact-budget.mjs stage report runner-e2e-merged-report bounded-runner-e2e-report + - name: Upload access-controlled merged report - if: always() + if: always() && needs.catalog.outputs.operations_cadence == '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} @@ -1210,6 +1423,33 @@ jobs: retention-days: 30 if-no-files-found: error + - name: Retain bounded report in private AWS storage + id: private_report + if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_history_credentials.outcome == 'success' + env: + AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + run: | + node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts upload report "$GITHUB_RUN_ID" bounded-runner-e2e-report + echo '{"version":1,"privateReportRetained":true}' > private-report-status.json + echo 'The recurring campaign report is retained in private AWS storage. See the eval operations runbook for authenticated retrieval.' >> "$GITHUB_STEP_SUMMARY" + + - name: Retain report availability for the schedule monitor + if: always() && needs.catalog.outputs.operations_cadence != '' && steps.private_report.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} + path: private-report-status.json + retention-days: 30 + if-no-files-found: error + + - name: Enforce complete bounded report retention + if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_report.outputs.complete != 'true' + run: exit 1 + - name: Verify normalized history source report id: history_source_ready if: always() @@ -1225,7 +1465,7 @@ jobs: publish_history: name: Publish S3 history and Pages bundle with declared screenshots needs: [authorize, catalog, report] - if: always() && needs.catalog.result == 'success' && needs.report.outputs.history_source_ready == 'true' + if: always() && needs.catalog.result == 'success' && needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest timeout-minutes: 15 outputs: @@ -1257,12 +1497,10 @@ jobs: with: version: 9.15.4 - # Resolve the trusted publication checkout's lockfile without lifecycle - # scripts, then install the exact result. This matches the report job's - # frozen-install preparation while keeping target-controlled code out of - # the AWS credentialed publisher. - - run: pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile - - run: pnpm install --frozen-lockfile + # Use a job-local store and disable lifecycle scripts throughout install. + # OIDC is available to every step, even before the AWS credential exchange. + - name: Install pinned publication dependencies + run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-publisher-store" - name: Install publisher-only Chromium run: pnpm exec playwright install --with-deps --only-shell chromium diff --git a/doc/agent-observability.md b/doc/agent-observability.md index 2932330f2dc..2968ee6f507 100644 --- a/doc/agent-observability.md +++ b/doc/agent-observability.md @@ -28,9 +28,13 @@ requires the qualified executable version. Codex ACPX writes its isolated home. The legacy ACPX adapter keeps export configuration in a private home per agent and settings compatibility key, and shares conversation files with its stable managed home; export changes do not change its logical conversation fingerprint. -Codex's bundled Rust OTLP exporter percent-decodes header values, so launch -material encodes them once to preserve literal percent sequences. Credentials -remain literal values in the company secret; environment interpolation is not used. +Codex receives authentication through `OTEL_EXPORTER_OTLP_LOGS_HEADERS` and +`OTEL_EXPORTER_OTLP_TRACES_HEADERS` in its process environment. Its managed +configuration and invocation profiles contain empty header maps, so exporting +does not save the ingestion credential in those files. The bundled Rust OTLP +exporter percent-decodes environment header values; launch material encodes +them once to preserve literal percent sequences. Credentials remain literal +values in the company secret; config-file environment interpolation is not used. A caller-owned/preconfigured Codex home cannot be automatically configured in this version: its files remain untouched and execution emits a diagnostic. Other harnesses, personal Codex desktop sessions and hosted claude_managed are @@ -69,8 +73,8 @@ entered secrets. Revisions prevent another administrator's edit being silently overwritten. Credential values are excluded from task environment manifests, run/config snapshots, argv, activity records, recovery configuration and query cache data. Export credentials are necessarily accessible to the exporting -harness itself. Claude Code supplies these headers through its process environment, -which its tools can inherit or inspect. Codex can read its managed configuration. +harness itself. Both Codex and Claude Code receive these headers through their +process environment, which their tools can inherit or inspect. This feature does not isolate export credentials from agent code. Use a credential limited to telemetry ingestion for the selected destination; do not supply a broad organization administration key. A user-operated collector can hold upstream diff --git a/doc/evals.md b/doc/evals.md index 985d358469f..9d89aa297eb 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -511,3 +511,8 @@ covers projectless task files, deferred repository acquisition, controller resta and nested Git/dirty-file persistence on legacy local, native local and native Daytona OpenCode. Its three configured cells use independent filesystem, public API and download oracles; no live qualification is implied by fixture registration. + +The explicit-only [feedback persistence case](../tests/runner-e2e/README.md#feedback-persistence-explicit-only) +checks real native complaint/suggestion submissions and retained bodies with +run-attributed tool receipts. It is manual, uses the normal Product E2E report, +and keeps synthetic feedback distinct from actionable product findings. diff --git a/packages/adapter-utils/src/agent-observability.test.ts b/packages/adapter-utils/src/agent-observability.test.ts index cb85c54eef9..22fb580541e 100644 --- a/packages/adapter-utils/src/agent-observability.test.ts +++ b/packages/adapter-utils/src/agent-observability.test.ts @@ -23,13 +23,23 @@ it("encodes headers losslessly and clears ambient exporter conflicts without cha expect(observabilitySecretValues(launch)).toContain(input.headers.authorization); expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb"); }); -it("preserves literal percent sequences through the qualified Codex exporter's header decoder", () => { +it("keeps Codex headers in the launch environment and preserves literal percent sequences", () => { const original = "Bearer a,b=%26 c%"; const launch = buildProviderObservabilityLaunch({ ...input, kind: "codex", headers: { authorization: original } }); const exporter = launch.otelConfig!.exporter as { "otlp-http": { headers: Record } }; - expect(decodeURIComponent(exporter["otlp-http"].headers.authorization)).toBe(original); + expect(exporter["otlp-http"].headers).toEqual({}); + for (const key of ["OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS"]) { + expect(launch.environment[key]).toBe(`authorization=${encodeURIComponent(original)}`); + } + expect(JSON.stringify(launch.otelConfig)).not.toContain(original); + expect(JSON.stringify(launch.otelConfig)).not.toContain(encodeURIComponent(original)); expect(observabilitySecretValues(launch)).toContain(original); + expect(observabilitySecretValues(launch)).toContain(encodeURIComponent(original)); expect(observabilitySecretValues(launch)).toContain(original.slice(7)); + expect(JSON.stringify(buildInvocationEnvForLogs({ ...launch.environment }))).not.toContain("a%2Cb"); + const disabled = buildProviderObservabilityLaunch({ ...input, kind: "codex", enabled: false }); + expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + expect(disabled.environment).not.toHaveProperty("OTEL_EXPORTER_OTLP_TRACES_HEADERS"); }); it("replaces a compact explicit profile without leaving competing profile flags", () => { const args = ["exec", "-preview", "-"]; @@ -52,6 +62,8 @@ it("creates invocation-owned profiles, preserves selected profile settings, and const a = await prepareCodexObservabilityProfile({ home: root, runId: "run-a", sourceProfile: "review", launch }); const b = await prepareCodexObservabilityProfile({ home: root, runId: "run-b", launch }); const parsed = parse(await readFile(join(root, a.filename), "utf8")); + expect(JSON.stringify(parsed)).not.toContain(input.headers.authorization); + expect(JSON.stringify(parsed)).not.toContain(encodeURIComponent(input.headers.authorization)); expect(parsed.model).toBe("chosen-model"); expect(parsed.otel).toMatchObject({ log_agent_responses: true, tool_result: { max_bytes: 65536 } }); expect((await stat(join(root, a.filename))).mode & 0o777).toBe(0o600); diff --git a/packages/adapter-utils/src/agent-observability.ts b/packages/adapter-utils/src/agent-observability.ts index aab46b24457..051e7cb2c78 100644 --- a/packages/adapter-utils/src/agent-observability.ts +++ b/packages/adapter-utils/src/agent-observability.ts @@ -35,7 +35,10 @@ export function observabilitySecretValues(launch: ProviderObservabilityLaunch | const headers = ((launch.otelConfig?.exporter as Record | undefined)?.["otlp-http"] as { headers?: Record } | undefined)?.headers; const encoded = launch.environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS; const decoded = (value: string) => { try { return decodeURIComponent(value); } catch { return value; } }; - const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").map(item => decoded(item.slice(item.indexOf("=") + 1)))] : [])]; + const values = [...Object.values(headers ?? {}).flatMap(value => [value, decoded(value)]), ...(encoded ? [encoded, ...encoded.split(",").flatMap(item => { + const value = item.slice(item.indexOf("=") + 1); + return [value, decoded(value)]; + })] : [])]; return [...new Set(values.flatMap(value => [value, value.replace(/^Bearer /i, ""), JSON.stringify(value).slice(1, -1)]).filter(Boolean))]; } @@ -50,10 +53,11 @@ export function buildProviderObservabilityLaunch(input: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000", OTEL_EXPORTER_OTLP_TRACES_TIMEOUT: "3000", }; if (kind === "codex") { - // Codex passes these through opentelemetry-otlp 0.31's with_headers, which - // percent-decodes values. Encode once so literal percent sequences survive. - const headers = Object.fromEntries(Object.entries(input.headers ?? {}).map(([name, value]) => [name, encodeURIComponent(value)])); - const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers } } : "none"; + if (enabled) Object.assign(environment, { + OTEL_EXPORTER_OTLP_LOGS_HEADERS: encodeObservabilityHeaders(input.headers ?? {}), + OTEL_EXPORTER_OTLP_TRACES_HEADERS: encodeObservabilityHeaders(input.headers ?? {}), + }); + const exporter = (endpoint: string | undefined) => enabled ? { "otlp-http": { endpoint, protocol: "binary", headers: {} } } : "none"; return { kind, compatibilityKey, environment, otelConfig: { exporter: exporter(input.logsEndpoint), trace_exporter: exporter(input.tracesEndpoint), metrics_exporter: "none", log_user_prompt: enabled && capture.userPrompts, log_agent_responses: enabled && capture.assistantResponses, diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs index 8f4e4dc9c18..43edd3e0196 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/acpx_sidecar_transport.rs @@ -302,6 +302,8 @@ impl AcpxSidecarTransport { "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG", "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", ]); } let pi_keys = if agent == "pi" { diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 07e5cf81451..e5c77b954be 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -651,6 +651,25 @@ fn run() -> Result<(), Box> { let require_codex_home_auth = args .iter() .any(|value| value == "--require-codex-home-auth"); + if args + .iter() + .any(|value| value == "--require-otel-environment") + { + for (name, expected) in [ + ( + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "authorization=Bearer%20logs-fixture", + ), + ( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "authorization=Bearer%20traces-fixture", + ), + ] { + assert_eq!(std::env::var(name).as_deref(), Ok(expected)); + } + assert!(std::env::var_os("OTEL_EXPORTER_OTLP_HEADERS").is_none()); + assert!(std::env::var_os("OTEL_EXPORTER_OTLP_ENDPOINT").is_none()); + } let durable_turn_ids = args.iter().any(|value| value == "--durable-turn-ids"); let hold_first_durable_turn = args .iter() diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index b6f7675c5e0..70aa6735095 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -823,6 +823,8 @@ const GITHUB_CREDENTIAL_ENVIRONMENT_KEYS: &[&str] = &[ const CODEX_PROVIDER_ENVIRONMENT_KEYS: &[&str] = &[ "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "PAPERCLIP_AI_PROVIDER_KEY", "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index 07c7912635c..5d8ccc10ab7 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -106,6 +106,19 @@ fn provider_receives_the_isolated_codex_auth_home() { .env("PAPERCLIP_CODEX_AUTH_TEST_HOME", &directory) .env("HOME", &directory) .env("CODEX_HOME", &directory) + .env( + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", + "authorization=Bearer%20logs-fixture", + ) + .env( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "authorization=Bearer%20traces-fixture", + ) + .env( + "OTEL_EXPORTER_OTLP_HEADERS", + "authorization=unwanted-ambient", + ) + .env("OTEL_EXPORTER_OTLP_ENDPOINT", "https://unwanted.example") .status() .expect("run isolated auth-home assertion"); assert!(status.success()); @@ -119,7 +132,10 @@ fn provider_receives_isolated_codex_auth_home_subprocess() { else { return; }; - let config = provider_config(&directory, &["--require-codex-home-auth"]); + let config = provider_config( + &directory, + &["--require-codex-home-auth", "--require-otel-environment"], + ); let mut provider = CodexProvider::start(&config, None) .expect("Codex provider should read auth from the isolated CODEX_HOME"); provider.shutdown().expect("stop fake Codex provider"); diff --git a/packages/paperclip-runner/scripts/build-inputs.test.mjs b/packages/paperclip-runner/scripts/build-inputs.test.mjs index ce650a871df..7822de2cbb3 100644 --- a/packages/paperclip-runner/scripts/build-inputs.test.mjs +++ b/packages/paperclip-runner/scripts/build-inputs.test.mjs @@ -491,26 +491,34 @@ test("two chains run by separate shells at the same time both record", { skip: p await withRepo(async ({ repoRoot, packageRoot }) => { const pendingDirectory = buildInputsPendingDirectory(packageRoot); const marker = path.join(repoRoot, "release-the-chains"); - const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).sort() : []); + const records = () => (existsSync(pendingDirectory) ? readdirSync(pendingDirectory).filter(name => name.endsWith(".json")).sort() : []); - const first = runChainInShell("sh", packageRoot, marker); - await waitFor(() => records().length === 1, "the first chain's record"); - const second = runChainInShell("sh", packageRoot, marker); - await waitFor(() => records().length === 2, "the second chain's record"); - const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, "")); - assert.notEqual(firstId, secondId); - for (const id of [firstId, secondId]) { - assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid"); - assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id); + const staged = path.join(pendingDirectory, "999999.json.999999.tmp"); + const chains = [runChainInShell("sh", packageRoot, marker)]; + let results; + try { + await waitFor(() => records().length === 1, "the first chain's record"); + await writeFile(staged, "{"); + chains.push(runChainInShell("sh", packageRoot, marker)); + await waitFor(() => records().length === 2, "the second chain's record"); + const [firstId, secondId] = records().map((name) => name.replace(/\.json$/, "")); + assert.notEqual(firstId, secondId); + for (const id of [firstId, secondId]) { + assert.match(id, /^\d+$/, "a chain's record is named after its shell's pid"); + assert.equal(readBuildInputsStamp(buildInputsPendingPath(packageRoot, id)).buildId, id); + } + } finally { + await rm(staged, { force: true }); + await writeFile(marker, "go\n"); + results = await Promise.allSettled(chains); } - await writeFile(marker, "go\n"); - const results = await Promise.all([first, second]); for (const result of results) { - assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `)); - assert.match(result.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording"); + assert.equal(result.status, "fulfilled", result.reason?.message); + assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} begin: `)); + assert.match(result.value.stdout, new RegExp(`${BUILD_INPUTS_SCHEMA} recorded: `), "a chain finished without recording"); } - assert.deepEqual(records(), []); + assert.deepEqual(await readdir(pendingDirectory), []); assert.equal(assessRunnerBuild({ repoRoot, packageRoot }).current, true); // The same chain under bash, which may exec its last command: `--finish` diff --git a/packages/paperclip-runner/src/drivers/agent-observability.test.ts b/packages/paperclip-runner/src/drivers/agent-observability.test.ts index 4fdc2f93348..9e932db0adc 100644 --- a/packages/paperclip-runner/src/drivers/agent-observability.test.ts +++ b/packages/paperclip-runner/src/drivers/agent-observability.test.ts @@ -9,8 +9,12 @@ import { createSanitizedAcpxSpawnInput } from "./acpx/environment.js"; import { configuredEnvironmentProjection } from "../configured-environment.js"; const roots: string[] = []; afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); -const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: { OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000" }, otelConfig: { - exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: { authorization: "Bearer secret-canary" } } }, metrics_exporter: "none", log_agent_responses: true, +const launch: ProviderObservabilityLaunch = { kind: "codex", compatibilityKey: "revision", environment: { + OTEL_EXPORTER_OTLP_LOGS_TIMEOUT: "3000", + OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary", + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20secret-canary", +}, otelConfig: { + exporter: { "otlp-http": { endpoint: "https://collector.example/v1/logs", protocol: "binary", headers: {} } }, metrics_exporter: "none", log_agent_responses: true, } }; it("transports a valid encoded header set in both Codex exporters without rejecting the launch", () => { const headers = Object.fromEntries([0, 1, 2, 3].map(index => [`x-custom-${index}`, encodeURIComponent("%".repeat(4000))])); @@ -28,8 +32,19 @@ it("materializes the real native home with both governed MCP and protected expor nativeMcp: { name: "paperclip-assigned", url: "https://paperclip.example/mcp", token: "mcp-secret" } }); const config = parse(await readFile(join(home, "config.toml"), "utf8")); expect(config.otel).toEqual(launch.otelConfig); expect(config.mcp_servers).toHaveProperty("paperclip-assigned"); + expect(JSON.stringify(config)).not.toContain("secret-canary"); expect((await stat(join(home, "config.toml"))).mode & 0o777).toBe(0o600); }); +it("delivers Codex headers only through its provider environment and redacts diagnostics", () => { + const source = applyProviderObservabilityEnvironment({ KEEP: "task" }, launch, true); + const sanitized = createSanitizedAcpxSpawnInput(source, "codex").env; + expect(sanitized).toMatchObject(launch.environment); + expect(sanitized.PAPERCLIP_OBSERVABILITY_CODEX_CONFIG).not.toContain("secret-canary"); + expect(configuredEnvironmentProjection(source).PAPERCLIP_CONFIGURED_ENV_KEYS).toBe('["KEEP"]'); + expect(createSanitizedAcpxSpawnInput(source, "pi").env).not.toHaveProperty("OTEL_EXPORTER_OTLP_LOGS_HEADERS"); + const diagnostic = redactObservabilityDiagnostic("Bearer secret-canary Bearer%20secret-canary secret-canary", sanitized); + expect(diagnostic).toBe("[REDACTED] [REDACTED] [REDACTED]"); +}); it("passes only explicit Claude observability through its sanitizer and excludes task projection", () => { const source = { OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20secret-canary", KEEP: "task" }; expect(createSanitizedAcpxSpawnInput(source, "claude").env).toMatchObject({ OTEL_LOGS_EXPORTER: "otlp", OTEL_EXPORTER_OTLP_LOGS_HEADERS: source.OTEL_EXPORTER_OTLP_LOGS_HEADERS }); diff --git a/packages/paperclip-runner/src/drivers/agent-observability.ts b/packages/paperclip-runner/src/drivers/agent-observability.ts index 37b11a13032..7ee8159df14 100644 --- a/packages/paperclip-runner/src/drivers/agent-observability.ts +++ b/packages/paperclip-runner/src/drivers/agent-observability.ts @@ -27,7 +27,7 @@ export function providerObservabilityEnvironment(source: NodeJS.ProcessEnv | und if (!source || !["codex", "claude"].includes(agent)) return {}; const result: NodeJS.ProcessEnv = {}; const names: readonly string[] = agent === "claude" ? AGENT_OBSERVABILITY_ENV_KEYS.filter(key => key !== "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG") - : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"]; + : ["OTEL_EXPORTER_OTLP_LOGS_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", "PAPERCLIP_OBSERVABILITY_CODEX_CONFIG"]; for (const key of names) if (source[key] !== undefined) { // Codex carries two percent-encoded header maps. A valid 16-KiB company // header set can exceed 64 KiB after encoding and duplication. @@ -74,6 +74,7 @@ export function redactObservabilityDiagnostic(message: string, environment: Node values.add(value); for (const item of value.split(",")) { const encoded = item.slice(item.indexOf("=") + 1); + values.add(encoded); try { values.add(decodeURIComponent(encoded)); } catch { values.add(encoded); } } } diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts index 4184701c56d..ddeef5fc806 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts @@ -25,6 +25,46 @@ describe("Codex app-server transport limits", () => { })).toEqual({ PAPERCLIP_AI_PROVIDER_KEY: "selected-provider-key", CODEX_HOME: "/isolated/connection" }); }); + it.each([true, false])("passes configured OTLP authentication to the child only when present (%s)", async (enabled) => { + const headers = enabled ? { + OTEL_EXPORTER_OTLP_LOGS_HEADERS: "authorization=Bearer%20fixture-logs", + OTEL_EXPORTER_OTLP_TRACES_HEADERS: "authorization=Bearer%20fixture-traces", + } : {}; + const transport = new ProcessCodexAppServerTransport({ + command: process.execPath, + args: ["-e", ` + process.stdin.resume(); + process.stdout.write(JSON.stringify({ method: "environment", params: { + OTEL_EXPORTER_OTLP_LOGS_HEADERS: process.env.OTEL_EXPORTER_OTLP_LOGS_HEADERS, + OTEL_EXPORTER_OTLP_TRACES_HEADERS: process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS, + DATABASE_URL: process.env.DATABASE_URL, + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + } }) + "\\n"); + `], + environment: createSanitizedCodexEnvironment({ + ...headers, DATABASE_URL: "excluded-database", OPENAI_API_KEY: "excluded-provider-key", + }), + }); + try { + const notification = await transport.notifications()[Symbol.asyncIterator]().next(); + expect(notification.value).toEqual({ method: "environment", params: headers }); + } finally { + await transport.close(); + } + }); + + it("does not inherit ambient OTLP credentials without an explicit launch environment", () => { + vi.stubEnv("OTEL_EXPORTER_OTLP_LOGS_HEADERS", "ambient-log-secret"); + vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "ambient-trace-secret"); + try { + const environment = createSanitizedCodexEnvironment(); + expect(environment.OTEL_EXPORTER_OTLP_LOGS_HEADERS).toBeUndefined(); + expect(environment.OTEL_EXPORTER_OTLP_TRACES_HEADERS).toBeUndefined(); + } finally { + vi.unstubAllEnvs(); + } + }); + it("passes only bounded controller-projected GitHub credentials", () => { expect( createSanitizedCodexEnvironment({ diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index 886e86c7a4e..c54663b1b28 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -251,7 +251,11 @@ export function createSanitizedCodexEnvironment( if (key.includes("PROXY") && proxyContainsCredentials(value)) continue; environment[key] = value; } - for (const key of ["PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY"]) { + // Secret-bearing launch material must come from the controller's explicit environment. + for (const key of [ + "PAPERCLIP_AGENT_KEY_ID", "PAPERCLIP_AGENT_PUBLIC_KEY", "PAPERCLIP_AGENT_PRIVATE_KEY", + "OTEL_EXPORTER_OTLP_LOGS_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + ]) { if (explicit?.[key] !== undefined) environment[key] = explicit[key]; } Object.assign(environment, githubCredentialEnvironment(source), configuredEnvironment(explicit)); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 30ed4b970a6..a63abc777f8 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -18,7 +18,7 @@ import { execFile, type ChildProcess } from "node:child_process"; import { EventEmitter } from "node:events"; import { promisify } from "node:util"; import { getCACertificates } from "node:tls"; -import { join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { @@ -1441,7 +1441,7 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); - it("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway", async () => { + it.each([false, true])("uses the native system channel, selected skill tree, instruction root, and assigned MCP gateway after startup retry=%s", async (retryStartup) => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-context-")); const workspace = await mkdtemp( @@ -1465,6 +1465,7 @@ describe("OpenCodeServerDriver", () => { ]); const systemInstructions = `${PAPERCLIP_EXECUTION_PROMPT}\n\nRead sibling.md\n\nRead-only instruction sibling root: ${instructionRoot}`; let submittedPrompt: Record | null = null; + let startupFailurePending = retryStartup; const driver = new OpenCodeServerDriver({ model: "openrouter/deepseek/deepseek-v4-flash-0731", runtimeDirectory: root, @@ -1479,6 +1480,10 @@ describe("OpenCodeServerDriver", () => { PAPERCLIP_NATIVE_MCP_TOKEN: "x".repeat(40), }, fetch: async (input, init) => { + if (startupFailurePending && String(input).endsWith("/session") && init?.method === "POST") { + startupFailurePending = false; + return new Response("Fixture startup failure", { status: 503 }); + } if (String(input).endsWith("/prompt_async")) submittedPrompt = JSON.parse(String(init?.body)); return fetch(input, init); @@ -1503,15 +1508,17 @@ describe("OpenCodeServerDriver", () => { PAPERCLIP_EXECUTION_PROMPT, ); const sessionRoot = join(root, "context"); - const isolatedHomes = (await readdir(sessionRoot)).filter((entry) => - entry.startsWith("home-"), + const runtimeEnvironment = JSON.parse( + await readFile(join(sessionRoot, "data", "fake-environment.json"), "utf8"), ); - expect(isolatedHomes).toHaveLength(1); + const isolatedHome = runtimeEnvironment.home; + expect(startupFailurePending).toBe(false); + expect(dirname(isolatedHome)).toBe(sessionRoot); + expect(basename(isolatedHome)).toMatch(/^home-/); await expect( readFile( join( - sessionRoot, - isolatedHomes[0]!, + isolatedHome, ".claude", "skills", "assigned", diff --git a/scripts/__tests__/eval-schedule-monitor.test.mjs b/scripts/__tests__/eval-schedule-monitor.test.mjs new file mode 100644 index 00000000000..ddf6db789e6 --- /dev/null +++ b/scripts/__tests__/eval-schedule-monitor.test.mjs @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { acknowledgeEvalNotifications, expectedCampaign, inspectEvalSchedule, monitorTransition, runEvalMonitor, MONITORED_SCHEDULES } from "../eval-schedule-monitor.mjs"; + +const now = Date.parse("2026-10-25T13:00:00Z"); +const enabledAt = "2026-10-24T00:00:00Z"; +const variables = { total_count: 3, variables: [ + { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: "true" }, + { name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" }, + { name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT", value: enabledAt }, +] }; +const workflow = { id: 17, path: ".github/workflows/runner-full-stack-e2e.yml", state: "active" }; +const run = (cadence, id, extra = {}) => ({ id, workflow_id: 17, run_attempt: 1, event: "schedule", status: "completed", conclusion: "success", + display_title: `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}`, + created_at: cadence === "daily" ? "2026-10-25T08:50:00Z" : "2026-10-25T09:50:00Z", ...extra }); +function fixture(overrides = {}) { + const calls = []; + const get = async endpoint => { + calls.push(endpoint); + if (endpoint.includes("/variables?")) return overrides.variables ?? structuredClone(variables); + if (endpoint.endsWith("runner-full-stack-e2e.yml")) return overrides.workflow ?? workflow; + if (endpoint.includes("/runs?")) { + const workflow_runs = overrides.runs ?? [run("daily", 101), run("weekly", 102)]; + return { total_count: overrides.total_count ?? workflow_runs.length, workflow_runs }; + } + if (endpoint.includes("/artifacts?")) { + const id = Number(endpoint.match(/\/runs\/(\d+)\//)[1]); + const artifacts = overrides.artifacts ?? [{ name: `runner-e2e-report-${id}-1`, expired: false, size_in_bytes: 100 }]; + return { artifacts, total_count: artifacts.length }; + } + throw new Error("Unexpected read"); + }; + return { get, calls }; +} + +test("UTC due windows honor grace, activation and the weekly boundary", () => { + assert.equal(expectedCampaign("daily", now, Date.parse(enabledAt)), "2026-10-25T08:47:00.000Z"); + assert.equal(expectedCampaign("weekly", now, Date.parse(enabledAt)), "2026-10-25T09:47:00.000Z"); + assert.equal(expectedCampaign("weekly", Date.parse("2026-10-25T10:00:00Z"), Date.parse(enabledAt)), undefined); + assert.equal(expectedCampaign("daily", now, now), undefined); + assert.equal(expectedCampaign("daily", Date.parse("2026-11-01T10:17:00Z"), 0), "2026-11-01T08:47:00.000Z"); +}); + +test("disabled gates do not query or diagnose old paid campaigns", async () => { + const f = fixture({ variables: { total_count: 0, variables: [] } }); + assert.equal((await inspectEvalSchedule({ ...f, now })).status, "disabled"); + assert.equal(f.calls.length, 1); +}); + +for (const budgetGate of [undefined, "false"]) test(`an enabled legacy schedule with ${budgetGate ?? "missing"} budget gate alerts once`, async () => { + const rows = [{ name: "RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED", value: "true" }]; + if (budgetGate !== undefined) rows.push({ name: "RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED", value: budgetGate }); + const f = fixture({ variables: { total_count: rows.length, variables: rows } }); + const inspection = await inspectEvalSchedule({ ...f, now }); + assert.equal(inspection.status, "configuration_incomplete"); + assert.equal(inspection.reason, "legacy_schedule_enabled_without_budget_gate"); + assert.equal(f.calls.length, 1); + const first = monitorTransition(undefined, inspection); + assert.equal(first.notifications.length, 1); + assert.equal(first.notifications[0].kind, "problem"); + assert.deepEqual(monitorTransition(first, inspection).notifications, first.notifications); +}); + +test("an explicitly disabled legacy gate stays quiet even if the new budget gate is enabled", async () => { + const changed = structuredClone(variables); changed.variables[1].value = "false"; + const f = fixture({ variables: changed }); + const inspection = await inspectEvalSchedule({ ...f, now }); + assert.equal(inspection.status, "disabled"); assert.equal(f.calls.length, 1); + assert.deepEqual(monitorTransition(undefined, inspection).notifications, []); +}); + +test("missing activation time and a disabled workflow stay distinct from missing runs", async () => { + const invalid = structuredClone(variables); invalid.variables[2].value = "not a date"; + assert.equal((await inspectEvalSchedule({ ...fixture({ variables: invalid }), now })).status, "configuration_incomplete"); + assert.equal((await inspectEvalSchedule({ ...fixture({ workflow: { ...workflow, state: "disabled_inactivity" } }), now })).status, "workflow_disabled"); +}); + +test("requires one completed scheduled campaign and its unexpired report for each cadence", async () => { + const result = await inspectEvalSchedule({ ...fixture(), now }); + assert.deepEqual(result.observations.map(row => row.status), ["healthy", "healthy"]); + assert.deepEqual(result.observations.map(row => row.runId), [101, 102]); + assert.deepEqual(monitorTransition(undefined, result).notifications, []); +}); + +for (const [name, overrides, status] of [ + ["no campaign", { runs: [] }, "missing"], + ["manual run cannot substitute for schedule", { runs: [run("daily", 101, { event: "workflow_dispatch" })] }, "missing"], + ["failed campaign", { runs: [run("daily", 101, { conclusion: "failure" })] }, "failed"], + ["canceled campaign", { runs: [run("daily", 101, { conclusion: "cancelled" })] }, "failed"], + ["overdue queued campaign", { runs: [run("daily", 101, { status: "queued" })] }, "overdue"], + ["unexpected paid rerun", { runs: [run("daily", 101, { run_attempt: 2 })] }, "unexpected_retry"], + ["multiple matching runs", { runs: [run("daily", 101), run("daily", 103)] }, "ambiguous_runs"], + ["missing report", { artifacts: [] }, "missing_report"], + ["expired report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: true, size_in_bytes: 100 }] }, "missing_report"], + ["empty report", { artifacts: [{ name: "runner-e2e-report-101-1", expired: false, size_in_bytes: 0 }] }, "missing_report"], +]) test(name, async () => { + const result = await inspectEvalSchedule({ ...fixture(overrides), now }); + assert.equal(result.observations[0].status, status); +}); + +test("new-day early run does not collide with the prior day during grace", async () => { + const runs = [run("daily", 100, { created_at: "2026-10-24T08:50:00Z" }), run("daily", 101)]; + const result = await inspectEvalSchedule({ ...fixture({ runs }), now: Date.parse("2026-10-25T09:00:00Z") }); + assert.equal(result.observations[0].runId, 100); + assert.equal(result.observations[0].status, "healthy"); +}); + +test("running is quiet inside the runtime limit", async () => { + const result = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { status: "in_progress", created_at: "2026-10-25T10:00:00Z" })] }), now }); + assert.equal(result.observations[0].status, "running"); +}); + +test("API errors and incomplete lists cannot imply a missing or successful run", async () => { + assert.equal((await inspectEvalSchedule({ ...fixture({ total_count: 101 }), now })).status, "api_unavailable"); + assert.equal((await inspectEvalSchedule({ get: async () => { throw new Error("private upstream body"); }, now })).status, "api_unavailable"); +}); + +test("alerts once per incident and recovery while keeping last observed run IDs through API failure", async () => { + const failed = await inspectEvalSchedule({ ...fixture({ runs: [run("daily", 101, { conclusion: "failure" }), run("weekly", 102)] }), now }); + const first = monitorTransition(undefined, failed); + assert.equal(first.notifications.length, 1); + assert.deepEqual(monitorTransition(first, failed).notifications, first.notifications); + const outage = { observedAt: new Date(now + 1000).toISOString(), status: "api_unavailable", observations: [] }; + const unavailable = monitorTransition({ ...first, notifications: [] }, outage); + assert.equal(unavailable.lastSuccessfulInspectionAt, first.lastSuccessfulInspectionAt); + assert.deepEqual(unavailable.lastInspectedRunIds, { daily: 101, weekly: 102 }); + assert.equal(unavailable.notifications[0].status, "api_unavailable"); + assert.deepEqual(monitorTransition(unavailable, outage).notifications, unavailable.notifications); + const resumed = monitorTransition({ ...unavailable, notifications: [] }, failed); + assert.deepEqual(resumed.notifications.map(row => [row.kind, row.cadence]), [["recovery", "monitor"]]); + const healthy = monitorTransition({ ...resumed, notifications: [] }, await inspectEvalSchedule({ ...fixture(), now })); + assert.deepEqual(healthy.notifications.map(row => [row.kind, row.cadence]), [["recovery", "daily"]]); +}); + +test("persists private state across invocations and refuses corrupt history", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-")); + try { + const file = path.join(directory, "state.json"); + const first = await runEvalMonitor(file, fixture({ runs: [] }).get, now); + assert.equal(first.notifications.length, 2); + const repeated = await runEvalMonitor(file, fixture({ runs: [] }).get, now); + assert.deepEqual(repeated.notifications, first.notifications); + assert.deepEqual(await acknowledgeEvalNotifications(file, [first.notifications[0].id]), { acknowledged: 1, pending: 1 }); + const afterPartialDelivery = await runEvalMonitor(file, fixture({ runs: [] }).get, now); + assert.deepEqual(afterPartialDelivery.notifications, first.notifications.slice(1)); + assert.deepEqual(await acknowledgeEvalNotifications(file, first.notifications.map(row => row.id)), { acknowledged: 1, pending: 0 }); + assert.deepEqual((await runEvalMonitor(file, fixture({ runs: [] }).get, now)).notifications, []); + assert.equal(JSON.parse(await readFile(file, "utf8")).repository, "paperclipai/paperclip"); + await writeFile(file, "bad history"); + await assert.rejects(runEvalMonitor(file, fixture().get, now)); + assert.equal(await readFile(file, "utf8"), "bad history"); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("pending alerts survive later recovery and are acknowledged only after delivery", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-outbox-")); + try { + const file = path.join(directory, "state.json"); + const lostOutput = await runEvalMonitor(file, fixture({ runs: [] }).get, now); + const recovered = await runEvalMonitor(file, fixture().get, now + 1000); + assert.deepEqual(recovered.notifications.slice(0, 2), lostOutput.notifications); + assert.deepEqual(recovered.notifications.map(row => row.kind), ["problem", "problem", "recovery", "recovery"]); + const unchanged = await runEvalMonitor(file, fixture().get, now + 2000); + assert.deepEqual(unchanged.notifications, recovered.notifications); + await assert.rejects(acknowledgeEvalNotifications(file, ["invalid"])); + assert.deepEqual(await acknowledgeEvalNotifications(file, recovered.notifications.map(row => row.id)), { acknowledged: 4, pending: 0 }); + assert.deepEqual((await runEvalMonitor(file, fixture().get, now + 3000)).notifications, []); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("rejects valid JSON with malformed history without querying or overwriting it", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "eval-monitor-corrupt-")); + try { + const file = path.join(directory, "state.json"); + await runEvalMonitor(file, fixture().get, now); + const valid = JSON.parse(await readFile(file, "utf8")); + for (const malformed of [null, false, 0, [], {}, { ...valid, signals: [] }, { ...valid, signals: { daily: null } }, + { ...valid, notifications: null }, { ...valid, notifications: [{ id: "invalid" }] }]) { + const original = JSON.stringify(malformed); + await writeFile(file, original); + const f = fixture(); + await assert.rejects(runEvalMonitor(file, f.get, now), /Invalid retained monitor state/); + assert.equal(f.calls.length, 0); + assert.equal(await readFile(file, "utf8"), original); + } + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +test("workflow titles carry the literal schedule used to distinguish daily and weekly runs", async () => { + const text = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8"); + assert.ok(text.includes("run-name: Runner Full-Stack E2E / ${{ github.event.schedule || inputs.cadence || 'manual' }}")); + for (const cron of Object.values(MONITORED_SCHEDULES)) assert.ok(text.includes(`cron: "${cron}"`)); +}); diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 8ea57dadc77..69b7df521dd 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -698,8 +698,22 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map( (match) => match[1], ); - assert.equal(crons.length, 1, `${name} must have one schedule`); - assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/); + if (name === "runner-full-stack-e2e.yml") { + assert.deepEqual(crons, ["47 8 * * *", "47 9 * * 0"]); + assert.ok(workflow.includes("vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'")); + assert.ok(workflow.includes("Prepare bounded campaign before credentials")); + assert.ok(workflow.includes("--max-parallel 1 --max-automatic-retries 0")); + const catalog = workflow.slice(workflow.indexOf(" catalog:"), workflow.indexOf(" daytona_image:")); + assert.ok(catalog.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON }}")); + assert.ok(catalog.includes("operating_overhead: ${{ steps.catalog.outputs.operating_overhead }}")); + const prepare = workflow.slice(workflow.indexOf(" - name: Prepare bounded campaign before credentials"), workflow.indexOf(" - name: Run bounded campaign and retain feedback review")); + assert.ok(prepare.includes("EVAL_OPERATING_OVERHEAD_JSON: ${{ needs.catalog.outputs.operating_overhead }}")); + assert.ok(prepare.includes("EVAL_TEST_RUNNER: ${{ needs.authorize.outputs.test_runner }}")); + assert.ok(!prepare.includes("vars.RUNNER_E2E_OPERATING_OVERHEAD_JSON")); + } else { + assert.equal(crons.length, 1, `${name} must have one schedule`); + assert.match(crons[0], /^\d{1,2} \d{1,2} \* \* 0$/); + } } const chaosWorkflow = actionPinWorkflows[2]; diff --git a/scripts/eval-schedule-monitor.mjs b/scripts/eval-schedule-monitor.mjs new file mode 100644 index 00000000000..6bee0e19566 --- /dev/null +++ b/scripts/eval-schedule-monitor.mjs @@ -0,0 +1,176 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; + +export const MONITORED_REPOSITORY = "paperclipai/paperclip"; +export const MONITORED_WORKFLOW = "runner-full-stack-e2e.yml"; +export const MONITORED_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" }; +const hour = 3_600_000; +const prefix = `repos/${MONITORED_REPOSITORY}/actions`; +const numericId = value => Number.isSafeInteger(value) && value > 0; + +export function expectedCampaign(cadence, now, enabledAt) { + if (!Object.hasOwn(MONITORED_SCHEDULES, cadence) || !Number.isFinite(now) || !Number.isFinite(enabledAt)) { + throw new Error("Invalid monitor schedule input"); + } + const due = new Date(now - 90 * 60_000); + due.setUTCHours(cadence === "daily" ? 8 : 9, 47, 0, 0); + if (due.getTime() > now - 90 * 60_000) due.setUTCDate(due.getUTCDate() - 1); + if (cadence === "weekly") due.setUTCDate(due.getUTCDate() - due.getUTCDay()); + return due.getTime() >= enabledAt ? due.toISOString() : undefined; +} + +function signal(cadence, status, dueAt, runId) { + return { cadence, status, ...(dueAt ? { dueAt } : {}), ...(runId ? { + runId, url: `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${runId}`, + } : {}) }; +} + +export async function inspectEvalSchedule({ get, now = Date.now() }) { + const observations = []; + try { + const variables = await get(`${prefix}/variables?per_page=100`); + if (!Array.isArray(variables.variables) || !Number.isSafeInteger(variables.total_count) + || variables.total_count !== variables.variables.length) throw new Error("Incomplete variables"); + const values = Object.fromEntries(variables.variables.map(row => [row.name, row.value])); + if (values.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED !== "true") { + return { observedAt: new Date(now).toISOString(), status: "disabled", observations: [] }; + } + if (values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED !== "true") { + return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete", + reason: "legacy_schedule_enabled_without_budget_gate", observations: [] }; + } + const enabledAt = Date.parse(values.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT ?? ""); + if (!Number.isFinite(enabledAt) || enabledAt > now) { + return { observedAt: new Date(now).toISOString(), status: "configuration_incomplete", observations: [] }; + } + const workflow = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}`); + if (workflow.path !== `.github/workflows/${MONITORED_WORKFLOW}` || !numericId(workflow.id)) throw new Error("Wrong workflow"); + if (workflow.state !== "active") return { observedAt: new Date(now).toISOString(), status: "workflow_disabled", observations: [] }; + const expected = Object.keys(MONITORED_SCHEDULES).map(cadence => ({ cadence, dueAt: expectedCampaign(cadence, now, enabledAt) })); + const due = expected.filter(row => row.dueAt); + if (!due.length) return { observedAt: new Date(now).toISOString(), status: "waiting", observations: [] }; + const since = due.map(row => row.dueAt).sort()[0]; + const runs = await get(`${prefix}/workflows/${MONITORED_WORKFLOW}/runs?event=schedule&per_page=100&created=${encodeURIComponent(`>=${since}`)}`); + if (!Array.isArray(runs.workflow_runs) || !Number.isSafeInteger(runs.total_count) + || runs.total_count !== runs.workflow_runs.length) throw new Error("Incomplete runs"); + for (const { cadence, dueAt } of expected) { + if (!dueAt) { observations.push(signal(cadence, "waiting")); continue; } + const matches = runs.workflow_runs.filter(run => run.event === "schedule" && run.workflow_id === workflow.id + && run.display_title === `Runner Full-Stack E2E / ${MONITORED_SCHEDULES[cadence]}` + && Date.parse(run.created_at) >= Date.parse(dueAt) && Date.parse(run.created_at) <= now + && Date.parse(run.created_at) < Date.parse(dueAt) + (cadence === "daily" ? 24 : 168) * hour); + if (!matches.length) { observations.push(signal(cadence, "missing", dueAt)); continue; } + if (matches.length !== 1) { observations.push(signal(cadence, "ambiguous_runs", dueAt)); continue; } + const run = matches[0]; + if (!numericId(run.id) || !numericId(run.run_attempt) || typeof run.status !== "string") throw new Error("Invalid run identity"); + if (run.run_attempt !== 1) { observations.push(signal(cadence, "unexpected_retry", dueAt, run.id)); continue; } + if (run.status !== "completed") { + observations.push(signal(cadence, now - Date.parse(run.created_at) > 4 * hour ? "overdue" : "running", dueAt, run.id)); + continue; + } + if (run.conclusion !== "success") { observations.push(signal(cadence, "failed", dueAt, run.id)); continue; } + const artifacts = await get(`${prefix}/runs/${run.id}/artifacts?per_page=100`); + if (!Array.isArray(artifacts.artifacts) || !Number.isSafeInteger(artifacts.total_count) + || artifacts.total_count !== artifacts.artifacts.length) throw new Error("Incomplete artifacts"); + const reports = artifacts.artifacts.filter(row => row.name === `runner-e2e-report-${run.id}-1` + && row.expired === false && Number.isSafeInteger(row.size_in_bytes) && row.size_in_bytes > 0); + observations.push(signal(cadence, reports.length === 1 ? "healthy" : "missing_report", dueAt, run.id)); + } + return { observedAt: new Date(now).toISOString(), status: "observed", observations }; + } catch { + return { observedAt: new Date(now).toISOString(), status: "api_unavailable", observations }; + } +} + +const quiet = new Set(["disabled", "waiting", "running", "healthy"]); +const statuses = new Set([...quiet, "observed", "configuration_incomplete", "workflow_disabled", "api_unavailable", + "missing", "ambiguous_runs", "unexpected_retry", "overdue", "failed", "missing_report"]); +const object = value => value !== null && typeof value === "object" && !Array.isArray(value); +const timestamp = value => typeof value === "string" && Number.isFinite(Date.parse(value)); +const notificationId = value => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(value); +const signalKey = row => !quiet.has(row.status) ? JSON.stringify([row.status, row.dueAt ?? null, row.runId ?? null]) : row.status; +const validSignal = row => object(row) && ["monitor", "daily", "weekly"].includes(row.cadence) && statuses.has(row.status) + && (row.dueAt === undefined || timestamp(row.dueAt)) && (row.runId === undefined || numericId(row.runId)) + && (row.runId === undefined ? row.url === undefined : row.url === `https://github.com/${MONITORED_REPOSITORY}/actions/runs/${row.runId}`); + +function validateState(previous) { + if (!object(previous) || previous.version !== 1 || previous.repository !== MONITORED_REPOSITORY + || previous.workflow !== MONITORED_WORKFLOW || !object(previous.signals) + || !timestamp(previous.checkedAt) || !(previous.lastSuccessfulInspectionAt === null || timestamp(previous.lastSuccessfulInspectionAt)) + || !object(previous.lastInspectedRunIds) || Object.entries(previous.lastInspectedRunIds).some(([cadence, id]) => !["daily", "weekly"].includes(cadence) || !numericId(id)) + || Object.entries(previous.signals).some(([cadence, row]) => !validSignal(row) || row.cadence !== cadence + || row.issue !== !quiet.has(row.status) || row.key !== signalKey(row)) + || !Array.isArray(previous.notifications) || previous.notifications.some(row => !validSignal(row) + || !notificationId(row.id) || !["problem", "recovery"].includes(row.kind)) + || new Set(previous.notifications.map(row => row.id)).size !== previous.notifications.length) { + throw new Error("Invalid retained monitor state; refusing to reset notification history"); + } +} + +export function monitorTransition(previous, inspection) { + if (previous !== undefined) validateState(previous); + const current = inspection.status === "observed" ? inspection.observations : [signal("monitor", inspection.status)]; + const signals = {}, notifications = []; + const prior = previous?.signals ?? {}; + for (const row of current) { + const issue = !quiet.has(row.status); + const key = signalKey(row); + signals[row.cadence] = { ...row, issue, key }; + if (issue && prior[row.cadence]?.key !== key) notifications.push({ kind: "problem", ...row }); + else if (!issue && prior[row.cadence]?.issue && row.status !== "disabled") notifications.push({ kind: "recovery", ...row }); + } + if (inspection.status === "observed" && prior.monitor?.issue) notifications.push({ kind: "recovery", cadence: "monitor", status: "observed" }); + const readable = !["api_unavailable", "configuration_incomplete"].includes(inspection.status); + return { version: 1, repository: MONITORED_REPOSITORY, workflow: MONITORED_WORKFLOW, + checkedAt: inspection.observedAt, + lastSuccessfulInspectionAt: readable ? inspection.observedAt : previous?.lastSuccessfulInspectionAt ?? null, + lastInspectedRunIds: { ...previous?.lastInspectedRunIds, + ...Object.fromEntries(inspection.observations.filter(row => row.runId).map(row => [row.cadence, row.runId])) }, + signals: inspection.status === "observed" || quiet.has(inspection.status) ? signals : { ...prior, ...signals }, + notifications: [...previous?.notifications ?? [], ...notifications.map(row => ({ id: randomUUID(), ...row }))] }; +} + +async function saveState(stateFile, next) { + await mkdir(path.dirname(stateFile), { recursive: true, mode: 0o700 }); + await writeFile(`${stateFile}.pending`, JSON.stringify(next, null, 2) + "\n", { mode: 0o600 }); + await rename(`${stateFile}.pending`, stateFile); +} + +export async function acknowledgeEvalNotifications(stateFile, ids) { + if (!Array.isArray(ids) || !ids.length || ids.some(id => !notificationId(id))) throw new Error("Invalid notification acknowledgments"); + const previous = JSON.parse(await readFile(stateFile, "utf8")); + validateState(previous); + const next = { ...previous, notifications: previous.notifications.filter(row => !ids.includes(row.id)) }; + await saveState(stateFile, next); + return { acknowledged: previous.notifications.length - next.notifications.length, pending: next.notifications.length }; +} + +export async function runEvalMonitor(stateFile, get, now = Date.now()) { + let previous; + try { previous = JSON.parse(await readFile(stateFile, "utf8")); validateState(previous); } + catch (error) { if (error.code !== "ENOENT") throw error; } + const inspection = await inspectEvalSchedule({ get, now }); + const next = monitorTransition(previous, inspection); + await saveState(stateFile, next); + return { ...inspection, lastSuccessfulInspectionAt: next.lastSuccessfulInspectionAt, + lastInspectedRunIds: next.lastInspectedRunIds, notifications: next.notifications }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.argv.length !== 3 && !(process.argv[3] === "--ack" && process.argv.length >= 5)) { + throw new Error("Usage: node scripts/eval-schedule-monitor.mjs [--ack ...]"); + } + const execute = promisify(execFile); + const get = async endpoint => { + if (!endpoint.startsWith(`${prefix}/`)) throw new Error("Unapproved monitor endpoint"); + const { stdout } = await execute("gh", ["api", "--method", "GET", endpoint], { timeout: 15_000, maxBuffer: 4 * 1024 * 1024 }); + return JSON.parse(stdout); + }; + console.log(JSON.stringify(process.argv[3] === "--ack" + ? await acknowledgeEvalNotifications(path.resolve(process.argv[2]), process.argv.slice(4)) + : await runEvalMonitor(path.resolve(process.argv[2]), get), null, 2)); +} diff --git a/server/src/__tests__/eval-feedback-recovery.integration.test.ts b/server/src/__tests__/eval-feedback-recovery.integration.test.ts new file mode 100644 index 00000000000..873dc701afc --- /dev/null +++ b/server/src/__tests__/eval-feedback-recovery.integration.test.ts @@ -0,0 +1,87 @@ +import { spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { once } from "node:events"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { eq } from "drizzle-orm"; +import { agents, heartbeatRuns } from "@paperclipai/db"; +import { expect, it } from "vitest"; +import { createLocalAgentJwt } from "../agent-auth-jwt.js"; +import { startRunnerApiTestServer } from "./helpers/runner-api-server.js"; +import { recoverInterruptedFeedback, readRecoveredFeedback } from "../../../tests/runner-e2e/feedback-recovery.js"; + +it("recovers real saved complaints and suggestions after their producer is killed", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-feedback-interruption-")); + const previousHome = process.env.PAPERCLIP_HOME, previousSecret = process.env.PAPERCLIP_AGENT_JWT_SECRET; + process.env.PAPERCLIP_HOME = path.join(root, "home"); + process.env.PAPERCLIP_AGENT_JWT_SECRET = randomUUID(); + let server: Awaited> | undefined; + let worker: ReturnType | undefined; + try { + server = await startRunnerApiTestServer({ deploymentMode: "local_trusted" }); + const fixture = await server.fixture({ disableWakeOnDemand: true }); + await server.db.update(heartbeatRuns).set({ runtimeMode: "legacy", nativeIssueId: null }).where(eq(heartbeatRuns.id, fixture.runId)); + await server.db.update(agents).set({ adapterType: "codex_local" }).where(eq(agents.id, fixture.agentId)); + const token = createLocalAgentJwt(fixture.agentId, fixture.companyId, "codex_local", fixture.runId)!; + const caseDirectory = path.join(root, "case"); + const snapshots = path.join(caseDirectory, "snapshots"); + const identity = { campaignId: randomUUID(), attemptId: randomUUID(), executionId: "synthetic.interruption.local" }; + await mkdir(snapshots, { recursive: true }); + await writeFile(path.join(snapshots, "fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId: fixture.companyId })); + worker = spawn(process.execPath, ["--input-type=module", "-e", ` + for (const kind of ['complaint', 'suggestion']) { + const response = await fetch(process.env.FEEDBACK_URL, { + method: 'POST', headers: { 'content-type': 'application/json', + authorization: 'Bearer ' + process.env.FEEDBACK_TOKEN, 'X-Paperclip-Run-Id': process.env.FEEDBACK_RUN }, + body: JSON.stringify({ kind, body: 'Synthetic interrupted producer ' + kind, idempotencyKey: kind }), + }); + if (response.status !== 201) throw new Error('Synthetic feedback submission failed: ' + response.status); + await response.json(); + } + process.send({ saved: 2 }); + setInterval(() => {}, 1000); + `], { stdio: ["ignore", "pipe", "pipe", "ipc"], env: { + PATH: process.env.PATH, FEEDBACK_URL: `${server.apiUrl}/api/companies/${fixture.companyId}/agent-commentary`, + FEEDBACK_TOKEN: token, FEEDBACK_RUN: fixture.runId, + } }); + const saved = await Promise.race([ + once(worker, "message", { signal: AbortSignal.timeout(10_000) }), + once(worker, "exit").then(([code]) => { throw new Error(`Producer exited before saving feedback: ${code}`); }), + ]); + expect(saved[0]).toEqual({ saved: 2 }); + const exited = once(worker, "exit"); + worker.kill("SIGKILL"); + expect((await exited)[1]).toBe("SIGKILL"); + const apiUrl = server.apiUrl; + await recoverInterruptedFeedback({ identity, caseDirectory, secrets: [token], deadlineAt: Date.now() + 5_000, + api: { async get(endpoint: string, options?: { timeout: number }): Promise { + const response = await fetch(apiUrl + endpoint, { signal: AbortSignal.timeout(options?.timeout ?? 1_000) }); + if (!response.ok) throw new Error(`Read failed: ${response.status}`); + return response.json() as Promise; + } }, + }); + expect(await readRecoveredFeedback(caseDirectory, identity)).toMatchObject({ + complaints: 1, suggestions: 1, bodiesCollected: 2, collection: "partial", invocationCollection: "partial", + invocations: { completed: 0, failed: 0, unknown: 0 }, + }); + await server.close(); server = undefined; + const evidenceText = await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8"); + const evidence = JSON.parse(evidenceText); + expect(evidence.entries.map((entry: { body: string }) => entry.body).sort()).toEqual([ + "Synthetic interrupted producer complaint", "Synthetic interrupted producer suggestion", + ]); + expect(evidence.entries.every((entry: { runId: string; agent: { id: string }; task: { id: string } }) => + entry.runId === fixture.runId && entry.agent.id === fixture.agentId && entry.task.id === fixture.issueId)).toBe(true); + expect(evidenceText).not.toContain(token); + expect((await readRecoveredFeedback(caseDirectory, identity))?.review).toHaveLength(2); + } finally { + if (worker && worker.exitCode === null && worker.signalCode === null) { + const exited = once(worker, "exit"); worker.kill("SIGKILL"); await exited; + } + await server?.close(); + if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome; + if (previousSecret === undefined) delete process.env.PAPERCLIP_AGENT_JWT_SECRET; else process.env.PAPERCLIP_AGENT_JWT_SECRET = previousSecret; + await rm(root, { recursive: true, force: true }); + } +}, 90_000); diff --git a/server/src/__tests__/fixtures/otlp-observability.proto b/server/src/__tests__/fixtures/otlp-observability.proto index ed258755a2d..98feb312054 100644 --- a/server/src/__tests__/fixtures/otlp-observability.proto +++ b/server/src/__tests__/fixtures/otlp-observability.proto @@ -13,7 +13,8 @@ message LogRecord { fixed64 time_unix_nano = 1; int32 severity_number = 2; strin message ScopeLogs { Scope scope = 1; repeated LogRecord log_records = 2; } message ResourceLogs { Resource resource = 1; repeated ScopeLogs scope_logs = 2; } message ExportLogsServiceRequest { repeated ResourceLogs resource_logs = 1; } -message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; } +message Status { string message = 2; int32 code = 3; } +message Span { bytes trace_id = 1; bytes span_id = 2; bytes parent_span_id = 4; string name = 5; int32 kind = 6; fixed64 start_time_unix_nano = 7; fixed64 end_time_unix_nano = 8; repeated KeyValue attributes = 9; Status status = 15; } message ScopeSpans { Scope scope = 1; repeated Span spans = 2; } message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; } message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fb08b142c6f..f33983f14c0 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -802,6 +802,9 @@ describe("remote runner process supervision", () => { await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));'); let child: ReturnType | undefined; let probes = 0; + let healthyProbes = 0; + let releaseExitProbe!: () => void; + const exitProbeReady = new Promise(resolve => { releaseExitProbe = resolve; }); let settled = false; const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; const execute = vi.fn(async (input: { command: string; args?: string[] }) => { @@ -817,12 +820,15 @@ describe("remote runner process supervision", () => { return ok; } if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true }; - return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => { + if (label === "paperclip-runner-monitor" && probes > 2) await exitProbeReady; + const observed = await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => { execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({ exitCode: error ? typeof error.code === "number" ? error.code : null : 0, signal: null, timedOut: !!error?.killed, stdout, stderr, })); }); + if (label === "paperclip-runner-monitor" && observed.exitCode === 0 && !observed.timedOut) healthyProbes++; + return observed; }); const handle = createRemoteRunnerProcessLauncher({ target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root }, @@ -831,17 +837,21 @@ describe("remote runner process supervision", () => { })({ command: fixture, args: [], cwd: root, environment: {} }); void handle.completion.then(() => { settled = true; }, () => { settled = true; }); try { - await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 }); + await vi.waitFor(() => expect(healthyProbes).toBe(1), { timeout: 5_000 }); expect(settled).toBe(false); expect(handle.isExitConfirmed?.()).toBe(false); expect(() => process.kill(child!.pid!, 0)).not.toThrow(); expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid)); expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + const childStopped = new Promise(resolve => { child!.once("exit", () => resolve()); }); child!.kill("SIGTERM"); + await childStopped; + releaseExitProbe(); await handle.completion; expect(handle.isExitConfirmed?.()).toBe(true); } finally { child?.kill("SIGKILL"); + releaseExitProbe(); await handle.completion.catch(() => undefined); await rm(root, { recursive: true, force: true }); } diff --git a/tests/e2e/cases-read-only.spec.ts b/tests/e2e/cases-read-only.spec.ts index 83f272b46b4..fbd98223df4 100644 --- a/tests/e2e/cases-read-only.spec.ts +++ b/tests/e2e/cases-read-only.spec.ts @@ -28,12 +28,9 @@ test.afterEach(async ({ request }, testInfo) => { test("browses passive cases and rolled-up task documents without starting work or editing", async ({ page }, testInfo) => { page.setDefaultTimeout(10_000); - // Announcements can arrive after navigation and cover otherwise valid links. - // Dismiss through the ordinary Board control, including after reloads. - const dismissAnnouncement = page.getByRole("complementary", { name: "Paperclip announcements", exact: true }) - .getByRole("button", { name: "Dismiss announcement", exact: true }); - const dismissAnnouncements = async () => { await dismissAnnouncement.click(); }; - await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements); + await page.route("**/api/announcements/current", async (route) => { + await route.fulfill({ json: null }); + }); const api = page.request; const original = await json(await api.get("/api/instance/settings/experimental")); const company = await json(await api.post("/api/companies", { data: { name: `Read-only cases ${randomUUID()}` } })); @@ -189,13 +186,9 @@ test("browses passive cases and rolled-up task documents without starting work o await expect(dialog.getByRole("button", { name: /^(edit|save|restore|delete)\b/i })).toHaveCount(0); await expect(dialog.locator('textarea, [contenteditable="true"]')).toHaveCount(0); await testInfo.attach("read-only-document-revisions", { body: await page.screenshot(), contentType: "image/png" }); - // The handler's own click moves focus. Keep it out of this focus-restoration - // assertion, then resume ordinary dismissal before the next navigation. - await page.removeLocatorHandler(dismissAnnouncement); await page.keyboard.press("Escape"); await expect(dialog).toHaveCount(0); await expect(openReceipt).toBeFocused(); - await page.addLocatorHandler(dismissAnnouncement, dismissAnnouncements); const auditLink = casePage.getByRole("link", { name: "View in Audit", exact: true }); await expect(auditLink).toHaveAttribute("href", `/${company.issuePrefix}/activity?entityType=case&entityId=${receipt.id}`); await auditLink.click(); diff --git a/tests/e2e/runner-e2e-dashboard.spec.ts b/tests/e2e/runner-e2e-dashboard.spec.ts index e9e1d8e550e..e9cc5c5237b 100644 --- a/tests/e2e/runner-e2e-dashboard.spec.ts +++ b/tests/e2e/runner-e2e-dashboard.spec.ts @@ -363,3 +363,32 @@ test("case status dots summarize matching results and open failures", async ({ p await page.getByRole("button", { name: "Legacy Codex · Daytona sandbox", exact: true }).click(); await expect(topLink).toHaveAttribute("data-case-status", "failed"); }); + +test("keeps long feedback fingerprints inside a scrollable review table on phones", async ({ page }) => { + const { emptyFeedbackSummary } = await import("../runner-e2e/feedback.js"); + const { feedbackReviewTemplate } = await import("../runner-e2e/feedback-dispositions.js"); + const execution = runnerExecutionById(executionIds[0]); + const entry = resultFor(execution, "passed"); + entry.result.feedback = { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", + complaints: 2, bodiesCollected: 2, + review: ["a", "b"].map(value => ({ fingerprint: value.repeat(64), kind: "complaint", occurrences: 1, disposition: "needs_investigation" })), + }; + const campaign = buildRunnerCampaign({ campaignId: "feedback-mobile", generatedAt: entry.result.finishedAt, expected: [execution.id], results: [entry.result] }); + campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent", + decisions: [{ fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 }, + { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) }], + }; + await page.setViewportSize({ width: 390, height: 844 }); + await openReport(page, renderRunnerE2EDashboard({ title: "Feedback review", generatedAt: campaign.generatedAt, + expected: campaign.expected, catalog: [execution], entries: [entry], campaign })); + await page.getByText("Campaign usage and spend", { exact: true }).click(); + const decisions = page.getByRole("region", { name: "Feedback review decisions" }); + await expect(decisions).toBeVisible(); + expect(await decisions.evaluate(element => element.scrollWidth > element.clientWidth)).toBe(true); + expect(await page.evaluate(() => document.documentElement.scrollWidth)).toBeLessThanOrEqual(390); + await decisions.focus(); + await page.keyboard.press("Tab"); + const followup = decisions.getByRole("link", { name: "#123" }); + await expect(followup).toBeFocused(); + await expect(followup).toBeInViewport(); +}); diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index e1a34eee982..e499517506b 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -1011,3 +1011,30 @@ The arithmetic journey records four delivered agent handoffs and one completed terminal intent. The final case is `done` at handoff version 5, while the same four provider work runs and unchanged workflow document revision finish the task. The oracle rejects a missing or pending terminal intent. + +## Feedback persistence (explicit only) + +`--suite feedback-persistence` selects two local native Claude/Codex cells. +Each creates one standard task through the browser, asks for two distinct +synthetic feedback items, and requires the real `submit_complaint` and +`submit_suggestion` tools. The five-minute case expects one task work run and +disables automatic retries. Company and agent accounted-work budgets are $10. +Setup probes and any background work must also pass through the separately +budgeted gateway when using an operations manifest. Local compute is unmetered. +Cost and duration forecasts remain unmeasured until a retained live attempt. +The runner E2E maintainers own this manual case; `--all` excludes it. + +After collection and fixture teardown, the oracle reads the retained evidence +file and requires exact nonce-bearing bodies, the original company/agent/task +and observed run, plus one completed native receipt for each tool. Missing or +partial collection, prose-only claims, foreign attribution, duplicate records, +and failed submissions cannot pass. Full bodies stay in private evidence; +public reports retain counts, fingerprints, and closed oracle checks. The usual +result screenshot, trace, secret scanning and supervisor cleanup still apply. +This case proves explicit submission and persistence, not spontaneous feedback +quality or survival after an abrupt worker kill. + +```sh +pnpm test:e2e:runner -- --list --suite feedback-persistence +pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1 +``` diff --git a/tests/runner-e2e/OPERATIONS-COVERAGE.md b/tests/runner-e2e/OPERATIONS-COVERAGE.md new file mode 100644 index 00000000000..da4e2efa6ec --- /dev/null +++ b/tests/runner-e2e/OPERATIONS-COVERAGE.md @@ -0,0 +1,232 @@ +# Eval coverage and operating costs + +Use this with [OPERATIONS.md](OPERATIONS.md). The bounded daily and weekly +selections cover Product E2E only. They do not authorize or cap the separate +Runner protocol workflow, manual matrix runs, or other live probes. + +## Initial cadence + +The runner E2E maintainers own these selections and their failure triage. +Both profiles use the exact model and limits in the reviewed pricecard. + +| Journey | Failure detected | Feature-map association | Cadence | +| --- | --- | --- | --- | +| Native Claude message marker | A real task cannot complete and persist its expected answer through the metered route | `runs-adapters/runs` | Daily and weekly | +| Native Claude complaint and suggestion | Completed tools fail to save attributed feedback or collection loses it | `activity/agent-commentary` | Daily and weekly | +| Native Codex message marker | The corresponding Codex task/connection path fails | `runs-adapters/runs` | Weekly | +| Native Codex complaint and suggestion | The corresponding Codex feedback/tool path fails | `activity/agent-commentary` | Weekly | + +These associations identify partial coverage. They do not qualify every run +lifecycle, the Commentary browser page, continuation, or remote execution. +The October 21 local pilot measured one successful sample per selected cell: +$0.427624800 and 72.757 summed attempt seconds for the daily selection, +$0.791539400 and 127.069 seconds for the weekly selection. These are historical +gateway estimates from the Linux ARM64 container. They include model setup +requests and attempt setup/teardown, but omit queue/build time and non-model +charges. Earlier failed and interrupted attempts remain in campaign spending. + +`scheduled-measurements.json` retains the exact attempt, source revision, +definition, model, price version, result hash and journal hashes for each sample. +`forecast.json` includes this evidence and a snapshot hash. It separates the +historical successful selection from samples matching the current definition, +model, environment, runtime, route and pricing. A missing matching cell makes +the compatible selection total unknown. The feedback definition changed after +these four samples; its earlier results are not current qualification. + +Per-cell mean and median are descriptive. P95 stays unknown below 20 samples +and then uses nearest rank; this is not a statistical confidence guarantee. +Do not sum per-cell percentiles into a campaign percentile. Hosted cost/time +and the selection's P95 remain unknown until separately measured. A passing +sample does not predict failure frequency or account for infrastructure, +storage, observability or subscriptions. Forecasts never reduce reservations +or the approved budget limits. + +At the October 21, 2026 catalog audit, 47 suites declared 738 cells. `--all` +included 179; the other 559 required explicit selection. The bounded weekly +selection included four cells, and daily two. These are dated inventory counts, +not live acceptance results or an enduring catalog-size contract. Inspect the +current `pnpm test:e2e:runner -- --list` before choosing work. In particular, +Everyday Workflows, provider connections, feedback persistence, interruption, +and case handoffs must not be inferred from a passing `--all` campaign. + +Keep the broader rotating lane disabled until its exact cells have a metered +route, complete retained attempts, a measured cost/time distribution, and a +budget that includes failures. Prefer targeted continuation, approval, and +cleanup cases that detect different failures over another model running the +same marker. A release lane may broaden coverage only with its own reviewed +manifest; the daily/weekly allowance does not authorize it. + +## Adding an eval or changing its cadence + +The change's fixture documentation and PR must declare: + +- A responsible maintainer/team and the concrete failure the case detects. +- Feature-map capability IDs, or an explicit unmapped gap. State what the + journey does not exercise within a broader capability. +- Exact suite, case, profiles, models, environment, and intended cadence. + State whether it is explicit-only and how it differs from existing coverage. +- A deterministic alternative considered, and why paid behavior is necessary. +- Attempt, request, output-token, concurrency, retry, and wall-time limits. + Include setup, judges, child/background work, and cleanup in the route audit. +- Estimated cost/time before the first run, then all-attempt measurements with + sample count, retained failures, source/definition IDs, and uncertainty. + An unmeasured field is unknown, never zero. +- Feedback collection/review, trace verification, missing-receipt handling, + and a failure/cleanup owner. Novel complaints are reviewed independently + of the product grade. + +Use the existing [Product E2E authoring skill](../../.agents/skills/add-product-e2e-eval/SKILL.md) +or the separate [Runner authoring skill](../../.agents/skills/add-runner-eval/SKILL.md). +Update the existing scheduled selection and forecast only after qualification; +adding a catalog entry alone must not enable recurring spending. + +## Reading campaign time + +The report's summed attempt time uses every retained accounting receipt, +including failed retries. A missing attempt makes the total unknown; the report +keeps the observed subtotal and missing count. The observed attempt window runs +from the earliest retained start to the latest retained finish. Parallel attempts +overlap in that window, while their individual durations add to the sum. Gaps +between attempts remain in the window. Carried-forward evidence can span older +runs, so this window is not the current workflow's duration. + +Agent execution and Daytona lease time remain separate measurements. Attempt +durations include the setup and cleanup inside their recorded boundaries. +New browser-worker results retain a separate phase receipt for fixture setup, +execution, feedback capture, cleanup and worker finalization. Repeated cleanup +segments add together. Atomic checkpoints preserve completed phases if the +worker is interrupted; the active phase has no inferred end time. Parent +recovery checks identity, interval bounds and file size before retaining that +partial receipt. Missing, malformed or unavailable receipts keep phase totals +unknown, while closed segments remain visible as observed subtotals. + +These phases start inside the test callback, after browser/server startup, and +end before result publication. Parent teardown/finalization, CI queue/build time +and provider-billed runtime need their own receipts. Historical results and +non-browser launchers without a worker receipt remain explicitly unmeasured. +Test details show each selected attempt's receipt status and raw timing metadata. +Do not infer missing measurements by subtracting summed agent time from the +observed window. Refreshing a report computes these views from retained +results without making provider calls or changing the original measurements. + +## Costs outside the model journal + +Keep a separate cost component for each resource, with its currency, service, +resource/request identity, source window, allocation basis, and status. Join +components to campaign/attempt IDs where evidence permits. Do not sum a model +estimate with the invoice that later reconciles that same usage. + +| Component | Evidence and allocation | When evidence is missing | +| --- | --- | --- | +| Provider models and paid tools | Dedicated project/key usage and billing exports; match provider request IDs or the narrowest available billing window | Preserve estimates, unknown requests, held reservations and the reconciliation difference | +| Daytona | Owned sandbox ID, actual resources, billable lifecycle and provider receipt; include failed creates, retries, stopped storage and delayed deletion | Retain the reservation or an explicitly partial estimate; a requested TTL does not prove deletion | +| CI compute | Workflow/job IDs, runner type, actual billed runtime and invoice rates; include setup, builds and failed jobs | Mark CI cost unknown; a short agent turn does not imply a cheap CI job | +| Storage and transfer | Artifact size, retention window and billed storage/egress | Report the unpriced component, including retained failure evidence | +| Braintrust/gateway usage | Project usage export and its window; distinguish included allowance from incremental overage | Do not treat free-looking trace metrics as proof of zero provider charges | +| Shared subscriptions | Show the existing Braintrust Pro $249/month separately under the October 21 operator policy; allocate newly incurred subscription/capacity charges from invoices with an explicit denominator | Keep new shared charges unallocated rather than assign an invented exact per-test cost | + +Retain both the service total and the allocation denominator so allocated +amounts can be reproduced. A zero incremental usage charge can coexist with a +nonzero shared subscription. Never add the entire subscription to every test. +The approved daily/weekly budgets cover additional run costs, so they exclude +the existing $249/month fee while including new usage and capacity charges. +Scheduled admission now reserves the declared CI, storage, observability and +subscription allowances inside the same $20/$100 journal limit. Missing or stale +allowances stop scheduling before the AWS test runner starts. Missing receipts +keep their reservations held and accounting incomplete. This is a local admission +control, not a provider billing cap. Actual resource bounds, cost collection and +reviewed allowances still need qualification before recurrence is enabled; see +[the allowance contract](OPERATIONS.md#recurring-campaigns). + +Recurring evidence and reports use private S3 archives with file/byte/request +boundaries, retained transfer receipts and verified access/expiration rules; +see [private recurring history](OPERATIONS.md#private-recurring-history). The +public workflow retains only a content-free availability marker for monitoring. +Public history is a separate explicit publication step. Private retention still +needs hosted OIDC qualification and actual storage/transfer billing evidence; +local archive tests and an infrastructure template do not establish those facts. + +## Separate Runner Eval audit + +The October 21 audit inspected Paperclip source on master `8b873e0112` plus the +eval operations stack, and the available `paperclip-evals` checkout at +`0bae4e0947dc7959cae25b73d27533c096f23a14`. This is a source audit, not a +new live run or proof of the workflow's configured private-repository revision. + +Runner reports already retain immutable attempts, retry-inclusive estimated +cost and provider list cost as separate views, compatible-cohort comparisons, +and missing-usage coverage. Its mock control plane is a different authority +from the Product E2E server/database; Product feedback persistence cannot be +claimed from a Runner tool assertion. + +The private `run_live_campaign.py` requires an explicit authorization envelope. +The hosted `runner-protocol-live-evals.yml` instead fans out individual +`run_live_roster.py` cells, supplies their direct provider credentials, and uses +configured per-attempt limits. It does not use the Product E2E shared gateway +journal, Braintrust attempt parent, indexing verifier, or feedback finalizer. +The private campaign wrapper's aggregate authorization must not be assumed to +protect that different hosted path. + +Before adding Runner recurrence, its owners must establish an aggregate +allowance across fanout and retries, meter setup/provider bypasses, and qualify +native tracing through its actual packaged runner. Preserve direct-provider +qualification as its own route. Implement definition/roster changes in the +maintained private repository, pin that reviewed revision in the existing +workflow, and retain Runner's existing Evalbook renderer and scores. These +gaps remain open; Product E2E qualification does not close them. + +## Missing campaign detection + +Workflow failure notifications cover runs that start. The independent observer +is `scripts/eval-schedule-monitor.mjs`. Run it through a separate host with an +authenticated GitHub CLI and one private state file: + +```sh +node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json" +``` + +The observer uses only GitHub GET requests. It never starts or retries paid +work and does not need provider credentials. It reads the two existing enable +gates plus `RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED_AT`, an ISO timestamp set +when recurrence is enabled. Missing activation time fails visibly; occurrences +before activation are not treated as missed. Workflow run titles carry the +literal cron expression to distinguish daily and weekly runs, including Sunday. + +Only the legacy `RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED` gate being off establishes +disabled recurrence. If that gate is on while the operations gate is absent or +off, the observer reports `configuration_incomplete` and retains one incident +notification. Older deployed workflows use only the legacy gate, so a missing +new gate must not imply that those paid schedules are disabled. Verify the +deployed default-branch workflow and both variables before enabling recurrence. + +The observer allows 90 minutes for schedule delay. It checks the most recent due +occurrence per cadence, marks a run overdue after four hours, and requires a +successful terminal run plus a nonempty, unexpired merged-report artifact. +The workflow's finalizer remains responsible for checking report content and +accounting. Metadata inspection does not repeat that validation. Unexpected +reruns, ambiguous matching runs, incomplete API lists and API failures remain +visible. The observer reads at most 100 rows per collection and fails closed +if a response is truncated. + +Retained state includes the last successful inspection and inspected run IDs. +`notifications` is a durable outbox of problems and recovery transitions. Each +entry has an ID. After confirmed delivery, the host acknowledges those IDs: + +```sh +node scripts/eval-schedule-monitor.mjs "$HOME/.config/paperclip-evals/schedule-monitor.json" --ack ... +``` + +Until acknowledgment, later inspections return the same pending entries even +if the problem has recovered. The host uses IDs to deduplicate delivery. This +survives a process exit between saving state and printing output; it does not +claim exactly-once delivery across an external notification service. After +acknowledgment, an unchanged problem stays quiet. A new missed occurrence is a +new incident. Disabled recurrence and unchanged healthy runs create no new +alerts. Use one writer for both checks and acknowledgments; do not reset the +state file to clear an incident. Malformed history fails before a query or +write. The external host owns notification delivery and its own liveness. + +Schedule this observer outside the paid workflow after source and recurrence +are ready. It should inspect at least daily, after both scheduled start times +and their grace windows. Until that external trigger and notification delivery +are configured and exercised, missed-run detection remains an activation gap. diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md new file mode 100644 index 00000000000..31b36ed856f --- /dev/null +++ b/tests/runner-e2e/OPERATIONS.md @@ -0,0 +1,707 @@ +# Bounded eval operations + +The request ledger and gateway are opt-in. The ordinary launcher still supports +unmetered legacy campaigns. Do not infer a spending limit from a campaign ID. +Enable the bounded path with `PAPERCLIP_E2E_OPERATIONS_MANIFEST=/absolute/file.json`. +Keep this manifest, its journal, and request receipts private. + +The [coverage and operating-cost guide](OPERATIONS-COVERAGE.md) records cadence +ownership, suite-growth requirements, non-model allocation, the separate Runner +Eval audit, and remaining activation gaps. + +## Supported admission + +The initial path admits local `core-compatibility` and `feedback-persistence` cells for `runner-codex`, +`runner-acpx-claude`, `legacy-codex`, and `legacy-claude`. Each cell must have one +provider credential and an exact priced model. Other suites, remote environments, +subscription profiles, hosted tools, multimodal inputs, premium service modes, +and separate judges fail admission. This whitelist is implementation scope, not +evidence that every admitted harness has passed live qualification. + +Before creating the journal or requesting credentials, manifest validation checks +that every selected model's worst-case single-request reservation fits the initial +campaign allowance after declared operating holds. Prices for unused models do +not affect this selection check. Passing it does not promise the whole campaign +will fit: later requests still use the journal's current commitments and limits. + +Bounded daily and weekly workflows check source-byte limits before upload: 16 MiB +for the resolved lockfile, 1 GiB for the build bundle, and 256 MiB each for cell +evidence and the merged report. Build artifacts expire after one day; the others +expire after 30 days. These are source-byte and retention controls, not measured +compressed storage charges. ZIP framing and platform billing need separate receipts. + +Evidence and reports are copied into fresh private staging directories. Up to +4,096 directory entries are inspected. Discovery visits accounting and snapshot +directories across attempts before archive and Playwright diagnostic directories. +Accounting under `scheduled/` and JSON records have copy priority over logs, +images and archives. Each retained file has a +SHA-256 digest. The staging receipt reserves 8 MiB inside the total and identifies +missing, changed, unsafe or oversized files and incomplete traversal. Originals +are never deleted or rewritten. Hidden paths remain excluded. The expected regular +`playwright-output/.last-run.json` bookkeeping file is outside the retained evidence +scope and does not fail retention. Other hidden paths make the scope incomplete. +Skipped paths and unvisited subtrees are named; overlong names use a bounded preview +and full-path digest. Partial staging is uploaded before the workflow +fails, so a storage limit cannot silently turn missing evidence into success. +These upload limits do not cap runtime disk writes or Actions caches. Each service +still needs billing evidence. + +### Private recurring history + +Daily and weekly evidence and merged reports are retained in a dedicated private +S3 bucket. They are not uploaded as plaintext GitHub artifacts, sent to the public +history publisher, or deployed to Pages. The report command's output also stays +inside the private report. The public workflow retains only a fixed availability +marker for the existing schedule monitor, with no report contents, storage address +or object identity. Public-repository GitHub artifacts are readable by signed-in +repository readers; authentication alone does not make those reports private. + +Deploy `.github/infra/runner-e2e-private-history.yml` with the existing GitHub OIDC +provider and set `RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET`, +`RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN` and +`RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION`. The role trusts only this repository's +`runner-e2e-private-paid` and `runner-e2e-private-history` environments. Create both +with a deployment branch policy allowing only the default branch. Reserve them +for this workflow: the shared `runner-e2e-paid` and `runner-e2e-history` environments +also serve protocol evaluations that execute target-branch code and must not +have access to the private archive role. Set the three archive variables in both +dedicated environments. This workflow separately requires the default branch before +paid access and reporting. Manual target-branch tests have no OIDC permission. A separate recurring +job receives that permission only when the target SHA equals the default-branch +workflow SHA. Both jobs reuse the same anchored setup and test steps. The role has configuration reads +and GetObject/PutObject under `runner-e2e-private/bounded-v1/`; no deletion, +policy changes or access to the public history bucket. AWS credentials are step +outputs supplied only to private transfer commands, never exported to paid agents. + +Admission verifies retention before paid execution. Every transfer verifies all +four public-access blocks, disabled object ACLs, no bucket-policy reader grants, +versioning, the 30-day/current and one-day/noncurrent expiration rules, and absence +of Object Lock and replication. Inspection failure stops the transfer. The private +bucket is not a CDN origin and has no external reader grant. + +Evidence and report archives each contain at most 256 MiB of staged files and +4,097 entries including the staging receipt, with an additional 8 MiB ceiling for +ZIP framing. Archives use stored entries, without decompression. Extraction checks +paths, entry types, duplicate names, directory size, payload totals and ZIP checksums +before committing a fresh destination. Transfers use one conditional PutObject or +one ranged GetObject. Automatic AWS retries and workflow reruns are refused. The +bucket also rejects PutObject requests without an If-None-Match condition. A local +transfer receipt cannot be overwritten by rerunning the command in the same directory. +Each command retains admission and outcome metadata. After private storage passes +inspection, completed and failed transfers each get one independent receipt-upload +attempt without repeating the original transfer. Successful receipts include the +archive hash and object version; lost responses remain failed or unknown. A failed +privacy inspection prevents all uploads, and an unsuccessful receipt upload can +leave only the local receipt. Its own upload remains pending in the snapshot, +so the snapshot is not proof of complete billing. Operator downloads and STS +credential exchanges also need separate cost attribution. + +For authenticated retrieval, set the bucket variable and AWS profile, then run: + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download report RUN_ID /tmp/eval-RUN_ID +``` + +Use `evidence` instead of `report` for the original staged accounting and diagnostics. +The requested destination must not already exist. Keep the downloaded report private. +To publish a selected campaign, review its retained normalized report and invoke the +existing trusted history publisher explicitly with the public destination; no agent +rerun is needed. Recurring execution never invokes that publication path. + +Reporting and publication install only the committed lockfile from the trusted +workflow revision, with lifecycle scripts disabled and a job-local package store. +They never resolve new dependency versions while the job has OIDC access. If +that revision has an outdated lockfile, reporting fails before accessing the +archive. Use the normal master lockfile maintenance process; do not bypass the +frozen install in the credential-bearing job. + +### Public history retention + +Public history remains an explicit manual operation. The optional bounded publisher +uses `/bounded-v1/` when `PAPERCLIP_E2E_BOUNDED_HISTORY=true`; existing +manual history otherwise retains its original prefix. Bounded publication is limited +to the original CI workflow attempt and is not retried under the same allowance. + +Before any upload, the publisher validates all outgoing bundles: at most 64 MiB +and 256 files for the immutable campaign including its manifest, 16 MiB and eight +files for mutable history, and 64 MiB for the Pages bundle. History JSON reads +are limited to 4 MiB. The active index keeps at most 64 campaigns from the last +28 days and rebuilds latest/green pointers from those records. If publication +stops, the static index can retain links that later expire. + +The bucket must have versioning enabled and exact prefix lifecycle rules for +30-day current-version expiration, one-day noncurrent-version expiration without +a retained-version floor, expired delete-marker removal and one-day incomplete +multipart cleanup. The publisher rejects overlapping transitions, different +expiration policies, Object Lock and replication. Access denied is a failure, +not proof that a feature is absent. Its role needs `s3:GetBucketVersioning`, +`s3:GetLifecycleConfiguration`, `s3:GetBucketObjectLockConfiguration` and +`s3:GetReplicationConfiguration` on that bucket. Configuration is verified at +each publication; it cannot guarantee that an operator will preserve it afterward. + +The publisher admits at most 280 S3 CLI commands over ten minutes, with a +30-second command timeout and automatic AWS retries disabled. Uploads use +single `PutObject` operations in STANDARD storage; immutable objects use +`If-None-Match`. A private receipt is synced before each command and retained +locally for the operator to retain privately. Unknown responses remain unknown. Source bytes, S3 +commands and expiration eligibility are not invoices or exact deletion times. +Credential lookup, public downloads/CDN egress, Pages hosting and shared service +charges require separate accounting; these limits do not cap public traffic. + +The launcher owns one loopback HTTP gateway per attempt. The real provider key +stays in the parent launcher. Playwright receives an ephemeral token, and fixture +setup saves that token through a shared AI connection with the existing custom +Responses/Messages routing contract. The agent does not receive ambient provider +keys. The server environment strips the gateway token and upstream credentials. + +Every upstream attempt reserves its maximum estimated charge in an fsynced +journal before dispatch. Requests, retries, and cancelled calls retain distinct +IDs. Unknown usage keeps the reservation held. One shared writer serializes +admission across cells, so concurrent requests cannot each spend the same remaining +allowance. Campaign deadline, request count, and in-flight count are also enforced. +Private request receipts retain transport IDs, response IDs, usage and cache TTLs, +route/cache/failover evidence, and final accounting status without request bodies +or credentials. Raw provider responses remain only in the normal private evidence. + +## Manifest + +A manifest contains no credentials. This structural example uses fixture prices; +replace them with independently verified model prices and context limits before +running a real campaign. The launcher does not discover current prices for you. + +```json +{ + "version": 1, + "limits": { + "campaignId": "qualification-example", + "maxUsd": "100", + "deadlineAt": "2026-10-20T23:00:00Z", + "maxRequests": 200, + "maxConcurrent": 1 + }, + "ledgerPath": "/private/path/qualification/ledger.jsonl", + "executionIds": ["EXACT_SELECTED_EXECUTION_ID"], + "maxAttemptsPerExecution": 2, + "route": "direct", + "models": [{ + "provider": "anthropic", + "maximumInputTokens": 1000000, + "maximumOutputTokens": 16384, + "timeoutMs": 180000, + "price": { + "provider": "anthropic", + "model": "EXACT_SELECTED_MODEL", + "version": "verified-pricecard-date", + "sourceUrl": "https://platform.claude.com/docs/en/about-claude/pricing", + "inputUsdPerMillion": "2", + "cachedReadUsdPerMillion": "0.2", + "cacheWrite5mUsdPerMillion": "2.5", + "cacheWrite1hUsdPerMillion": "4", + "outputUsdPerMillion": "10" + } + }] +} +``` + +`executionIds` must equal the launcher's selected set, and the retry option must +fit `maxAttemptsPerExecution`. Use `--list` to resolve the selection first. +Operations records this attempt cap in the journal's immutable limits and fsyncs +each attempt admission before starting its gateway. The count is per execution +across the whole shared journal, including restarts, renamed output campaigns, +and attempts that fail before making any provider request. Changing the cap or +reusing an admitted attempt ID is refused. +The maximum input value is the verified provider context ceiling, **not** a +requested input cap. The gateway clamps output to `maximumOutputTokens`; it does +not tokenize or truncate prompts. The conservative reservation prices the whole +input ceiling at the highest cache-write/long-context rate, plus capped output. +A small daily budget may therefore admit Claude but refuse Codex even when the +expected charge is small. Do not reduce the context ceiling to bypass this guard. + +For modern OpenAI models, specify both `cacheWrite30mUsdPerMillion` and +`cacheWrite12hUsdPerMillion`. Optional `longContext` contains `aboveInputTokens` +and a complete `rates` object with the same token-rate fields. Above that total +input threshold (including cache), the estimate uses those rates for the entire +request. Reservations use the maximum rate across both tiers. Missing cache-write +counters, ambiguous TTL splits for nonzero writes, and model mismatches remain +unknown. Older OpenAI models without these usage fields need separate qualification. +Write prices are total token prices, not surcharges. Sources: +[OpenAI caching](https://developers.openai.com/api/docs/guides/prompt-caching), +[OpenAI prices](https://developers.openai.com/api/docs/pricing), and +[Claude prices](https://platform.claude.com/docs/en/about-claude/pricing). + +For `gpt-5.6-sol`, a response that omits the entire write-TTL breakdown can use +the actual request's documented implicit cache policy: `prompt_cache_options.ttl` +or its 30-minute default. This fallback requires a visible input, no explicit +breakpoints, no stored prompt/conversation/response references, and no compaction. +The private receipt records `cacheWriteAttribution` with the policy version and +lifetime. These amounts remain estimates. Provider splits take precedence; +partial or malformed splits are never repaired. `prompt_cache_retention: "24h"` +does not imply a 12-hour write lifetime. The gateway does not change cache policy, +and this fallback does not reprice historical requests or release old holds. + +For Braintrust, use `route: "braintrust"`, add the exact `projectId` UUID, and +provide `BRAINTRUST_API_KEY` securely. Configure project providers named +`OPENAI_API_KEY` and `ANTHROPIC_API_KEY`. Disable default provider fallback in the +project. The request pins that endpoint and project, disables gateway response +caching, and never automatically retries or requests fallback. A cache hit, +failover header, or unexpected successful endpoint invalidates the receipt and +holds its reservation. Provider prompt caching stays enabled and is metered. +The response stream is passed through and cumulative usage is counted once. +Gateway and native tracing are separate opt-ins; see the README tracing section. + +When both a bounded operations manifest and explicit Braintrust tracing are +configured, the launcher starts a separate loopback OTLP relay for each attempt. +The worker and its native providers receive an expiring ingest token. The +Braintrust service key stays in the controller, even when model requests use +Braintrust without native tracing. The relay accepts only trace/log exports to +the configured project; neither request headers nor payload parent metadata can +select another project or an API endpoint. JSON is canonicalized before export; +protobuf payloads retain their original fields after bounded validation, which +rejects ambiguous singular fields and routing values. + +Each relay allows at most 2,048 admitted exports, 8 MiB per request before and +after decompression, 64 MiB of forwarded payloads, and four simultaneous exports. +It also limits each attempt to 16,384 records and 256 MiB of expanded source +JSON. This measurement includes the full decoded source plus each record in +its own envelope with the shared resource and scope fields repeated. It counts +span names, IDs, times, events, links and log bodies, not only routing attributes. +Protobuf fields use the pinned OTLP v1.9 schema; unknown fields fail admission +until the schema is updated. Existing valid payloads are forwarded unchanged. +It closes at the campaign deadline or four hours after startup, whichever comes +first. Admission consumes the allowance before upstream dispatch. Failures and +client retries do not refund it; the relay itself never retries. A durable receipt +must be written before dispatch and before acknowledging the receiver outcome. + +Version 2 private `.otlp/.jsonl` receipts record policy, +admission time, wire/payload bytes, payload digest, dispatch/outcome, partial +rejection counts, record counts, expanded source bytes, and closure totals. +Version 1 receipts remain readable with unknown expanded size and record count. +They contain no payload bodies, tokens or +upstream error text. Receipt files are exclusive and cannot be reset on restart. +An admission without an outcome is unresolved. Accepted ingestion does not prove +indexing, and attempted forwarding bytes are not billed Braintrust storage. +These receipts do not replace the observability overhead reservation or invoice +reconciliation. After the worker relay closes, bounded controller finalization +uses a separate one-request, 64 KiB relay with a 30-second deadline. Its exclusive +expanded-source allowance is 1 MiB with at most 256 records. Its +receipt is `.otlp/-finalization.jsonl`. This preserves +the original trace and grade while accounting for the final controller export. +This path currently supports the admitted local cells; remote transport remains +unqualified. + +Scheduled experiment creation, grade insertion and summary retrieval share a +60-second window, three-request cap, 4 MiB total payload allowance and eight-score +cap. They run serially without automatic retry. Admission is persisted before +dispatch in `.experiment-ingestion.jsonl`; uncertain responses retain +their allowance and unresolved status. Reusing that file fails instead of +resetting the budget. These limits cover the selected daily/weekly campaigns; +the standalone experiment-import command does not use this transport budget. + +Private `ingestion-usage.json` binds all three receipt paths to the campaign's +admitted attempts and ledger hash. Missing, malformed, unclosed or unresolved +receipts leave the total unknown while preserving readable partial measurements. +These are admitted transport payload bytes, not Braintrust processed-data units +or an invoice. Vendor charges remain unreconciled. The current two-attempt daily +maximum is 132.125 MiB and the four-attempt weekly maximum is 260.25 MiB, +including finalization and the campaign's experiment payload allowance. These +bounds do not include other projects, standalone imports or retained storage. +The corresponding expanded-source ceilings are 514 MiB daily and 1,028 MiB +weekly before the separate 4-MiB experiment allowance. These source measurements +support a priced forecast; they are not Braintrust's processed-data meter or a +vendor-enforced dollar limit. Gateway auto-logging is a separate input to that +forecast. Retain vendor usage reports for reconciliation rather than equating +transport or expanded source bytes with an invoice. + +## Recurring campaigns + +The existing `runner-full-stack-e2e.yml` owns both schedules: daily at 08:47 UTC +and weekly on Sunday at 09:47 UTC. Scheduled runs require both +`RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED=true` and +`RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED=true`. Leave the operations gate unset +until the workflow, billing coverage and infrastructure allowance are qualified. +There is no separate scheduler. The protected `runner-e2e-private-paid` environment needs +`BRAINTRUST_API_KEY`; repository variable `RUNNER_E2E_BRAINTRUST_PROJECT_ID` names +the exact project UUID. The key is available only to the bounded execution and +finalization step, after dependency installation and immutable builds. + +Native Claude gateway cases also require the pinned Claude CLI for the managed +account's setup hello probe. The agent still runs through the pinned ACP runtime. +The workflow installs and checks the CLI before providing credentials for both +native and legacy Claude selections, including the aggregate scheduled selection. +For a local campaign, verify that same CLI prerequisite before starting a paid +attempt. Installing the ACP package alone does not satisfy this setup path. + +Bounded cases keep the gateway in the setup phase until the created agent's +asynchronous lifecycle verification reaches ready. This wait uses the remaining +attempt deadline, allows at most 45 seconds, and stops on an observed setup error +or hold. The private `budgeted-setup.json` snapshot retains state transitions. +Only then does the case begin agent-phase accounting and create its task. +Older evidence that advanced after fixture creation may label hello probes as +agent cost. Its campaign totals still include those requests; retain that phase +limitation when comparing setup and task spending. + +Attempt wall time includes setup and teardown. Agent time is the sum of captured +heartbeat-run windows, which can overlap. Missing runtime evidence, no captured +runs or an invalid run timestamp leaves agent time unknown; the report never substitutes attempt +wall time. A campaign total stays unknown if one of its retained attempts lacks +that measurement. A measured zero remains zero. Missing or invalid Daytona lease +windows likewise leave lease time and its runtime estimate unknown. Local attempts +do not contribute Daytona lease time. Existing published numeric measurements are +retained as historical evidence; this change does not reconstruct their provenance. + +Bounded daily/weekly campaigns require a public repository and use the existing +standard `ubuntu-latest` runner for both compilation and execution. They do not +follow the manual campaign's AWS runner switch. GitHub charges no standard +hosted compute minutes for public repositories; retain runner/job evidence and +recheck the [billing policy](https://docs.github.com/en/billing/concepts/product-billing/github-actions) +when reviewing the allowance. This does not establish zero artifact, cache, +external history, observability or subscription cost. Manual selections keep +the existing AWS/GitHub runner choice. + +Daily selects native Claude's marker and feedback-persistence cases with a shared +$20 allowance. Weekly adds native Codex's two corresponding cases with a +shared $100 allowance. Model requests and declared operating overhead share +this limit. Each campaign runs sequentially in one job with one +journal, one attempt per execution, zero automatic retries, a 30-minute request +deadline and a finite request count. Codex's worst-case request reservation exceeds +$20, so it is absent from daily selection. Its context ceiling is unchanged. +The $20/$100 limits cover additional costs caused by each run. Show the existing +Braintrust Pro subscription separately at $249/month (operator policy confirmed +October 21, 2026); do not allocate it into these run limits. New processing, +scoring, retention or capacity charges still count toward the run allowance. +The monthly run allowance is at most $1,120 (31 daily plus five weekly runs), +or $1,369 including the separately reported existing subscription. These are +maximum allowances, not forecasts of actual spending. +This is an admission limit with retained reservations, not a reconciled invoice +total or a provider-enforced billing limit. + +Before enabling recurrence, set `RUNNER_E2E_OPERATING_OVERHEAD_JSON` to a reviewed +allowance with `version: 1`, an evidence `reference`, the actual `runner` +(`github-hosted` for bounded schedules), and `verifiedAt`/`expiresAt` timestamps no more +than 31 days apart. Its `components` array must contain exactly `ci_compute`, +`artifact_storage`, `observability`, and `shared_subscription`. Each component +needs a USD-string `maxUsd` and its own evidence `reference`. References are +stable audit IDs or paths, not credentials or arbitrary prose. No default +amounts are supplied. The supporting audit must include rates, resource/time +bounds, included allowances, allocation denominators, and the controls that +keep actual resource use within those bounds. Zero requires evidence of an +included or zero incremental charge; setting a zero allowance is not a receipt. +Under the current policy, `shared_subscription` may reserve zero for the +existing Braintrust subscription with a reference to that policy and current +plan evidence. Any newly incurred subscription or capacity charge needs its own +incremental allocation; the existing fee must remain visible separately. + +The catalog checks that this allowance matches the selected runner, covers at +least the next four hours, and leaves room for a full conservative model-request +reservation. This happens before the build and test runners start. The workflow passes +the exact validated allowance to preparation; it does not read a potentially +changed repository variable again. Preparation rechecks it and retains it in the +manifest and forecast. Setup, builds, failed jobs, publication and retained +artifacts must all fit the declared components, not just the agent's runtime. + +The journal initializes `overhead:` reservations under +`campaign-overhead` from its immutable limits before admitting model requests. +These are allocation holds, not dispatched requests or completed resource +lifecycles. They do not occupy model concurrency, but they count toward the +finite charge-record limit. Resuming cannot change or drop them. Until matched +cost observations arrive, the report shows them as unknown and keeps their +allowance held; even a zero allowance does not claim a known zero charge. +The existing finalizer reports incomplete accounting while these receipts are +missing. Do not dismiss that finding to enable recurrence. + +Reconcile a component through its existing request ID with provenance-bound +`infrastructure` or `invoice` observations. Partial observations keep the +remaining reservation held; a final nonconflicting observation can replace it. +Never mark a receipt final before its resource and billing window are complete. +An external bill above the allowance remains visible and stops further model +admission. The journal cannot stop AWS, storage or subscription billing itself; +provider resource limits and a verified receipt collector remain activation +requirements. No operating-cost allowance has been configured or qualified by +the presence of this code. + +Collect retained GitHub usage after the workflow completes, including failed or +cancelled workflows, without dispatching or retrying anything: + +```sh +node --import tsx tests/runner-e2e/operating-receipts.ts github RUN_ID /private/path/github-usage.json +``` + +The collector uses the existing `gh` authentication and only reads +`paperclipai/paperclip`. It verifies the Product E2E workflow, first attempt, +revision and repository, reads every returned job/artifact page up to 1,000 +records per inventory, and rejects changed or incomplete inventories. It retains +job identities, summed job seconds, run-start delay, artifact sizes and expiration +dates. It omits arbitrary job names, artifact names and upstream responses. +Existing output files cannot be replaced. This is a current inventory, not an +invoice: deleted artifacts are absent, compressed bytes are not billed +byte-hours, job duration is not VM lifetime, and present repository visibility +does not establish visibility throughout an earlier run. Caches and external +S3 history remain separate. A visible public-hosted zero compute rate never +automatically releases an overhead reservation. + +Import a reviewed cost receipt and its exact source evidence into an existing +campaign's declared overhead reservation: + +```sh +node --import tsx tests/runner-e2e/operating-receipts.ts import /private/path/ledger.jsonl /private/path/receipt.json /private/path/source-evidence.json +``` + +The receipt contains `version: 2`, `costBasis: "component_cumulative"`, +`campaignId`, `allowanceReference`, `component`, +the source file's `evidenceSha256`, `windowStartedAt`, `windowEndedAt`, `coverage` +(`partial` or `complete`), `source` (`infrastructure` or `invoice`), `provider`, +`classification`, and an explicit USD-string `costUsd` when known. Estimated and +allocated charges also require a stable `pricingVersion`. `reconciled` requires +an invoice source. An unknown amount has no `costUsd` and always remains partial. +The observation window must have ended; completeness is the reviewer's assertion +that this component's entire resource and billing window is represented. A +checksum proves which file was used, not that the invoice or allocation is true. +Shared allocations must record their numerator, denominator, window and excluded +costs in that source file. Do not import a whole shared invoice as one campaign's +charge without an explicit allocation method. + +Each amount is the cumulative total for the entire component, including all its +providers and separate charges through `windowEndedAt`. Later receipts must keep +the same `windowStartedAt` and cannot shorten the window or make completed +coverage partial again. Combine separate billing windows in the retained source +evidence before importing their total; two separate $0.80 charges require a +cumulative $1.60 receipt. Version 1 receipts lacked this declaration and are +rejected on import. Existing journal records are preserved; the importer also +refuses to extend a component whose prior receipts lack cumulative evidence. +It also checks the ledger's precedence rules before import: a larger cumulative +estimate cannot be hidden by an earlier, smaller invoice. Such imports fail +without changing retained evidence or the journal; reconcile the entire expanded +window with evidence the ledger can account for before retrying. + +The importer validates campaign/allowance identity and source hash, retains both +files privately under `ledger.jsonl.overhead/`, and syncs them before appending +the observation. Files are written and synced under unique temporary names, then +published atomically without replacing existing evidence. A partial write is +removed without occupying the final path, so retry needs no manual repair. +Retries sync matching existing files too. Reimporting an identical +receipt adds no journal entry. A partial amount +keeps the rest of the allowance held. Complete invoices can supersede estimates; +conflicting complete invoices remain visible and block admission. This command +does not open a new spending window, change limits, dispatch tests or rewrite +previous records. Regenerate the campaign report from the same ledger afterward; +an old report snapshot is not rewritten by an offline receipt import. + +The checked pricecard expires on November 19, 2026. Admission stops when the +pricecard expires, the catalog model changes, the project is missing, or the +preflight already exists. Review current official prices and context limits +before updating `scheduled-prices.json`. `forecast.json` records the selected +IDs, provenance-linked pilot measurements, compatibility gaps, uncertainty, +reservations and price expiry. The historical daily/weekly successful model +subtotals are $0.427624800/$0.791539400; they are not campaign bills. Setup model +calls and attempt setup/teardown are included, while CI setup/build/queue and +non-model charges are excluded. Current compatible totals stay unknown if any +selected cell lacks a matching measurement. Hosted timing is still unmeasured. +Per-cell medians describe only the retained samples; P95 requires at least 20 +matching samples and does not establish reliability. See +[measurement provenance and limitations](OPERATIONS-COVERAGE.md#initial-cadence). +Daily coverage is limited to the two Claude journeys. The feedback definition +has changed since its passing pilot. Broader rotation and Daytona remain +unqualified and are not silently included by `--all`. + +Dispatch `cadence: daily` or `weekly` from the default branch to qualify the same +path manually. Bounded dispatch rejects custom target revisions and selectors. +Bounded authorization pins the commit from the workflow event; a later merge +does not replace that revision or invalidate a queued campaign. +Workflow attempts after the first cannot start paid work with a fresh journal. +Rerunning reporting can reuse the original attempt-one evidence. Recover the +retained ledger for a deliberately resumed campaign; do not reset it or launch a +new campaign merely to hide a failure. The existing `manual` mode remains an +explicit, unbounded qualification path and does not inherit these allowances. + +After execution, deterministic review dismisses only exact synthetic feedback +bodies corroborated by all feedback-persistence graders. Novel feedback remains +`needs_investigation`; no extra model call is made. Missing accounting, unresolved +receipts, feedback collection/submission failures, novel feedback, export failure +and incomplete experiment import or indexing fail the job with bounded diagnostic codes. +Successful unchanged runs produce artifacts without posting messages. GitHub's +workflow failure notifications remain the operator's notification channel. +Missing/non-starting schedules require an external workflow monitor; this change +does not claim that a workflow can detect its own failure to start. + +The launcher starts the public Playwright JavaScript entry point directly with +Node and sends SIGINT for cancellation and watchdog cleanup. The package-manager +wrapper can exit before Playwright completes its shutdown, so it is excluded. +Playwright handles that signal by stopping workers and closing its web server; +SIGTERM can exit before web-server cleanup. The web-server supervisor still +receives SIGTERM and owns its server's shutdown. Both paths retain the existing +bounded grace period and forced cleanup for surviving owned processes. +When a worker exits without a result, the launcher also recovers its original +trace identity from the private fixture snapshots. The supervisor can finalize +that trace with the failed outcome. A mismatched project, execution or company +is rejected. Recovered identity does not prove native export or complete phases. + +If a worker exits before writing its result, the server supervisor spends at +most five seconds collecting feedback from that attempt's saved company before +stopping the isolated server. Recovery snapshots remain private and are kept +separate from the worker's original evidence. The failed result includes any +recovered counts and review fingerprints; active runs or missing reads remain +partial or unavailable. Each atomic recovery checkpoint carries its attempt +identity, so a later write timeout cannot hide already saved feedback when the +final receipt is absent. Completed results are never replaced. The launcher +retains the private database whenever a worker result is missing, including when +the supervisor was killed before it could capture feedback. This retention is +reported as unconfirmed cleanup and requires inspecting the retained state. +Authenticated public-MCP fixtures do not use the local operator recovery path. + +Original grades are imported into Braintrust without rerunning agents. The +access-controlled artifact includes the journal, forecast, feedback dispositions, +experiment receipt, indexing receipt and findings. The trusted report job rebuilds the dashboard +and retains its evidence-bound experiment link. Export acceptance and native +indexing are separate. A bounded, read-only Braintrust query verifies the attempt +root, finalization, native API activity and any expected feedback tools. It also +compares indexed experiment rows with original grades, revisions and attempt IDs. +The verifier tries at most three rounds within 45 seconds, retains missing/query +failures, and never reruns an agent. Native record counts measure export volume, +not distinct provider requests or charges. Codex exports can be high volume and +Braintrust can double-count native/log observations; use the deduplicated gateway +ledger as the estimate authority. Private indexing receipts contain only counts, +identities and verification status, without trace contents or credentials. + +## Inspection and limits + +Retained deterministic grades can be imported into a private Braintrust experiment +without invoking agents or judges. Prepare a JSON array of absolute paths to the +original `result.json` for **every** retained attempt in `normalized-results.json`. +The original files supply each attempt's actual source and definition hash; +the selected result's revision must not be assigned to older attempts. + +```sh +PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID=PROJECT_UUID \ +node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/braintrust-experiment-import.ts \ + /private/campaign/normalized-results.json /private/campaign/original-result-paths.json \ + /private/campaign/experiment-plan.json --dry-run +``` + +Inspect the plan. To import it, provide `BRAINTRUST_API_KEY` securely, remove +`--dry-run`, and use a separate `experiment-receipt.json` output path. The importer +requires explicit accounting identities and source/definition hashes; older +incomplete provenance fails closed. It retains original failed and successful +grades, cleanup outcomes, feedback counts, source revisions and trace identities. +Carried-forward CI attempts retain their original campaign IDs and must match +the report's retained receipts. Request costs attach only when the attempt's +campaign also matches the journal's campaign, even if attempt IDs coincide. +It does not upload prompts, feedback bodies, matcher details or raw errors. + +The evidence digest determines the private experiment name. Identical evidence +reuses that experiment and stable row IDs after an interrupted insert. Changed +evidence creates a different experiment and requires a separate receipt path. +The importer validates an existing receipt before retrying and never lowers its +confirmed accepted-row count. The receipt checkpoints accepted batches; +HTTP acceptance still requires independent indexing/UI verification. Retain failed +receipts. A retry must use the same original evidence, not newly run agents. +Set `PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT` to the receipt path when rebuilding the +campaign report to retain the experiment link. Its evidence hash must match the +report's selected source/definition identities, attempts, request-ledger checkpoint +and feedback review. Refreshing only +the report rendering preserves the link; changed evidence requires a new import. + +Request-ledger amounts remain metadata with their journal hash and classifications. +They are not additive cost metrics, and missing rows remain unknown rather than +zero. The campaign ledger may include setup probes outside the imported tests. +Do not add experiment metadata amounts to native or gateway trace costs. Native +trace rollups are not invoice reconciliation and can omit setup or duplicate logs. +API contracts: [create experiment](https://www.braintrust.dev/docs/api-reference/experiments/create-experiment) +and [insert retained events](https://www.braintrust.dev/docs/api-reference/experiments/insert-experiment-events). + +- `operations-preflight.json` preserves the exact manifest in the campaign folder. +- `ledger.jsonl` is the append-only authority for request admission and observations. +- `ledger.jsonl.requests/` contains private per-request checkpoints. +- `ledger.jsonl.summary.json` preserves known, reserved/held, conflicting, and + unknown amounts on normal finalization, including failed campaign workers. +- `campaign.json` and the dashboard preserve retained attempts and show missing + selected results after worker failure. Their request-ledger section shows the + shared ledger's known subtotal, classifications, additional held reservations, + unresolved requests and per-attempt totals. It can include earlier campaigns + sharing that budget. It is a separate view of model spending: do not add it to + heartbeat totals. The journal hash and record count identify the checkpoint. +- If the journal cannot be read or validated, the report records accounting as + unavailable with no numeric totals. Retained results and reports are still + written, and the launcher/report command exits unsuccessfully. Recover the + original journal before resuming paid work; reporting does not repair or reset it. +- To regenerate a merged report with the same ledger, pass + `PAPERCLIP_E2E_OPERATIONS_LEDGER=/absolute/ledger.jsonl` to `report.ts`. + Dashboard refresh and public publication preserve and validate its closed + numeric projection. Raw request receipts and the journal remain private. + +All model amounts from usage times a pricecard are **estimated**, not billed or +reconciled. This path does not yet price CI compute, storage, subscriptions, +Daytona, separate judges, or gateway overhead. Unknown observations and these +coverage gaps must be included in operational reporting. Provider billing can +lag and discounts can differ; a local list-price cap is not a provider-side cap. + +Resuming the same journal requires exactly the original limits. A dead process +can leave a lock and held reservations; inspect the owner and upstream receipts +before recovering it. Never delete a live writer's lock or reset a ledger merely +to regain budget. Older journals without an attempt cap remain readable for +reporting and reconciliation. Operations refuses an implicit upgrade because request receipts cannot reconstruct +zero-request attempts. Use the explicit audited migration below only after +reviewing all retained campaign attempts. Do not replace or reset an active pilot +ledger to evade this check. + +No recurring schedule is enabled by this integration. Qualification of actual +agent streams, tools, continuation, cancellation, native traces, and remote routes +is required before treating those routes as operationally proven. Forecasts, +provider reconciliation, and the campaign feedback disposition workflow are +separate remaining integration work. + +### Setup and agent request attribution + +Each admitted attempt creates a private phase checkpoint under +`.attempts/`. The launcher passes its path only to the test worker; +the Paperclip server does not inherit it. Before fixture setup finishes, +requests are recorded as `setup`, including connection checks and hello probes. +The worker atomically advances the checkpoint to `agent` immediately after +fixture creation and before task execution. Cleanup does not reset this phase, +so late agent requests retain their task attribution. + +The gateway reads and validates the checkpoint for each request before reserving +spend. That admission fixes the request's phase even if the phase changes while +its response streams. Missing, malformed, or foreign-attempt checkpoints stop +new dispatches rather than guessing a phase. Request records remain in the +fsynced journal after the worker exits. This boundary applies to the admitted +local core cells; it does not establish coverage for separate judges, paid tools, +child-agent attribution, or remote routes. Historical journals retain their +original attribution and are not retroactively rewritten. + +### Auditing a legacy journal before adding an attempt cap + +A journal created before durable attempt admission can acquire one attempt policy +through `ledger-attempt-policy.ts`. Stop its launcher and gateway first. Review +all retained campaigns, including startup failures with no model request. Save +the review and its source references as a private evidence file. Each prior +attempt needs its original execution and accounting attempt ID; request receipts +alone are insufficient. If that history is incomplete, do not resume paid work. + +Create a private JSON audit with these fields: + +- `version`: `1`. +- `ledgerPath` and `evidencePath`: absolute private file paths. +- `previousJournalHash`: the current journal's final record hash. +- `evidenceSha256`: SHA-256 of the reviewed evidence file. +- `maxAttemptsPerExecution`: the finite cap to adopt, including past attempts. +- `priorAttempts`: an array of `{ "executionId": "...", "attemptId": "..." }`. + +Run `node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/ledger-attempt-policy.ts /absolute/private-audit.json`. +The command checks evidence and journal hashes, takes the normal exclusive writer +lock, and appends one fsynced policy record. It refuses unfinished requests, +omitted recorded request identities, duplicate attempts, or an existing attempt +policy. The imported count can already exceed the cap; then no new attempt is +admitted for that execution. The command cannot prove the completeness of the +operator's review of zero-request attempts. Retain the evidence for inspection. + +All original journal bytes, request counts, charges, held reservations, campaign +identity, dollar limit, concurrency limit and deadline remain intact. Subsequent +launch manifests must use the adopted cap. The migration cannot extend an expired +deadline, change an existing cap, or replenish an allowance. An interrupted append +remains an accounting failure and must be recovered before more paid work. + +The explicit local `feedback-persistence` native Claude/Codex cells also accept +an operations manifest. They use the same metered connection, prices, campaign +journal, setup boundary, and durable attempt allowance as core compatibility. +Collection and the independent saved-evidence oracle require no additional +model call. The synthetic items need review as test data, not product incidents. diff --git a/tests/runner-e2e/OTLP-LICENSE.txt b/tests/runner-e2e/OTLP-LICENSE.txt new file mode 100644 index 00000000000..261eeb9e9f8 --- /dev/null +++ b/tests/runner-e2e/OTLP-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/tests/runner-e2e/OTLP-NOTICE.md b/tests/runner-e2e/OTLP-NOTICE.md new file mode 100644 index 00000000000..6ae059f624c --- /dev/null +++ b/tests/runner-e2e/OTLP-NOTICE.md @@ -0,0 +1,11 @@ +# OTLP field definitions + +The field definitions in `eval-otlp-schema.ts` are derived from +[OpenTelemetry Protocol v1.9.0](https://github.com/open-telemetry/opentelemetry-proto/tree/v1.9.0). + +Copyright The OpenTelemetry Authors. +Licensed under the [Apache License, Version 2.0](./OTLP-LICENSE.txt). + +The source comments and package declarations were removed to embed these field +definitions in the eval harness. The harness adds a combined response message. +Accounting behavior and limits are documented in [OPERATIONS.md](./OPERATIONS.md). diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index df516223c84..5f199a2fad4 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1054,9 +1054,32 @@ timings, token and cost accounting, and evidence links. The campaign header aggregates input, output, and cached tokens, provider-reported LLM spend, Daytona list-price runtime estimates, and pricing coverage. Missing provider usage is labeled `unavailable` or `unpriced`; it is never presented as zero -cost. The CI report job stages the same portable site at +cost. A known subtotal from a partially priced run stays in campaign spending, +with a partial-amount label and incomplete billing coverage. The CI report job stages the same portable site at `normalized/index.html` inside the access-controlled merged report artifact. +Campaign billing retains a separate numeric receipt for each attempt, including +failed retries. The selected result still controls the grade and result card. +Copies of the same attempt do not add another charge. The supervisor stamps a +campaign/attempt identity; CI cells share `PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID` +while retaining their separate artifact directories. Report regeneration keeps +these receipts and their totals. A later cleanup update replaces an earlier +observation of the same attempt; contradictory costs at the same timestamp are +rejected instead of silently choosing one. + +The report shows missing attempt receipts and artifacts excluded from another +campaign. Legacy CI evidence can be grouped by workflow run URL. Unscoped local +artifacts cannot establish campaign membership, so only the selected result is +used. Older normalized bundles without receipts are labeled `selected_results`; +rendering them again cannot recover lost retry costs. + +Billing reads decimal `costUsdExact` receipts and separates model estimates +from provider-reported amounts. Explicit zero usage differs from a missing or +unfinished receipt. Coverage is limited to captured usage. It does not certify +setup probes, subscription allocations, external tools, CI, storage, or provider +invoice reconciliation. These require separate receipts. Do not use the current +numeric report subtotals as an enforced spending limit. + The trusted publisher discovers display-only entries for selected execution IDs absent from its local catalog, so branch-only suites remain visible in the dashboard, filters, gallery, and summary image. It validates execution identity @@ -2748,3 +2771,173 @@ The arithmetic journey records four delivered agent handoffs and one completed terminal intent. The final case is `done` at handoff version 5, while the same four provider work runs and unchanged workflow document revision finish the task. The oracle rejects a missing or pending terminal intent. + +### Complaints and suggestions + +Each ordinary live attempt collects company-wide saved feedback after retiring +runs and before fixture teardown, even when cancellation or grading fails. This +includes feedback from child and background runs. Full paginated bodies and +native tool invocation outcomes are checkpointed in the access-controlled +`snapshots/agent-feedback.json` artifact. Collection has a 30-second deadline; +missing detail pages, truncated run lists, active work, and unavailable APIs +remain explicit partial/unavailable observations. Legacy shell helpers do not +have complete native invocation coverage. An empty saved-feedback list does not +prove that a tool was available, and a completed tool receipt does not by itself +prove a saved submission. + +Results and retained-attempt receipts contain only counts, closed statuses, and +body fingerprints. Campaign reports group identical fingerprints across attempts +into distinct items awaiting review. Feedback presence does not change product +grades. Inspect full bodies in private evidence and record a disposition before +considering the review complete; collection itself performs no model call or +semantic triage. Historical artifacts without feedback fields are labeled as +uncaptured. A killed worker that never reaches finalization is also uncaptured, +not evidence of zero feedback. + +Create a private review template from the merged `normalized-results.json`: + +```sh +pnpm exec tsx tests/runner-e2e/feedback-dispositions.ts /private/report/normalized-results.json /private/feedback-review.json +``` + +Inspect the full bodies in the retained attempt snapshots. Set `reviewedAt` to +an ISO timestamp with an explicit timezone, such as `2026-10-20T01:00:00Z`, +and `reviewedBy` to `human` or `agent`. Timestamp comments and free-form dates +are rejected so private notes cannot enter public report metadata. Assign each fingerprint +one disposition: `actionable`, `duplicate`, `already_fixed`, `unsupported`, or +`needs_investigation`. A duplicate must name a non-duplicate fingerprint in the +same campaign via `duplicateOf`. Optional `issueNumber` links a public +`paperclipai/paperclip` issue; keep private rationale outside this file. + +Import a completed review into an existing bundle without rerunning agents: + +```sh +pnpm test:e2e:runner:dashboard -- /private/report --feedback-review /private/feedback-review.json +``` + +Alternatively, rebuild the same campaign and selection with +`PAPERCLIP_E2E_FEEDBACK_REVIEW=/private/feedback-review.json`. Keep its original +campaign ID. The importer requires exactly one decision per collected group +and rejects a review if retained attempts or feedback coverage changed. Blank +templates cannot count as reviewed. Dashboard regeneration and publication +preserve the review. Regeneration refreshes the review status in an existing +`summary.md` while preserving its other measurements and links. An empty review +can record that no saved items were found; it never closes collection gaps, +failed submissions, or changes product grades. +This workflow makes no additional model call. Review performed by a separate +paid model must be budgeted and recorded as `feedback_review` work separately. + +### Cost-ledger integration contract + +`cost-ledger.ts` provides the request-level journal used by the eval operations +integration. It is a library; the existing launcher is not yet universally routed +through it, and merely setting a campaign ID does not enforce a spending cap. +Every paid dispatch must await a durable reservation before contacting a provider. +The caller supplies an explicit campaign ceiling, deadline, request count, and +concurrency. Monetary decisions use integer nanodollars. Model list prices remain +estimates with a pinned version; they do not become provider invoices. + +The journal retains setup/agent/judge/tool/infrastructure attribution separately +from heartbeat IDs. Multiple sources for the same request are retained but only +one cost contributes to the total: reconciled, reported, estimated, then allocated. +Missing receipts and cancellations retain their reservation. Contradictory final +receipts stop new admissions. An observed overshoot is retained in full and stops +further work; local estimated limits are not provider-enforced dollar limits. +A request bound must account for input/context, output limits, cache-write pricing, +and any paid tools before claiming a bounded overshoot. + +A single writer serializes admissions and fsyncs each journal record. Read-only +inspection survives process loss; truncated/tampered records fail closed. A stale +`.lock` requires inspection of the recorded owner PID before operator recovery; +never remove a live writer's lock. Resuming a campaign cannot silently change its +limits. The ledger rejects unreviewed fields and contains no prompts, responses, +headers, or credentials. Cross-source request joins must use a verified request +identity; ambiguous provider-request joins are rejected rather than double counted. + +### Opt-in Braintrust traces + +Set `PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID` to the destination project UUID and +provide `BRAINTRUST_API_KEY` securely to the launcher. An ambient key alone does +not enable export. The launcher includes this key in secret scans and removes +Braintrust variables from the server's inherited environment. Fixture setup +saves the company's native observability configuration through the public, +revision-fenced API using a non-Playwright sensitive request, before credential, +agent, and setup-probe creation. The onboarding flow installs it immediately +after company creation. The existing company setting controls native capture. + +Each ordinary `runner.spec.ts` attempt records orchestration spans for setup, +execution, feedback, and cleanup, and exports protobuf over HTTPS with a ten +second deadline and no redirect/retry. `snapshots/braintrust-trace.json` checkpoints +the identifiers/configuration and final export receipt. Normalized results retain +the receipt, including failed exports. Export failure does not rewrite the product +verdict; operational qualification must separately require accepted and verified +traces. A killed worker can leave a pending receipt, not a claimed successful export. + +`accepted` means the OTLP receiver accepted the batch; it does not establish that +Braintrust indexed it or that native agent spans arrived. Search the project for +`paperclip.eval.attempt_id` and its trace ID, then verify native records using the +company ID and `paperclip.eval.run_ids`. The disposable company uses the custom +OTLP destination at `https://api.braintrust.dev/otel` with a private `x-bt-parent` +header containing the exported attempt parent. Braintrust attaches independent +native trace roots beneath that attempt; native children keep their own hierarchy. +This uses the existing company observability API without changing production +trace parentage or enabling export on other companies. + +The parent encoder uses Braintrust's V4 project-log wire format with JSON identity +fields (version 4, project-log type 2, zero compressed fields). Its fixture was +round-tripped through the public `SpanComponentsV4.fromStr` export in +`braintrust/util` 3.38.0. The implementation is specified in the +[official SDK source](https://github.com/braintrustdata/braintrust-sdk-javascript/blob/403ae94fb3f1916a101d8d84de4c4871e4ae3bec/js/util/span_identifier_v4.ts). +A model-free live probe verified the indexed child's `root_span_id` and +`span_parents` in Braintrust; a native-agent run must separately qualify the path. +The parent contains identifiers only and is installed through the sensitive +credential-header path, outside browser evidence. + +Attempt spans include suite ID and definition hash, profile, case, configured +provider/model, source revision (or explicit `unknown`), and the runner-spec grader +path. The source revision identifies that grader's implementation. Worker exports +are marked `paperclip.eval.outcome_stage=worker`. After evidence packaging, +gateway shutdown and outer cleanup, the supervisor updates the same attempt root +with its final outcome and adds a finalization span. This preserves failures first +discovered after the worker returned. The result retains a separate finalization +export receipt; old or killed attempts can explicitly lack one. Export failure +remains separate from the product grade. Orchestration spans omit model usage and +cost metrics. Gateway model logging remains a separate observation lane; +its model costs must not be added to native costs without a verified request join. +The separate provider connection suite and standalone judges do not yet use this +attempt exporter. + +### Bounded local gateway pilot + +[OPERATIONS.md](./OPERATIONS.md) documents the opt-in operations manifest, exact +selection admission, durable request reservations, local Responses/Messages proxy, +and Braintrust routing. Current support is deliberately limited to local core +Codex/Claude cells. The document distinguishes implemented accounting controls +from live qualification and lists remaining billing/reporting coverage gaps. + +## Feedback persistence (explicit only) + +`--suite feedback-persistence` selects two local native Claude/Codex cells. +Each creates one standard task through the browser, asks for two distinct +synthetic feedback items, and requires the real `submit_complaint` and +`submit_suggestion` tools. The five-minute case expects one task work run and +disables automatic retries. Company and agent accounted-work budgets are $10. +Setup probes and any background work must also pass through the separately +budgeted gateway when using an operations manifest. Local compute is unmetered. +Cost and duration forecasts remain unmeasured until a retained live attempt. +The runner E2E maintainers own this manual case; `--all` excludes it. + +After collection and fixture teardown, the oracle reads the retained evidence +file and requires exact nonce-bearing bodies, the original company/agent/task +and observed run, plus one completed native receipt for each tool. Missing or +partial collection, prose-only claims, foreign attribution, duplicate records, +and failed submissions cannot pass. Full bodies stay in private evidence; +public reports retain counts, fingerprints, and closed oracle checks. The usual +result screenshot, trace, secret scanning and supervisor cleanup still apply. +This case proves explicit submission and persistence, not spontaneous feedback +quality or survival after an abrupt worker kill. + +```sh +pnpm test:e2e:runner -- --list --suite feedback-persistence +pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1 +``` diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md index 093f2066130..2a84a315e51 100644 --- a/tests/runner-e2e/SECURITY.md +++ b/tests/runner-e2e/SECURITY.md @@ -423,6 +423,66 @@ Two passive seed cases support browser list assertions; screenshots remain on the exact primary case/task routes and document dialogs preserve that case URL. The public screenshot allowlist is unchanged. +## Eval feedback capture + +`snapshots/agent-feedback.json` retains company-wide complaint/suggestion bodies +and attributed invocation outcomes in the access-controlled Actions artifact. +It passes the existing credential scanner and is excluded from public history +like other snapshots. Normalized results and campaign attempt receipts contain +only numeric counts, closed collection statuses, and SHA-256 fingerprints for +review grouping. Public reports never copy feedback bodies, previews, tool +arguments, or free-form collection errors. Collecting feedback performs no new +model call and does not forward it to an external service. + +Feedback capture retains at most 1,000 full entries and 8 MiB of serialized +entries/invocations per attempt. Event inspection stops at 10,000 events or +16 MiB of serialized event data. Limits produce partial collection and a closed +`evidence_limit_reached` diagnostic; omitted evidence is never reported as absent. +Checkpoints are considered every ten details, at page boundaries, and at stage +completion; geometric data growth bounds repeated writes. Reads stop before the +overall deadline, reserving up to five seconds (20% of shorter windows) to persist +the collected prefix. Persistence shares the overall collection deadline and +receives an abort signal. The fixture writes an abortable temporary +snapshot before replacing the last complete checkpoint, so a stalled write does +not overwrite earlier evidence with a partially written JSON file. +Review counts and fingerprints advance only after a checkpoint succeeds. A later +write failure or expired deadline retains the last saved bodies and marks coverage +incomplete. + +Braintrust export requires an explicit eval project UUID. Native capture includes +prompts, responses, and tool content only after that opt-in configures the isolated +company's observability destination. The ingestion credential crosses the public API via +Node fetch, outside Playwright tracing, and is included in launcher evidence +redaction. Orchestration spans contain identifiers, phase timing, outcome, and +feedback counts; they never contain provider credentials or feedback bodies. + +With a bounded operations manifest, the ingestion credential is an expiring +per-attempt relay token. The organization-wide Braintrust key stays in the +controller. Only authenticated POSTs to the relay's trace/log ingest paths are +allowed, with a fixed HTTPS upstream and redirects disabled. Header and payload +`braintrust.parent` values must reference the selected project. Duplicate protobuf +singular/oneof fields are refused, and JSON is canonicalized to remove duplicate +key ambiguity. The token cannot call model, read, delete, or other API endpoints; +it can still write records within the selected project until its finite request, +byte or time allowance is exhausted. It is not an attempt-level row permission. + +The worker environment removes upstream provider/Braintrust credentials before +inserting the model and OTLP relay tokens. Ambient relay URLs are discarded, and +the fixture server receives the selected endpoint through company settings. +Both tokens participate in evidence redaction. Private OTLP receipts retain +metadata only, and persistence errors stop further admission. The relay closes +with the model gateway before controller finalization. The controller's direct +finalization export and tracing without an operations manifest retain the +existing direct credential path. Remote relay publication is not enabled here. + +The bounded operations manifest is credential-free and private. Its parent-owned +loopback gateway uses an ephemeral per-attempt token. Upstream credentials stay +in the launcher, are included in secret scans, and are removed from the child +provider environment. Only the selected Responses or Messages endpoint is +forwarded to a fixed provider or Braintrust HTTPS origin; request headers do not +select arbitrary destinations. Gateway response caching and fallback are refused. +Request journal and usage receipts contain no prompt/response bodies or keys. +The manifest, journal, and receipts are not public report projections. ## Configured repository and conversion proof diff --git a/tests/runner-e2e/api.ts b/tests/runner-e2e/api.ts index 398691d1e27..5032c3e56f9 100644 --- a/tests/runner-e2e/api.ts +++ b/tests/runner-e2e/api.ts @@ -58,15 +58,27 @@ export class RunnerApi { * never serialized into trace/blob evidence before Paperclip encrypts it. */ async postSensitive(path: string, data: unknown): Promise { - const response = await fetch(new URL(path, this.baseURL), { - method: "POST", - headers: { "content-type": "application/json", ...this.sessionOptions().headers }, - body: JSON.stringify(data), - }); + return this.sensitive("POST", path, data); + } + + async putSensitive(path: string, data: unknown): Promise { + return this.sensitive("PUT", path, data); + } + + private async sensitive(method: "POST" | "PUT", path: string, data: unknown): Promise { + const url = new URL(path, this.baseURL); + if (url.origin !== this.baseURL || !url.pathname.startsWith("/api/")) throw new Error("Sensitive request must use the fixture API"); + let response: Response; + try { + response = await fetch(url, { + method, redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { "content-type": "application/json", ...this.sessionOptions().headers }, + body: JSON.stringify(data), + }); + } catch { throw new Error(`Sensitive ${method} request failed; details withheld`); } if (!response.ok) { - throw new Error( - `Sensitive POST ${path} returned ${response.status}; response body withheld`, - ); + await response.body?.cancel(); + throw new Error(`Sensitive ${method} ${url.pathname} returned ${response.status}; response body withheld`); } return response.json() as Promise; } diff --git a/tests/runner-e2e/artifact-budget.mjs b/tests/runner-e2e/artifact-budget.mjs new file mode 100644 index 00000000000..cf84bbcb8ad --- /dev/null +++ b/tests/runner-e2e/artifact-budget.mjs @@ -0,0 +1,163 @@ +import { constants } from "node:fs"; +import { createHash } from "node:crypto"; +import { lstat, mkdir, open, opendir, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +export const ARTIFACT_BOUNDS = Object.freeze({ + lock: 16 * 1024 * 1024, + build: 1024 * 1024 * 1024, + evidence: 256 * 1024 * 1024, + report: 256 * 1024 * 1024, + receipt: 8 * 1024 * 1024, + entries: 4096, +}); + +export async function checkArtifactFiles(kind, files) { + if (!["lock", "build"].includes(kind) || !files.length) throw new Error("Invalid artifact size check"); + let bytes = 0; + for (const file of files) { + const info = await lstat(file); + if (!info.isFile() || info.isSymbolicLink() || info.size === 0) throw new Error("Invalid artifact input file"); + bytes += info.size; + if (!Number.isSafeInteger(bytes) || bytes > ARTIFACT_BOUNDS[kind]) throw new Error(`${kind} artifact exceeds its source-byte allowance`); + } + return { kind, files: files.length, sourceBytes: bytes, limitBytes: ARTIFACT_BOUNDS[kind] }; +} + +function priority(relative) { + if (relative === "scheduled" || relative.startsWith("scheduled/")) return 0; + if (/\.(json|jsonl)$/.test(relative)) return 1; + if (/\.(md|html|xml|txt|log)$/.test(relative)) return 2; + return 3; +} + +export async function stageBoundedArtifact(kind, source, destination, bounds = ARTIFACT_BOUNDS) { + if (!["evidence", "report"].includes(kind) || !Number.isSafeInteger(bounds[kind]) + || !Number.isSafeInteger(bounds.receipt) || bounds[kind] <= bounds.receipt + || !Number.isSafeInteger(bounds.entries) || bounds.entries < 1 || bounds.entries > ARTIFACT_BOUNDS.entries) { + throw new Error("Invalid artifact staging allowance"); + } + const root = path.resolve(source), out = path.resolve(destination); + if (out === root || out.startsWith(`${root}${path.sep}`) || root.startsWith(`${out}${path.sep}`)) { + throw new Error("Artifact staging paths must be separate"); + } + await mkdir(out, { mode: 0o700 }); + const receipt = { version: 1, kind, sourceComplete: true, inventoryComplete: true, + scope: "Artifact evidence; regular playwright-output/.last-run.json bookkeeping is excluded", + sourceBytes: 0, retainedBytes: 0, limitBytes: bounds[kind], entryLimit: bounds.entries, + findings: [], files: [] }; + const candidates = []; + let visited = 0; + const finding = code => { receipt.sourceComplete = false; if (!receipt.findings.includes(code)) receipt.findings.push(code); }; + const skipped = (relative, reason, subtree = false) => { + const shortened = relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024; + receipt.files.push({ path: shortened ? `${relative.slice(0, 160)}…${relative.slice(-80)}` : relative, + ...(shortened ? { pathTruncated: true, pathSha256: createHash("sha256").update(relative).digest("hex") } : {}), + ...(subtree ? { scope: "subtree" } : {}), retained: false, reason }); + }; + const scan = async () => { + const pending = [{ absolute: root, relative: ".", depth: 0, order: 0 }]; + while (pending.length) { + pending.sort((a, b) => a.order - b.order || a.depth - b.depth || a.relative.localeCompare(b.relative)); + const directory = pending.shift(); + if (visited >= bounds.entries || directory.depth > 24) { + const reason = visited >= bounds.entries ? "entry_limit" : "directory_depth_limit"; + receipt.inventoryComplete = false; finding(reason); skipped(directory.relative, reason, true); continue; + } + try { + const handle = await opendir(directory.absolute); + for await (const entry of handle) { + if (visited >= bounds.entries) { + receipt.inventoryComplete = false; finding("entry_limit"); skipped(directory.relative, "entry_limit", true); break; + } + visited++; + const absolute = path.join(directory.absolute, entry.name), relative = path.relative(root, absolute).split(path.sep).join("/"); + if (relative.length > 512 || Buffer.byteLength(JSON.stringify(relative)) > 1024) { + receipt.inventoryComplete = false; finding("path_length_limit"); skipped(relative, "path_length_limit", entry.isDirectory()); continue; + } + if (entry.isFile() && /(?:^|\/)playwright-output\/\.last-run\.json$/.test(relative)) { + skipped(relative, "playwright_bookkeeping_excluded"); continue; + } + if (entry.name.startsWith(".")) { + receipt.inventoryComplete = false; finding("hidden_path_excluded"); skipped(relative, "hidden_path_excluded", entry.isDirectory()); continue; + } + if (entry.isDirectory()) { + const diagnostic = /(?:^|\/)(?:blob-report|html-report|playwright-output)(?:\/|$)/.test(relative); + pending.push({ absolute, relative, depth: directory.depth + 1, + order: diagnostic ? 3 : relative === "scheduled" || relative.startsWith("scheduled/") ? 0 : 1 }); + } else if (entry.isFile()) candidates.push({ absolute, relative }); + else { finding("non_regular_file"); skipped(relative, "non_regular_file"); } + } + } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(directory.relative, "inventory_failed", true); } + } + }; + try { + const metadata = await lstat(root); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw new Error("Invalid evidence root"); + await scan(); + } catch { receipt.inventoryComplete = false; finding("inventory_failed"); skipped(".", "inventory_failed", true); } + const canonicalRoot = await realpath(root).catch(() => root); + candidates.sort((a, b) => priority(a.relative) - priority(b.relative) || a.relative.localeCompare(b.relative)); + for (const { absolute, relative } of candidates) { + let input, output; + const target = path.join(out, relative); + const row = { path: relative, retained: false }; + receipt.files.push(row); + try { + if (relative === "artifact-budget-receipt.json") throw new Error("reserved_path"); + const canonical = await realpath(absolute); + if (!canonical.startsWith(`${canonicalRoot}${path.sep}`)) throw new Error("source_escaped"); + input = await open(absolute, constants.O_RDONLY | constants.O_NOFOLLOW); + const before = await input.stat(); + if (!before.isFile()) throw new Error("non_regular_file"); + row.sourceBytes = before.size; + receipt.sourceBytes += before.size; + if (!Number.isSafeInteger(receipt.sourceBytes)) throw new Error("source_size_overflow"); + if (receipt.retainedBytes + before.size > bounds[kind] - bounds.receipt) throw new Error("byte_limit"); + await mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + output = await open(target, "wx", 0o600); + const buffer = Buffer.alloc(64 * 1024), hash = createHash("sha256"); + let copied = 0; + while (copied < before.size) { + const read = await input.read(buffer, 0, Math.min(buffer.length, before.size - copied), copied); + if (!read.bytesRead) throw new Error("source_changed"); + hash.update(buffer.subarray(0, read.bytesRead)); + let written = 0; + while (written < read.bytesRead) { + const part = await output.write(buffer, written, read.bytesRead - written, copied + written); + if (!part.bytesWritten) throw new Error("copy_failed"); + written += part.bytesWritten; + } + copied += read.bytesRead; + } + const after = await input.stat(); + if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("source_changed"); + await output.sync(); + row.retained = true; row.sha256 = hash.digest("hex"); + receipt.retainedBytes += copied; + } catch (error) { + const code = ["reserved_path", "source_escaped", "non_regular_file", "source_size_overflow", "byte_limit", "source_changed"].includes(error.message) ? error.message : "copy_failed"; + row.reason = code; finding(code); + } finally { + await input?.close(); await output?.close(); + if (!row.retained) await rm(target, { force: true }); + } + } + const serialized = `${JSON.stringify(receipt, null, 2)}\n`; + if (Buffer.byteLength(serialized) > bounds.receipt) throw new Error("Artifact receipt exceeds reserved allowance"); + await writeFile(path.join(out, "artifact-budget-receipt.json"), serialized, { mode: 0o600, flag: "wx" }); + return { ...receipt, stagedBytes: receipt.retainedBytes + Buffer.byteLength(serialized) }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, kind, ...args] = process.argv.slice(2); + if (command === "check") console.log(JSON.stringify(await checkArtifactFiles(kind, args))); + else if (command === "stage" && args.length === 2) { + const receipt = await stageBoundedArtifact(kind, args[0], args[1]); + if (process.env.GITHUB_OUTPUT) await writeFile(process.env.GITHUB_OUTPUT, `complete=${receipt.sourceComplete}\n`, { flag: "a" }); + const summary = `Eval ${kind}: retained ${receipt.retainedBytes} source bytes; staged ${receipt.stagedBytes} bytes; complete=${receipt.sourceComplete}. Findings: ${receipt.findings.join(", ") || "none"}.`; + if (process.env.GITHUB_STEP_SUMMARY) await writeFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`, { flag: "a" }); + console.log(summary); + } else throw new Error("Usage: artifact-budget.mjs check | stage "); +} diff --git a/tests/runner-e2e/artifact-budget.test.mjs b/tests/runner-e2e/artifact-budget.test.mjs new file mode 100644 index 00000000000..06493ca2a64 --- /dev/null +++ b/tests/runner-e2e/artifact-budget.test.mjs @@ -0,0 +1,120 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, rm, stat, symlink, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { ARTIFACT_BOUNDS, checkArtifactFiles, stageBoundedArtifact } from "./artifact-budget.mjs"; + +const bounds = { ...ARTIFACT_BOUNDS, evidence: 12_288, receipt: 8192, entries: 32 }; +async function fixture(run) { + const root = await mkdtemp(path.join(os.tmpdir(), "eval-artifacts-")); + try { const source = path.join(root, "source"); await mkdir(source); await run({ root, source, out: path.join(root, "out") }); } + finally { await rm(root, { recursive: true, force: true }); } +} + +test("retains exact accounting and feedback before oversized diagnostics and keeps the source intact", async () => fixture(async ({ source, out }) => { + await mkdir(path.join(source, "scheduled")); + await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"reserved":"4.16"}\n'); + await writeFile(path.join(source, "feedback.json"), '{"complaint":"retained"}\n'); + await writeFile(path.join(source, "trace.zip"), Buffer.alloc(5000, 1)); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, true); + assert.deepEqual(result.findings, ["byte_limit"]); + assert.equal(await readFile(path.join(out, "scheduled/ledger.jsonl"), "utf8"), '{"reserved":"4.16"}\n'); + assert.equal(await readFile(path.join(out, "feedback.json"), "utf8"), '{"complaint":"retained"}\n'); + assert.equal((await stat(path.join(source, "trace.zip"))).size, 5000); + await assert.rejects(stat(path.join(out, "trace.zip")), { code: "ENOENT" }); + const receipt = JSON.parse(await readFile(path.join(out, "artifact-budget-receipt.json"), "utf8")); + assert.equal(receipt.files.find(row => row.path === "trace.zip").reason, "byte_limit"); + assert.match(receipt.files.find(row => row.path === "feedback.json").sha256, /^[a-f0-9]{64}$/); + assert.ok(result.stagedBytes <= bounds.evidence); +})); + +test("keeps complete evidence unchanged at the exact payload boundary and refuses destination reuse", async () => fixture(async ({ source, out }) => { + const content = Buffer.alloc(bounds.evidence - bounds.receipt, 42); + await writeFile(path.join(source, "result.json"), content); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + assert.equal(result.sourceComplete, true); assert.equal(result.retainedBytes, content.length); + assert.deepEqual(await readFile(path.join(out, "result.json")), content); + await assert.rejects(stageBoundedArtifact("evidence", source, out, bounds), { code: "EEXIST" }); +})); + +test("refuses symlinks and records missing evidence without reading outside the source", async () => fixture(async ({ root, source, out }) => { + await writeFile(path.join(root, "private"), "secret outside source"); + await symlink(path.join(root, "private"), path.join(source, "result.json")); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + assert.equal(result.sourceComplete, false); assert.deepEqual(result.findings, ["non_regular_file"]); + assert.equal(result.retainedBytes, 0); + const missing = await stageBoundedArtifact("evidence", path.join(root, "missing"), path.join(root, "missing-out"), bounds); + assert.equal(missing.inventoryComplete, false); assert.deepEqual(missing.findings, ["inventory_failed"]); +})); + +test("bounds entry inventory and exposes partial traversal", async () => fixture(async ({ source, out }) => { + for (let i = 0; i < 5; i++) await writeFile(path.join(source, `${i}.json`), "{}"); + const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 3 }); + assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false); + assert.equal(result.files.filter(row => row.retained).length, 3); assert.deepEqual(result.findings, ["entry_limit"]); + assert.deepEqual(result.files.find(row => row.reason === "entry_limit"), { path: ".", scope: "subtree", retained: false, reason: "entry_limit" }); +})); + +test("keeps hidden files excluded from upload and exposes that incomplete scope", async () => fixture(async ({ source, out }) => { + await mkdir(path.join(source, ".private")); + await writeFile(path.join(source, ".private/token"), "fixture"); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + assert.equal(result.inventoryComplete, false); assert.equal(result.sourceComplete, false); + assert.deepEqual(result.findings, ["hidden_path_excluded"]); + assert.deepEqual(result.files, [{ path: ".private", scope: "subtree", retained: false, reason: "hidden_path_excluded" }]); + await assert.rejects(stat(path.join(out, ".private")), { code: "ENOENT" }); +})); + +test("excludes ordinary packaged Playwright bookkeeping without failing passing evidence", async () => fixture(async ({ source, out }) => { + const folder = "core-compatibility/runner-codex/local/message-marker/attempt-1"; + await mkdir(path.join(source, folder, "playwright-output"), { recursive: true }); + await writeFile(path.join(source, folder, "playwright-output/.last-run.json"), '{"status":"passed","failedTests":[]}'); + await writeFile(path.join(source, folder, "result.json"), '{"status":"passed"}'); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + assert.equal(result.sourceComplete, true); assert.equal(result.inventoryComplete, true); + assert.deepEqual(result.findings, []); + assert.equal(result.files.find(row => row.path.endsWith(".last-run.json")).reason, "playwright_bookkeeping_excluded"); + assert.equal(await readFile(path.join(out, folder, "result.json"), "utf8"), '{"status":"passed"}'); +})); + +test("discovers snapshots in all attempts before archive directories can consume the entry limit", async () => fixture(async ({ source, out }) => { + for (const attempt of ["attempt-a", "attempt-b"]) { + await mkdir(path.join(source, attempt, "blob-report"), { recursive: true }); + await mkdir(path.join(source, attempt, "snapshots")); + await writeFile(path.join(source, attempt, "snapshots/feedback.json"), '{"saved":true}'); + for (let i = 0; i < 20; i++) await writeFile(path.join(source, attempt, `blob-report/${i}.zip`), "archive"); + } + const result = await stageBoundedArtifact("evidence", source, out, { ...bounds, entries: 12 }); + assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false); + for (const attempt of ["attempt-a", "attempt-b"]) { + assert.equal(await readFile(path.join(out, attempt, "snapshots/feedback.json"), "utf8"), '{"saved":true}'); + assert.ok(result.files.some(row => row.path === `${attempt}/blob-report` && row.scope === "subtree" && row.reason === "entry_limit")); + } +})); + +test("identifies an overlong skipped subtree with a bounded preview and full path digest", async () => fixture(async ({ source, out }) => { + const segments = Array.from({ length: 6 }, (_, index) => `${index}-${"x".repeat(90)}`); + await mkdir(path.join(source, ...segments), { recursive: true }); + await writeFile(path.join(source, ...segments, "feedback.json"), "{}"); + const result = await stageBoundedArtifact("evidence", source, out, bounds); + const omitted = result.files.find(row => row.reason === "path_length_limit"); + assert.equal(result.sourceComplete, false); assert.equal(result.inventoryComplete, false); + assert.equal(omitted.scope, "subtree"); assert.equal(omitted.pathTruncated, true); + assert.match(omitted.pathSha256, /^[a-f0-9]{64}$/); assert.ok(omitted.path.startsWith(segments[0])); + assert.ok(omitted.path.endsWith(segments[5].slice(-80))); + assert.equal(omitted.pathSha256, createHash("sha256").update(segments.join("/")).digest("hex")); + assert.ok(result.stagedBytes <= bounds.evidence); +})); + +test("checks immutable build and lock source totals before upload without reading their contents", async () => fixture(async ({ source }) => { + const a = path.join(source, "a"), b = path.join(source, "b"); + await writeFile(a, "a"); await writeFile(b, "b"); + await truncate(a, ARTIFACT_BOUNDS.lock - 1); + assert.equal((await checkArtifactFiles("lock", [a, b])).sourceBytes, ARTIFACT_BOUNDS.lock); + await truncate(a, ARTIFACT_BOUNDS.lock); + await assert.rejects(checkArtifactFiles("lock", [a, b]), /exceeds/); + await assert.rejects(checkArtifactFiles("other", [a]), /Invalid/); +})); diff --git a/tests/runner-e2e/attempt-accounting.test.ts b/tests/runner-e2e/attempt-accounting.test.ts new file mode 100644 index 00000000000..e7335879eef --- /dev/null +++ b/tests/runner-e2e/attempt-accounting.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it } from "vitest"; +import { buildCampaignAccounting, isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js"; +import { buildRunnerCampaign, campaignHistoryRecord } from "./history.js"; +import type { RunnerE2EResult } from "./types.js"; + +function result(attempt: number, cost: number, overrides: Partial = {}): RunnerE2EResult { + return { + schema: "paperclip.runner-e2e.result/v2", + accountingIdentity: { campaignId: "campaign", attemptId: `attempt-${attempt}` }, + executionId: "core-compatibility.runner-codex.local.message-marker", suiteId: "core-compatibility", + attempt, status: attempt === 1 ? "failed" : "passed", profileId: "runner-codex", + environmentId: "local", caseId: "message-marker", provider: "codex", model: "fixture", runtimeMode: "native", + startedAt: `2026-10-20T00:0${attempt}:00Z`, finishedAt: `2026-10-20T00:0${attempt}:01Z`, durationMs: 1000, + runIds: [`run-${attempt}`], usage: { inputTokens: 100, outputTokens: 10, costUsd: cost }, cleanup: "passed", + ...overrides, + }; +} + +describe("retained attempt accounting", () => { + it("adds failed and successful attempts exactly once without changing the chosen verdict", () => { + const failed = result(1, 0.01); + const passed = result(2, 0.02); + const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: passed.finishedAt, + expected: [passed.executionId], results: [passed], attempts: [failed, passed, structuredClone(passed)] }); + expect(campaign).toMatchObject({ passed: 1, failed: 0, selected: 1, + accounting: { scope: "retained_attempts", missingAttempts: 0, attempts: [{ attempt: 1 }, { attempt: 2 }] }, + billing: { testCount: 1, testsWithCompleteBilling: 1, reportedLlmCostUsd: 0.03, + llm: { runCount: 2, inputTokens: 200, outputTokens: 20 } } }); + expect(campaign.results[0]!.billing!.reportedCostUsd).toBe(0.02); + expect(campaign.suites[0]!.billing.reportedLlmCostUsd).toBe(0.03); + expect(campaignHistoryRecord(campaign, "https://example.test").billing.reportedLlmCostUsd).toBe(0.03); + }); + + it("retains known costs from interrupted attempts and marks missing attempts", () => { + const interrupted = result(2, 0.02, { status: "failed", cleanup: "failed" }); + const campaign = buildRunnerCampaign({ campaignId: "campaign", generatedAt: interrupted.finishedAt, + expected: [interrupted.executionId], results: [interrupted], attempts: [interrupted] }); + expect(campaign.accounting!.missingAttempts).toBe(1); + expect(campaign.billing.reportedLlmCostUsd).toBe(0.02); + expect(campaign.billing.testsWithCompleteBilling).toBe(0); + const missing = result(0, 0, { runIds: [], usage: null }); + expect(buildCampaignAccounting({ results: [missing], attempts: [] })).toMatchObject({ missingAttempts: 1, attempts: [] }); + }); + + it("labels legacy selected-only bundles without inventing receipts for retries", () => { + const selected = result(2, 0.02, { accountingIdentity: undefined }); + expect(buildCampaignAccounting({ results: [selected] })).toMatchObject({ + scope: "selected_results", missingAttempts: 1, attempts: [{ attempt: 2 }], + }); + }); + + it("excludes other campaigns and ambiguous local artifacts", () => { + const selected = result(2, 0.02); + const other = result(1, 0.9, { accountingIdentity: { campaignId: "other", attemptId: "attempt-1" } }); + const first = result(1, 0.01); + expect(retainedCampaignAttempts([selected], [first, selected, other])).toEqual({ attempts: [first, selected], excludedArtifacts: 1 }); + const legacy = { ...selected, accountingIdentity: undefined }; + expect(retainedCampaignAttempts([legacy], [legacy, { ...legacy }])).toEqual({ attempts: [legacy], excludedArtifacts: 1 }); + }); + + it("uses the final supervisor receipt but rejects contradictory same-version costs", () => { + const selected = result(1, 0.02); + const early = { ...selected, finishedAt: selected.startedAt, usage: { inputTokens: 20, costUsd: 0.01 } }; + expect(buildCampaignAccounting({ results: [selected], attempts: [selected, early] }).attempts).toMatchObject([ + { billing: { reportedCostUsd: 0.02 } }, + ]); + expect(() => buildCampaignAccounting({ results: [selected], attempts: [selected, + { ...selected, usage: { inputTokens: 20, costUsd: 0.1 } }] })).toThrow("Conflicting accounting receipts"); + }); +}); + +it("requires exact selector provenance before admitting a carried workflow result", () => { + const retained = result(1, 0.01, { accountingIdentity: { campaignId: "gha-123-1", attemptId: "a" }, + source: { sha: "source", ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/123" } }); + const artifactName = `runner-e2e-123-1-${retained.executionId}`; + const input = { requestedCampaignId: "gha-123-2", result: retained, + relativeResultPath: `${artifactName}/gha-123-1-${retained.executionId}/attempt-1/result.json`, + selection: { version: 1, workflowRunId: "123", workflowRunAttempt: 2, sourceSha: retained.source!.sha, + sourceRef: retained.source!.ref, workflowRunUrl: retained.source!.workflowRunUrl, + selections: [{ executionId: retained.executionId, workflowAttempt: 1, artifactName }] } }; + expect(isSelectedCarryForward(input)).toBe(true); + expect(isSelectedCarryForward({ ...input, selection: undefined })).toBe(false); + expect(isSelectedCarryForward({ ...input, requestedCampaignId: "gha-999-2" })).toBe(false); + expect(isSelectedCarryForward({ ...input, relativeResultPath: `unselected/${input.relativeResultPath}` })).toBe(false); + expect(isSelectedCarryForward({ ...input, result: { ...retained, source: { ...retained.source!, sha: "other" } } })).toBe(false); + expect(isSelectedCarryForward({ ...input, result: { ...retained, accountingIdentity: { campaignId: "gha-123-3", attemptId: "a" } } })).toBe(false); +}); diff --git a/tests/runner-e2e/attempt-accounting.ts b/tests/runner-e2e/attempt-accounting.ts new file mode 100644 index 00000000000..3fea6442d7f --- /dev/null +++ b/tests/runner-e2e/attempt-accounting.ts @@ -0,0 +1,99 @@ +import { createHash } from "node:crypto"; +import { summarizeExecutionBilling } from "./billing.js"; +import type { RunnerE2EAttemptReceipt, RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; + +export function isSelectedCarryForward(input: { + requestedCampaignId: string; result: RunnerE2EResult; relativeResultPath: string; selection: unknown; +}): boolean { + const s = input.selection as Record | null; + if (!s || s.version !== 1 || typeof s.workflowRunId !== "string" || !/^[0-9]+$/.test(s.workflowRunId) + || !Number.isSafeInteger(s.workflowRunAttempt) || Number(s.workflowRunAttempt) < 1 + || input.requestedCampaignId !== `gha-${s.workflowRunId}-${s.workflowRunAttempt}` || !Array.isArray(s.selections)) return false; + const source = input.result.source; + if (!source || !source.sha || !source.ref || !source.workflowRunUrl + || source.sha !== s.sourceSha || source.ref !== s.sourceRef || source.workflowRunUrl !== s.workflowRunUrl) return false; + if (input.relativeResultPath.split("/").some(part => !part || part === "." || part === "..")) return false; + return s.selections.some(value => { + if (!value || typeof value !== "object") return false; + const selected = value as Record; + const attempt = selected.workflowAttempt; + const campaign = `gha-${s.workflowRunId}-${attempt}`; + const artifact = `runner-e2e-${s.workflowRunId}-${attempt}-${input.result.executionId}`; + return selected.executionId === input.result.executionId && Number.isSafeInteger(attempt) + && Number(attempt) >= 1 && Number(attempt) < Number(s.workflowRunAttempt) + && selected.artifactName === artifact && input.result.accountingIdentity?.campaignId === campaign + && input.relativeResultPath.startsWith(`${artifact}/${campaign}-${input.result.executionId}/`); + }); +} + +function scope(result: RunnerE2EResult): string | undefined { + return result.accountingIdentity?.campaignId ?? result.source?.workflowRunUrl ?? undefined; +} + +export function retainedCampaignAttempts( + selected: readonly RunnerE2EResult[], + candidates: readonly RunnerE2EResult[], +): { attempts: RunnerE2EResult[]; excludedArtifacts: number } { + const attempts = candidates.filter(candidate => selected.some(result => + result.executionId === candidate.executionId && + (scope(result) ? scope(result) === scope(candidate) : result === candidate), + )); + return { attempts, excludedArtifacts: candidates.length - attempts.length }; +} + +function receipt(result: RunnerE2EResult): RunnerE2EAttemptReceipt { + const identity = [scope(result) ?? null, result.executionId, + result.accountingIdentity?.attemptId ?? [result.attempt, result.startedAt]]; + return { + id: createHash("sha256").update(JSON.stringify(identity)).digest("hex"), + executionId: result.executionId, + attempt: result.attempt, + startedAt: result.startedAt, + finishedAt: result.finishedAt, + durationMs: result.durationMs, + ...(result.workerTiming ? { workerTiming: result.workerTiming } : {}), + status: result.status, + cleanup: result.cleanup, + billing: summarizeExecutionBilling(result), + ...(result.feedback ? { feedback: result.feedback } : {}), + ...(result.tracing ? { tracing: result.tracing } : {}), + }; +} + +export function buildCampaignAccounting(input: { + results: readonly RunnerE2EResult[]; + attempts?: readonly RunnerE2EResult[]; + excludedArtifacts?: number; +}): RunnerE2ECampaignAccounting { + const receipts = new Map(); + for (const result of input.attempts ?? input.results) { + if (result.attempt === 0) continue; // Missing result placeholders are not free attempts. + const next = receipt(result); + const previous = receipts.get(next.id); + if (previous && previous.finishedAt === next.finishedAt && + JSON.stringify(previous.billing) !== JSON.stringify(next.billing)) { + throw new Error(`Conflicting accounting receipts for ${result.executionId} attempt ${result.attempt}`); + } + if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && next.workerTiming && + JSON.stringify(previous.workerTiming) !== JSON.stringify(next.workerTiming)) { + throw new Error(`Conflicting timing receipts for ${result.executionId} attempt ${result.attempt}`); + } + if (previous && previous.finishedAt === next.finishedAt && previous.workerTiming && !next.workerTiming) next.workerTiming = previous.workerTiming; + if (!previous || Date.parse(next.finishedAt) >= Date.parse(previous.finishedAt)) receipts.set(next.id, next); + } + const attempts = [...receipts.values()].sort((a, b) => + a.executionId.localeCompare(b.executionId) || a.attempt - b.attempt || a.id.localeCompare(b.id)); + let missingAttempts = 0; + for (const result of input.results) { + const retained = attempts.filter(attempt => attempt.executionId === result.executionId); + const latest = Math.max(result.attempt, ...retained.map(attempt => attempt.attempt), 1); + missingAttempts += Math.max(0, latest - new Set(retained.map(attempt => attempt.attempt)).size); + } + return { + version: 1, + scope: input.attempts ? "retained_attempts" : "selected_results", + attempts, + missingAttempts, + excludedArtifacts: input.excludedArtifacts ?? 0, + }; +} diff --git a/tests/runner-e2e/attempt-phase.ts b/tests/runner-e2e/attempt-phase.ts new file mode 100644 index 00000000000..1c8204e4c66 --- /dev/null +++ b/tests/runner-e2e/attempt-phase.ts @@ -0,0 +1,34 @@ +import { randomUUID } from "node:crypto"; +import { readFile, rename, rm, writeFile } from "node:fs/promises"; +import type { EvalChargeIdentity } from "./cost-ledger.js"; + +type Identity = Pick; +interface Checkpoint extends Identity { version: 1; phase: "setup" | "agent" } + +export async function createEvalAttemptPhase(file: string, identity: Identity) { + await writeFile(file, JSON.stringify({ version: 1, ...identity, phase: "setup" }), { mode: 0o600, flag: "wx" }); +} + +export async function readEvalAttemptPhase(file: string, expected: Identity): Promise { + const text = await readFile(file, "utf8"); + if (text.length > 4096) throw new Error("Invalid eval phase checkpoint"); + const value = JSON.parse(text) as Checkpoint; + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).some(key => !["version", "executionId", "attemptId", "phase"].includes(key)) + || value.version !== 1 || value.executionId !== expected.executionId || value.attemptId !== expected.attemptId + || !["setup", "agent"].includes(value.phase)) throw new Error("Invalid eval phase checkpoint"); + return { ...expected, phase: value.phase }; +} + +export async function beginEvalAgentPhase(executionId: string, env: NodeJS.ProcessEnv = process.env) { + const file = env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH; + if (!file && !env.PAPERCLIP_E2E_GATEWAY_BASE_URL) return; + if (!file || !env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID) throw new Error("Missing eval phase checkpoint"); + const identity = await readEvalAttemptPhase(file, { executionId, attemptId: env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID }); + if (identity.phase === "agent") return; + const temporary = `${file}.${randomUUID()}.tmp`; + try { + await writeFile(temporary, JSON.stringify({ version: 1, ...identity, phase: "agent" }), { mode: 0o600, flag: "wx" }); + await rename(temporary, file); + } finally { await rm(temporary, { force: true }); } +} diff --git a/tests/runner-e2e/billing.test.ts b/tests/runner-e2e/billing.test.ts index f16e2d9a60d..48352815c0e 100644 --- a/tests/runner-e2e/billing.test.ts +++ b/tests/runner-e2e/billing.test.ts @@ -6,6 +6,8 @@ import { summarizeExecutionBilling, } from "./billing.js"; import type { RunnerE2EResult } from "./types.js"; +import { durationLabel, llmCostLabel } from "./report-format.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; function result(overrides: Partial = {}): RunnerE2EResult { return { @@ -29,6 +31,80 @@ function result(overrides: Partial = {}): RunnerE2EResult { } describe("runner E2E billing summaries", () => { + it("keeps missing agent time separate from attempt duration and measured zero", () => { + const interrupted = result({ status: "failed", durationMs: 108_850 }); + expect(summarizeExecutionBilling(interrupted).runtime.agentRunDurationMs).toBeNull(); + const noCapturedRuns = result({ ...interrupted, runIds: [], + runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [] }) }); + expect(noCapturedRuns.runtimeUsage?.agentRunDurationMs).toBeNull(); + expect(() => validateRetainedRunnerResult(noCapturedRuns)).not.toThrow(); + const measured = result({ runtimeUsage: buildRuntimeUsage({ environmentId: "local", runs: [ + { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" }, + { startedAt: "2026-10-21T00:00:01Z", finishedAt: "2026-10-21T00:00:05Z" }, + ] }) }); + expect(aggregateCampaignBilling([measured]).agentRunDurationMs).toBe(8000); + expect(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs).toBeNull(); + expect(aggregateCampaignBilling([measured, noCapturedRuns]).agentRunDurationMs).toBeNull(); + expect(durationLabel(aggregateCampaignBilling([measured, interrupted]).agentRunDurationMs)).toBe("Unknown"); + const zero = buildRuntimeUsage({ environmentId: "local", runs: [ + { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:00Z" }, + ] }); + expect(zero.agentRunDurationMs).toBe(0); + expect(durationLabel(zero.agentRunDurationMs)).toBe("0ms"); + }); + + it.each([undefined, "not-a-date", "2026-10-20T23:59:59Z"])("retains unknown time for a missing or invalid end timestamp: %s", finishedAt => { + const runtimeUsage = buildRuntimeUsage({ environmentId: "daytona", runs: [ + { startedAt: "2026-10-21T00:00:00Z", finishedAt: "2026-10-21T00:00:04Z" }, + { startedAt: "2026-10-21T00:00:00Z", finishedAt }, + ], leases: [{ acquiredAt: "2026-10-21T00:00:00Z", releasedAt: finishedAt, + metadata: { cpu: 4, memory: 4, disk: 10 } }] }); + expect(runtimeUsage).toMatchObject({ agentRunDurationMs: null, leaseDurationMs: null, costStatus: "unavailable" }); + expect(runtimeUsage.estimatedListCostUsd).toBeUndefined(); + const retained = result({ environmentId: "daytona", runtimeUsage }); + expect(() => validateRetainedRunnerResult(retained)).not.toThrow(); + expect(aggregateCampaignBilling([retained, result()]).leaseDurationMs).toBeNull(); + expect(aggregateCampaignBilling([result()]).leaseDurationMs).toBe(0); + }); + + it("reads precise provider amounts without relabeling estimates or unpriced usage", () => { + const precise = summarizeExecutionBilling(result({ usage: { + inputTokens: 0, outputTokens: 0, cachedInputTokens: 5, + costUsdExact: "0.000000123", costStatus: "reported", accountingReceiptReady: true, + } })); + expect(precise.llm).toMatchObject({ reportedCostUsd: 0.000000123, runsWithReportedCost: 1, runsWithTokenUsage: 1 }); + const estimated = summarizeExecutionBilling(result({ usage: { + inputTokens: 100, outputTokens: 10, costUsdExact: "0.012345678", costStatus: "estimated", + } })); + expect(estimated.llm).toMatchObject({ reportedCostUsd: 0, estimatedCostUsd: 0.012345678, + runsWithReportedCost: 0, runsWithEstimatedCost: 1, costStatus: "estimated" }); + expect(estimated.observedAndEstimatedCostUsd).toBe(0.012345678); + const unpriced = summarizeExecutionBilling(result({ usage: { + inputTokens: 100, outputTokens: 10, costUsd: 0, costStatus: "unpriced", + } })); + expect(unpriced.llm).toMatchObject({ runsWithReportedCost: 0, costStatus: "unpriced" }); + }); + + it("distinguishes explicit zero usage from a partial or missing receipt", () => { + const zero = summarizeExecutionBilling(result({ usage: { inputTokens: 0, outputTokens: 0, costUsdExact: "0", accountingReceiptReady: true } })); + expect(zero.complete).toBe(true); + const partial = summarizeExecutionBilling(result({ usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01, accountingReceiptReady: false } })); + expect(partial.reportedCostUsd).toBe(0.01); + expect(partial.complete).toBe(false); + expect(summarizeExecutionBilling(result({ usage: null })).complete).toBe(false); + }); + it("retains a known OpenCode retry subtotal while the missing turn price stays incomplete", () => { + const partial = result({ usage: { inputTokens: 30, outputTokens: 6, costUsd: 0.5, costStatus: "unpriced", accountingReceiptReady: true } }); + const billing = summarizeExecutionBilling(partial); + expect(billing.llm).toMatchObject({ reportedCostUsd: 0.5, runsWithReportedCost: 1, runsWithPartialReportedCost: 1, costStatus: "partial" }); + expect(billing.observedAndEstimatedCostUsd).toBe(0.5); + expect(billing.complete).toBe(false); + expect(llmCostLabel(billing.reportedCostUsd, billing.llm)).toBe("$0.5000 (partial amount)"); + const campaign = aggregateCampaignBilling([partial, result({ executionId: "other", usage: { inputTokens: 1, outputTokens: 1, costUsd: 0.2 } })]); + expect(campaign).toMatchObject({ reportedLlmCostUsd: 0.7, observedAndEstimatedCostUsd: 0.7, testsWithCompleteBilling: 1, + llm: { runsWithReportedCost: 2, runsWithPartialReportedCost: 1, costStatus: "partial" } }); + expect(llmCostLabel(campaign.reportedLlmCostUsd, campaign.llm)).toBe("$0.7000 (partial amount)"); + }); it("counts completion judge reservations and preserves unknown spend after interruption", () => { const pending = { ...reserveCompletionQuality({ sourceId: "chat", marker: "x", worker: { id: "task", status: "done", completedAt: "2026-09-01" }, documents: [{ id: "doc", issueId: "task", body: "result" }], comments: [{ id: "reply", issueId: "chat", authorAgentId: "agent", createdAt: "2026-09-02", body: "ready" }], runs: [] }, 0.5), name: "completion", expectedPass: true }; const unknown = summarizeExecutionBilling(result({ completionQuality: [pending] })); diff --git a/tests/runner-e2e/billing.ts b/tests/runner-e2e/billing.ts index 4681696a7ee..35f70e83569 100644 --- a/tests/runner-e2e/billing.ts +++ b/tests/runner-e2e/billing.ts @@ -32,21 +32,30 @@ export interface CampaignBillingSummary { assistant?: import("./types.js").RunnerE2EAggregateBillingSummary["assistant"]; judge?: import("./types.js").RunnerE2EJudgeBillingSummary; testCount: number; - agentRunDurationMs: number; - leaseDurationMs: number; + agentRunDurationMs: number | null; + leaseDurationMs: number | null; llm: RunnerE2EBillingSummary["llm"]; reportedLlmCostUsd: number; + estimatedLlmCostUsd?: number; estimatedRuntimeCostUsd: number; observedAndEstimatedCostUsd: number | null; testsWithCompleteBilling: number; } /** Render observed subtotals without presenting absent measurements as zero. */ -export function billingCoverageLabel(value: string, covered: number, total: number): string { +export function billingCoverageLabel(value: string, covered: number, total: number, incompleteAmount = false): string { if (!Number.isInteger(covered) || !Number.isInteger(total) || covered <= 0 || total <= 0 || covered > total) { return "Unavailable"; } - return covered < total ? `${value} (partial: ${covered}/${total} runs)` : value; + return covered < total ? `${value} (partial: ${covered}/${total} runs)` : incompleteAmount ? `${value} (partial amount)` : value; +} + +function costCoverageStatus(runs: number, tokens: number, reported: number, estimated: number, partial = 0): RunnerE2EBillingSummary["llm"]["costStatus"] { + if (partial > 0) return "partial"; + if (runs > 0 && reported === runs) return "reported"; + if (runs > 0 && reported + estimated === runs) return "estimated"; + if (reported + estimated > 0) return "partial"; + return tokens > 0 ? "unpriced" : "unavailable"; } function record(value: unknown): Record { @@ -92,24 +101,25 @@ function usageMeasurement(usage: Record) { } function usageCostUsd(usage: Record) { + if (usage.billingType === "subscription") return undefined; const measurement = usageMeasurement(usage); + const exact = typeof usage.costUsdExact === "string" && /^\d+(?:\.\d+)?$/.test(usage.costUsdExact) + ? finiteNumber(Number(usage.costUsdExact)) : undefined; const direct = - firstNumber(usage, [ - "cacheAdjustedCostUsd", - "costUsd", - "providerCostUsd", - ]) ?? + firstNumber(usage, ["cacheAdjustedCostUsd"]) ?? exact ?? + firstNumber(usage, ["costUsd", "providerCostUsd"]) ?? firstNumber(measurement, [ "cacheAdjustedCostUsd", "costUsd", "providerCostUsd", ]); - if (direct !== undefined) return direct; + if (direct !== undefined) return usage.costStatus === "unpriced" && direct === 0 ? undefined : direct; const cost = record(usage.cost); const currency = typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; if (currency !== "USD") return undefined; - return firstNumber(cost, ["amount", "total"]); + const amount = firstNumber(cost, ["amount", "total"]); + return usage.costStatus === "unpriced" && amount === 0 ? undefined : amount; } function usageEntries( @@ -144,9 +154,13 @@ function durationBetween( ) { const started = startedAt ? new Date(startedAt).getTime() : Number.NaN; const finished = finishedAt ? new Date(finishedAt).getTime() : Number.NaN; - return Number.isFinite(started) && Number.isFinite(finished) - ? Math.max(0, finished - started) - : 0; + return Number.isFinite(started) && Number.isFinite(finished) && finished >= started + ? finished - started + : null; +} + +function sumDurations(values: readonly (number | null)[]): number | null { + return values.some(value => value === null) ? null : values.reduce((total, value) => total + value!, 0); } function resourceValue( @@ -162,10 +176,9 @@ export function buildRuntimeUsage(input: { leases?: readonly RuntimeLeaseUsageInput[]; fallbackFinishedAt?: string | Date; }): RunnerE2ERuntimeUsage { - const agentRunDurationMs = input.runs.reduce( - (total, run) => total + durationBetween(run.startedAt, run.finishedAt), - 0, - ); + const agentRunDurationMs = input.runs.length > 0 + ? sumDurations(input.runs.map(run => durationBetween(run.startedAt, run.finishedAt))) + : null; if (input.environmentId === "local") { return { provider: "local", @@ -179,6 +192,7 @@ export function buildRuntimeUsage(input: { const leases = input.leases ?? []; let leaseDurationMs = 0; + let leaseDurationsComplete = leases.length > 0; let estimatedListCostUsd = 0; let resourcesComplete = leases.length > 0; const resourceRows: Array<{ cpu: number; memory: number; disk: number }> = []; @@ -186,6 +200,11 @@ export function buildRuntimeUsage(input: { const finishedAt = lease.releasedAt ?? input.fallbackFinishedAt ?? lease.updatedAt ?? null; const durationMs = durationBetween(lease.acquiredAt, finishedAt); + if (durationMs === null) { + leaseDurationsComplete = false; + resourcesComplete = false; + continue; + } leaseDurationMs += durationMs; const cpu = resourceValue(lease.metadata, "cpu"); const memory = resourceValue(lease.metadata, "memory"); @@ -209,7 +228,7 @@ export function buildRuntimeUsage(input: { return { provider: "daytona", agentRunDurationMs, - leaseDurationMs: leases.length > 0 ? leaseDurationMs : null, + leaseDurationMs: leaseDurationsComplete ? leaseDurationMs : null, leaseCount: leases.length, ...(commonResource("cpu") === undefined ? {} @@ -243,7 +262,7 @@ export function fallbackRuntimeUsage( if (result.runtimeUsage) return result.runtimeUsage; return { provider: result.environmentId, - agentRunDurationMs: result.durationMs, + agentRunDurationMs: null, leaseDurationMs: null, leaseCount: 0, costStatus: @@ -269,7 +288,11 @@ export function summarizeExecutionBilling( let cachedInputTokens = 0; let runsWithTokenUsage = 0; let runsWithReportedCost = 0; + let runsWithEstimatedCost = 0; + let runsWithPartialReportedCost = 0; let reportedCostUsd = 0; + let estimatedCostUsd = 0; + let incompleteReceipt = false; for (const usage of entries) { if (!usage) continue; const measurement = usageMeasurement(usage); @@ -290,22 +313,24 @@ export function summarizeExecutionBilling( inputTokens += input; outputTokens += output; cachedInputTokens += cached; - if (input > 0 || output > 0 || cached > 0) runsWithTokenUsage += 1; + const completeTokens = firstNumber(measurement, ["inputTokens", "input", "promptTokens"]) !== undefined && + firstNumber(measurement, ["outputTokens", "output", "completionTokens"]) !== undefined && usage.accountingReceiptReady !== false; + if (completeTokens) runsWithTokenUsage += 1; + if (usage.accountingReceiptReady === false) incompleteReceipt = true; const costUsd = usageCostUsd(usage); - if (costUsd !== undefined && (costUsd > 0 || input > 0 || output > 0)) { - runsWithReportedCost += 1; - reportedCostUsd += costUsd; + if (costUsd !== undefined) { + if (usage.costStatus === "estimated") { + runsWithEstimatedCost += 1; + estimatedCostUsd += costUsd; + } else { + runsWithReportedCost += 1; + if (usage.costStatus === "unpriced") runsWithPartialReportedCost += 1; + reportedCostUsd += costUsd; + } } } const runCount = Math.max(requestedRunCount, entries.length); - const costStatus = - runsWithReportedCost === runCount - ? "reported" - : runsWithReportedCost > 0 - ? "partial" - : runsWithTokenUsage > 0 - ? "unpriced" - : "unavailable"; + const costStatus = costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost); const runtime = { ...fallbackRuntimeUsage(result) }; if (browserOnly) runtime.agentRunDurationMs = 0; const estimatedRuntimeCostUsd = runtime.estimatedListCostUsd ?? 0; @@ -315,12 +340,13 @@ export function summarizeExecutionBilling( // The operations receipt covers setup and agent requests for ONE owning attempt. // Descendants reuse the fixture context without copying its receipt or charges. const operations = result.productJourney?.operations; - const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd; + const modelSubtotal = operations ? operations.modelCostUsd : reportedCostUsd + estimatedCostUsd; const complete = + !incompleteReceipt && runsWithPartialReportedCost === 0 && (!quality || quality.estimatedCostUsd !== null) && (!operations || operations.modelCostUsd !== null) && runsWithTokenUsage === runCount && - runsWithReportedCost === runCount && + runsWithReportedCost + runsWithEstimatedCost === runCount && runtime.costStatus !== "unavailable"; return { ...(result.publicMcp ? { assistant: result.publicMcp } : {}), @@ -328,11 +354,14 @@ export function summarizeExecutionBilling( runCount, runsWithTokenUsage, runsWithReportedCost, + runsWithEstimatedCost, + runsWithPartialReportedCost, inputTokens, outputTokens, cachedInputTokens, totalTokens: inputTokens + cachedInputTokens + outputTokens, reportedCostUsd, + estimatedCostUsd, costStatus, }, runtime, @@ -347,7 +376,16 @@ export function summarizeExecutionBilling( export function aggregateCampaignBilling( results: readonly RunnerE2EResult[], ): CampaignBillingSummary { - const summaries = results.map(summarizeExecutionBilling); + return aggregateBillingSummaries(results.map(result => ({ + executionId: result.executionId, billing: summarizeExecutionBilling(result), + }))); +} + +export function aggregateBillingSummaries( + entries: readonly { executionId: string; billing: RunnerE2EBillingSummary }[], +): CampaignBillingSummary { + const summaries = entries.map(entry => entry.billing); + const executionIds = [...new Set(entries.map(entry => entry.executionId))]; const runCount = summaries.reduce( (total, summary) => total + summary.llm.runCount, 0, @@ -364,6 +402,9 @@ export function aggregateCampaignBilling( (total, summary) => total + summary.reportedCostUsd, 0, ); + const runsWithEstimatedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithEstimatedCost ?? 0), 0); + const runsWithPartialReportedCost = summaries.reduce((sum, summary) => sum + (summary.llm.runsWithPartialReportedCost ?? 0), 0); + const estimatedLlmCostUsd = summaries.reduce((sum, summary) => sum + (summary.llm.estimatedCostUsd ?? 0), 0); const estimatedRuntimeCostUsd = summaries.reduce( (total, summary) => total + summary.estimatedRuntimeCostUsd, 0, @@ -386,19 +427,16 @@ export function aggregateCampaignBilling( reservedCostUsd: judges.reduce((n, q) => n + q.reservedCostUsd, 0), attemptsWithUnknownUsage: judges.filter(q => q.estimatedCostUsd === null).length, } } : {}), - testCount: results.length, - agentRunDurationMs: summaries.reduce( - (total, summary) => total + summary.runtime.agentRunDurationMs, - 0, - ), - leaseDurationMs: summaries.reduce( - (total, summary) => total + (summary.runtime.leaseDurationMs ?? 0), - 0, - ), + testCount: executionIds.length, + agentRunDurationMs: sumDurations(summaries.map(summary => summary.runtime.agentRunDurationMs)), + leaseDurationMs: sumDurations(summaries.filter(summary => summary.runtime.provider === "daytona") + .map(summary => summary.runtime.leaseDurationMs)), llm: { runCount, runsWithTokenUsage, runsWithReportedCost, + runsWithEstimatedCost, + runsWithPartialReportedCost, inputTokens: summaries.reduce( (total, summary) => total + summary.llm.inputTokens, 0, @@ -416,19 +454,14 @@ export function aggregateCampaignBilling( 0, ), reportedCostUsd: reportedLlmCostUsd, - costStatus: - runsWithReportedCost === runCount - ? "reported" - : runsWithReportedCost > 0 - ? "partial" - : runsWithTokenUsage > 0 - ? "unpriced" - : "unavailable", + estimatedCostUsd: estimatedLlmCostUsd, + costStatus: costCoverageStatus(runCount, runsWithTokenUsage, runsWithReportedCost, runsWithEstimatedCost, runsWithPartialReportedCost), }, reportedLlmCostUsd, + estimatedLlmCostUsd, estimatedRuntimeCostUsd, observedAndEstimatedCostUsd: summaries.some(s => s.observedAndEstimatedCostUsd === null) ? null : summaries.reduce((total, summary) => total + (summary.observedAndEstimatedCostUsd ?? 0), 0), - testsWithCompleteBilling: summaries.filter((summary) => summary.complete) - .length, + testsWithCompleteBilling: executionIds.filter(id => + entries.filter(entry => entry.executionId === id).every(entry => entry.billing.complete)).length, }; } diff --git a/tests/runner-e2e/braintrust-experiment-import.ts b/tests/runner-e2e/braintrust-experiment-import.ts new file mode 100644 index 00000000000..eef8f3598c5 --- /dev/null +++ b/tests/runner-e2e/braintrust-experiment-import.ts @@ -0,0 +1,45 @@ +import { readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { evalBraintrustConfig } from "./braintrust.js"; +import { buildBraintrustExperiment, importBraintrustExperiment } from "./braintrust-experiment.js"; + +const [campaignFile, originalPathsFile, outputFile, mode] = process.argv.slice(2); +if (!campaignFile || !originalPathsFile || !outputFile || ![undefined, "--dry-run"].includes(mode)) { + throw new Error("Usage: braintrust-experiment-import.ts [--dry-run]"); +} +let totalBytes = 0; +const readJson = async (file: string) => { + const bytes = await readFile(file); + totalBytes += bytes.length; + if (bytes.length > 8 * 1024 * 1024 || totalBytes > 64 * 1024 * 1024) throw new Error("Experiment input exceeds its file or campaign size limit"); + return JSON.parse(bytes.toString("utf8")); +}; +const paths = await readJson(originalPathsFile); +if (!Array.isArray(paths) || paths.length === 0 || paths.length > 10_000 + || paths.some(file => typeof file !== "string" || !path.isAbsolute(file))) throw new Error("Expected explicit absolute original result paths"); +if ([campaignFile, originalPathsFile, ...paths].some(file => path.resolve(file) === path.resolve(outputFile))) { + throw new Error("Experiment output must not replace input evidence"); +} +const campaign = await readJson(campaignFile), originals = []; +for (const file of paths) originals.push(await readJson(file)); +const projectId = process.env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? ""; +const plan = buildBraintrustExperiment(projectId, campaign, originals); +const persist = async (value: unknown) => { + const temporary = `${outputFile}.${randomUUID()}.tmp`; + await writeFile(temporary, `${JSON.stringify(value, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + await rename(temporary, outputFile); +}; +if (mode === "--dry-run") { + await persist(plan); + console.log(`Prepared ${plan.events.length} retained rows; no network requests made.`); +} else { + const config = evalBraintrustConfig(process.env); + if (!config) throw new Error("Braintrust project and ingestion credential required"); + const previous = await readJson(outputFile).catch(error => { + if (error?.code === "ENOENT") return undefined; + throw error; + }); + const receipt = await importBraintrustExperiment(config, campaign, originals, persist, fetch, previous); + console.log(`Braintrust accepted ${receipt.acceptedRows} retained rows in experiment ${receipt.experimentId}; indexed verification remains required.`); +} diff --git a/tests/runner-e2e/braintrust-experiment.test.ts b/tests/runner-e2e/braintrust-experiment.test.ts new file mode 100644 index 00000000000..41c61fc5b20 --- /dev/null +++ b/tests/runner-e2e/braintrust-experiment.test.ts @@ -0,0 +1,328 @@ +import { expect, it, vi } from "vitest"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { buildBraintrustExperiment, importBraintrustExperiment, renderBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import { buildRunnerCampaign } from "./history.js"; +import { runnerMatrix } from "./catalog.js"; +import { emptyFeedbackSummary } from "./feedback.js"; +import { openExperimentIngestion } from "./experiment-ingestion.js"; +import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js"; + +const projectId = "11111111-1111-4111-8111-111111111111"; +const experimentId = "22222222-2222-4222-8222-222222222222"; +function fixture() { + const cell = runnerMatrix.find(cell => cell.id === "core-compatibility.runner-codex.local.message-marker")!; + const first: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, + suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash, + accountingIdentity: { campaignId: "retained-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null }, + attempt: 1, status: "failed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id, + environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode, + startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000, + error: "private-canary-error", usage: { privateCanary: "private-canary-usage" }, feedback: emptyFeedbackSummary() }; + const second: RunnerE2EResult = { ...structuredClone(first), source: { ...first.source!, sha: "b".repeat(40) }, + accountingIdentity: { campaignId: "retained-pilot", attemptId: "second" }, attempt: 2, status: "passed", error: undefined, + startedAt: "2026-10-20T00:01:00.000Z", finishedAt: "2026-10-20T00:01:01.000Z", + tracing: { projectId, traceId: "c".repeat(32), spanId: "d".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured" } }; + const originals = [first, second]; + const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: second.finishedAt, + expected: [cell.id], results: [second], attempts: originals }); + return { campaign, originals }; +} + +it("imports all attempts with their own source and original grades, excluding private evidence and duplicate cost metrics", () => { + const { campaign, originals } = fixture(); + const plan = buildBraintrustExperiment(projectId, campaign, originals); + expect(plan.events).toHaveLength(2); + expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]); + expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]); + expect(JSON.stringify(plan)).not.toMatch(/private-canary|estimated_cost|prompt_tokens/); + expect(plan.events.every(row => row.metadata.request_ledger === null)).toBe(true); + expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.trace).toMatchObject({ root_span_id: "c".repeat(32) }); + expect(buildBraintrustExperiment(projectId, { ...campaign, generatedAt: "2026-10-21T00:00:00Z" }, [...originals].reverse())).toEqual(plan); + expect(buildBraintrustExperiment(projectId, campaign, [...originals, originals[0]!])).toEqual(plan); +}); + +it("imports retained grades through the scheduled transport allowance without changing the evidence", async () => { + const root = await mkdtemp(path.join(tmpdir(), "bounded-experiment-import-")); + const { campaign, originals } = fixture(), before = structuredClone({ campaign, originals }); + const plan = buildBraintrustExperiment(projectId, campaign, originals), receiptPath = path.join(root, "ingestion.jsonl"); + const meter = await openExperimentIngestion({ projectId, campaignId: campaign.campaignId, receiptPath, deadlineAt: Date.now() + 60_000, + transport: async (url, init) => String(url).endsWith("/experiment") + ? Response.json({ ...JSON.parse(String(init!.body)), id: experimentId }) + : String(url).endsWith("/insert") ? Response.json({ row_ids: plan.events.map(row => row.id) }) : Response.json({}) }); + try { + expect(await importBraintrustExperiment({ projectId, apiKey: "private-fixture" }, campaign, originals, async () => {}, meter.transport)) + .toMatchObject({ status: "accepted", acceptedRows: 2, expectedRows: 2 }); + await meter.close(); + const rows = (await readFile(receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + expect(rows.at(-1)).toMatchObject({ admittedRequests: 3, admittedScores: 4, unresolvedRequests: 0 }); + expect({ campaign, originals }).toEqual(before); + } finally { await meter.close(); await rm(root, { recursive: true, force: true }); } +}); + +it.each(["missing", "foreign", "modified-grade", "ambiguous-source", "missing-source", "foreign-trace", "missing-hash"])("rejects %s evidence before network requests", async (kind) => { + const { campaign, originals } = fixture(); + if (kind === "missing") originals.pop(); + if (kind === "foreign") originals[0]!.accountingIdentity!.campaignId = "other"; + if (kind === "modified-grade") originals[0]!.status = "passed"; + if (kind === "ambiguous-source") originals.push({ ...originals[0]!, source: { ...originals[0]!.source!, sha: "c".repeat(40) } }); + if (kind === "missing-source") originals[0]!.source = undefined; + if (kind === "foreign-trace") originals[1]!.tracing!.projectId = experimentId; + if (kind === "missing-hash") originals[0]!.suiteDefinitionHash = undefined; + const transport = vi.fn(); + await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, transport)).rejects.toThrow(); + expect(transport).not.toHaveBeenCalled(); +}); + +it("changes experiment identity for changed evidence while keeping unknown accounting unknown", () => { + const { campaign, originals } = fixture(); + const before = buildBraintrustExperiment(projectId, campaign, originals); + campaign.accounting!.missingAttempts = 1; + const after = buildBraintrustExperiment(projectId, campaign, originals); + expect(after.payloadHash).not.toBe(before.payloadHash); + expect(after.metadata.request_ledger).toBeNull(); + expect(after.metadata.missing_attempts).toBe(1); +}); + +it("imports carried-forward receipts with their original campaign, revision, and grades", () => { + const { campaign, originals } = fixture(); + const report = { ...campaign, campaignId: "later-ci-attempt" }; + const before = structuredClone(originals); + const plan = buildBraintrustExperiment(projectId, report, originals); + expect(plan.metadata.campaign_id).toBe("later-ci-attempt"); + expect(plan.events.every(row => row.metadata.campaign_id === "retained-pilot")).toBe(true); + expect(plan.events.map(row => row.input.source_revision).sort()).toEqual(["a".repeat(40), "b".repeat(40)]); + expect(plan.events.map(row => row.scores.product_pass).sort()).toEqual([0, 1]); + expect(plan.events.map(row => row.id)).toEqual(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.id)); + expect(originals).toEqual(before); + const wrong = structuredClone(originals); + wrong[0]!.accountingIdentity!.campaignId = report.campaignId; + expect(() => buildBraintrustExperiment(projectId, report, wrong)).toThrow(); +}); + +it("imports original grades when report evidence validation changes the selected outcome", () => { + const { campaign, originals } = fixture(); + const adjusted = structuredClone(campaign); + adjusted.results[0]!.status = "failed"; + adjusted.results[0]!.error = "Missing final-state.png"; + const plan = buildBraintrustExperiment(projectId, adjusted, originals); + expect(plan.events.find(row => row.metadata.attempt_id === "second")!.scores.product_pass).toBe(1); + expect(plan).toEqual(buildBraintrustExperiment(projectId, campaign, originals)); + expect(adjusted.results[0]!.status).toBe("failed"); + const changed = structuredClone(originals); + changed[1]!.status = "failed"; + expect(() => buildBraintrustExperiment(projectId, adjusted, changed)).toThrow(); +}); + +it("accepts current catalog model names including Cursor qualification parameters", () => { + for (const model of new Set(runnerMatrix.map(cell => cell.profile.model))) { + const { campaign, originals } = fixture(); + const changed = originals.map(original => ({ ...original, model })); + const selected = { ...campaign, results: campaign.results.map(result => ({ ...result, model })) }; + const plan = buildBraintrustExperiment(projectId, selected, changed); + expect(plan.events.every(row => row.input.model === model)).toBe(true); + } +}); + +it.each(["", "model\nprivate", "model\u0000private", "model", "a".repeat(257)])("rejects malformed model names %j", model => { + const { campaign, originals } = fixture(); + expect(() => buildBraintrustExperiment(projectId, campaign, originals.map(result => ({ ...result, model })))).toThrow(); +}); + +it("keeps precise ledger amounts and reservations separate from trace metrics and unobserved attempts", () => { + const { campaign, originals } = fixture(); + campaign.operations = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: campaign.campaignId, + journalHash: "c".repeat(64), records: 5, deadlineAt: "2026-10-21T01:00:00Z", limitUsd: "100.000000000", + knownUsd: "0.000000001", heldUsd: "4.000000000", committedUsd: "4.000000001", requests: 1, + unresolvedRequests: 1, conflictingRequests: 0, inflight: 0, + breakdown: { estimated: "0.000000001", reported: "0.000000000", reconciled: "0.000000000", allocated: "0.000000000" }, + attempts: [{ executionId: originals[0]!.executionId, attemptId: "first", requests: 1, unresolvedRequests: 1, + knownUsd: "0.000000001", heldUsd: "4.000000000" }] }; + const plan = buildBraintrustExperiment(projectId, campaign, originals); + expect(plan.metadata.request_ledger).toMatchObject({ known_usd: "0.000000001", additional_held_usd: "4.000000000" }); + expect(plan.events.find(row => row.metadata.attempt_id === "first")!.metadata.request_ledger).toMatchObject({ unresolved_requests: 1 }); + expect(plan.events.find(row => row.metadata.attempt_id === "second")!.metadata.request_ledger).toBeNull(); + expect(plan.events.every(row => Object.keys(row.metrics).sort().join() === "end,start")).toBe(true); + const carried = buildBraintrustExperiment(projectId, { + ...campaign, campaignId: "later-ci-attempt", + operations: { ...campaign.operations, ledgerCampaignId: "later-ci-attempt" }, + }, originals); + expect(carried.metadata.request_ledger).toMatchObject({ requests: 1 }); + expect(carried.events.every(row => row.metadata.request_ledger === null)).toBe(true); +}); + +function receiver(campaign: ReturnType["campaign"], originals: RunnerE2EResult[]) { + const plan = buildBraintrustExperiment(projectId, campaign, originals); + return vi.fn(async (url, init) => { + expect(String(url).startsWith("https://api.braintrust.dev/v1/")).toBe(true); + expect(init?.redirect).toBe("error"); + if (String(url).endsWith("/summarize")) return Response.json({ experiment_name: plan.name, + project_url: "https://www.braintrust.dev/app/Example/p/Evals", experiment_url: `https://www.braintrust.dev/app/Example/p/Evals/experiments/${plan.name}` }); + const sent = JSON.parse(String(init?.body)); + if (String(url).endsWith("/experiment")) { + expect(sent).toMatchObject({ public: false, ensure_new: false, project_id: projectId, name: plan.name }); + return Response.json({ ...sent, id: experimentId }); + } + return Response.json({ row_ids: sent.events.map((row: { id: string }) => row.id) }); + }); +} + +it("retries identical evidence using the same private experiment and stable row IDs without invoking agents", async () => { + const { campaign, originals } = fixture(); + const transport = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = []; + for (let i = 0; i < 2; i++) await importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals, + async receipt => { receipts.push(receipt); }, transport); + expect(receipts.map(receipt => receipt.status)).toEqual(["created", "accepted", "accepted", "created", "accepted", "accepted"]); + expect(receipts[2]).toEqual(receipts[5]); + expect(receipts[1]).toMatchObject({ acceptedRows: 2, expectedRows: 2, verification: "unverified" }); + expect(transport.mock.calls[0]![1]!.body).toBe(transport.mock.calls[3]![1]!.body); + expect(transport.mock.calls[1]![1]!.body).toBe(transport.mock.calls[4]![1]!.body); +}); + +it("preserves an evidence-bound experiment link through report regeneration and rejects stale or unsafe links", async () => { + const { campaign, originals } = fixture(); + const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals)); + campaign.braintrustExperiment = receipt; + const root = await mkdtemp(path.join(tmpdir(), "experiment-report-")); + try { + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + expect(JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")).braintrustExperiment).toEqual(receipt); + expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain(receipt.experimentUrl); + expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, experimentUrl: "https://untrusted.example/app/secret" } })).toThrow(); + expect(() => renderBraintrustExperiment({ ...campaign, braintrustExperiment: { ...receipt, privateNote: "private-canary" } as never })).toThrow(); + campaign.accounting!.missingAttempts++; + expect(() => renderBraintrustExperiment(campaign)).toThrow(); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("rebuilds a report with a missing cell and its saved experiment without rerunning retained attempts", async () => { + const root = await mkdtemp(path.join(tmpdir(), "experiment-missing-report-")); + const repositoryRoot = path.resolve(import.meta.dirname, "../.."); + const original = fixture().originals[1]!; + const missing = runnerMatrix.find(cell => cell.id !== original.executionId)!; + const input = path.join(root, "input"), attempt = path.join(input, "attempt"); + const receiptPath = path.join(root, "experiment.json"); + async function report(output: string, savedReceipt?: string) { + const result = await promisify(execFile)(process.execPath, + [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, + env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: input, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_E2E_CAMPAIGN_ID: "retained-pilot", PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([original.executionId, missing.id]), + PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT: savedReceipt }, + }).catch(error => error as { code: number; stderr: string }); + // Missing coverage still fails the campaign, but must not prevent rendering it. + expect(result).toMatchObject({ code: 1 }); + expect(result.stderr).not.toContain("Invalid retained experiment evidence"); + return JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")) as RunnerE2ECampaign; + } + try { + await mkdir(attempt, { recursive: true }); + const raw = JSON.stringify(original); + await writeFile(path.join(attempt, "result.json"), raw); + await writeFile(path.join(attempt, "final-state.png"), "fixture-png"); + await writeFile(path.join(attempt, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] })); + const before = await report(path.join(root, "before")); + const receipt = await importBraintrustExperiment({ projectId, apiKey: "fixture-key" }, before, [original], async () => {}, receiver(before, [original])); + await writeFile(receiptPath, JSON.stringify(receipt)); + const after = await report(path.join(root, "after"), receiptPath); + expect(after.results.find(result => result.attempt === 0)!.startedAt).not.toBe(before.results.find(result => result.attempt === 0)!.startedAt); + expect(after.braintrustExperiment).toEqual(receipt); + expect(after.accounting).toEqual(before.accounting); + expect(await readFile(path.join(root, "after", "index.html"), "utf8")).toContain(receipt.experimentUrl); + expect(await readFile(path.join(attempt, "result.json"), "utf8")).toBe(raw); + const changed = structuredClone(after); + changed.results.find(result => result.attempt > 0)!.startedAt = "2026-10-20T00:00:30.000Z"; + expect(() => renderBraintrustExperiment(changed)).toThrow(); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("checkpoints accepted batches before a later insert fails", async () => { + const base = fixture().originals[0]!; + const originals = Array.from({ length: 101 }, (_, i) => ({ ...base, attempt: i + 1, + accountingIdentity: { campaignId: "retained-pilot", attemptId: `attempt-${i}` } })); + const campaign = buildRunnerCampaign({ campaignId: "retained-pilot", generatedAt: base.finishedAt, + expected: [base.executionId], results: [originals[100]!], attempts: originals }); + const normal = receiver(campaign, originals), receipts: BraintrustExperimentReceipt[] = []; + let calls = 0; + await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, + async receipt => { receipts.push(receipt); }, async (url, init) => { + if (++calls === 3) throw new Error("private-network-error"); + return normal(url, init); + })).rejects.toThrow("Remote details withheld"); + expect(receipts).toHaveLength(2); + expect(receipts[1]).toMatchObject({ status: "created", acceptedRows: 100, expectedRows: 101 }); + const previous = structuredClone(receipts[1]!); + calls = 0; + await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, + async receipt => { receipts.push(receipt); }, async (url, init) => { + if (++calls === 2) throw new Error("retry insert failed"); + return normal(url, init); + }, previous)).rejects.toThrow("Remote details withheld"); + expect(receipts).toHaveLength(2); + expect(receipts.at(-1)).toEqual(previous); + await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, + async receipt => { receipts.push(receipt); }, normal, previous); + expect(receipts.slice(2).map(receipt => receipt.acceptedRows)).toEqual([100, 101, 101]); + expect(receipts.at(-1)!.status).toBe("accepted"); +}); + +it.each(["source", "definition"])("rejects a saved link and original import after selected %s provenance changes", async kind => { + const { campaign, originals } = fixture(); + const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals)); + const changed = structuredClone(campaign); + changed.braintrustExperiment = receipt; + if (kind === "source") changed.results[0]!.source!.sha = "e".repeat(40); + else changed.results[0]!.suiteDefinitionHash = "f".repeat(64); + expect(() => renderBraintrustExperiment(changed)).toThrow(); + expect(() => buildBraintrustExperiment(projectId, changed, originals)).toThrow(); + expect(buildBraintrustExperiment(projectId, campaign, originals).events.map(row => row.input.source_revision).sort()) + .toEqual(["a".repeat(40), "b".repeat(40)]); +}); + +it("propagates a failed receipt save after successful summary lookup while retaining the accepted checkpoint", async () => { + const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = []; + await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async receipt => { + if (receipt.experimentUrl) throw new Error("disk write failed"); + receipts.push(receipt); + }, receiver(campaign, originals))).rejects.toThrow("import incomplete"); + expect(receipts.at(-1)).toMatchObject({ status: "accepted", acceptedRows: 2 }); + expect(receipts.at(-1)!.experimentUrl).toBeUndefined(); +}); + +it("rejects conflicting retry receipts before overwriting saved evidence", async () => { + const { campaign, originals } = fixture(); + const receipt = await importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, async () => {}, receiver(campaign, originals)); + const persist = vi.fn(), transport = vi.fn(); + await expect(importBraintrustExperiment({ projectId, apiKey: "key" }, campaign, originals, persist, transport, + { ...receipt, projectId: experimentId })).rejects.toThrow(); + expect(persist).not.toHaveBeenCalled(); + expect(transport).not.toHaveBeenCalled(); +}); + +it.each(["public", "wrong-project", "wrong-hash", "partial-insert", "receiver-error", "oversized"])("retains safe incomplete status for %s", async kind => { + const { campaign, originals } = fixture(), receipts: BraintrustExperimentReceipt[] = []; + const normal = receiver(campaign, originals); + const transport: typeof fetch = async (url, init) => { + const response = await normal(url, init), value = await response.json() as Record; + if (String(url).endsWith("/experiment")) { + if (kind === "public") value.public = true; + if (kind === "wrong-project") value.project_id = experimentId; + if (kind === "wrong-hash") value.metadata = { payload_hash: "a".repeat(64) }; + if (kind === "oversized") return new Response("x".repeat(1_048_577)); + } else { + if (kind === "partial-insert") value.row_ids = []; + if (kind === "receiver-error") return new Response("private-upstream-failure", { status: 503 }); + } + return Response.json(value); + }; + await expect(importBraintrustExperiment({ projectId, apiKey: "private-key" }, campaign, originals, + async receipt => { receipts.push(receipt); }, transport)).rejects.toThrow("Remote details withheld"); + expect(receipts.some(receipt => receipt.status === "accepted")).toBe(false); + expect(JSON.stringify(receipts)).not.toMatch(/private-key|private-upstream/); + if (["partial-insert", "receiver-error"].includes(kind)) expect(receipts[0]).toMatchObject({ status: "created", acceptedRows: 0 }); +}); diff --git a/tests/runner-e2e/braintrust-experiment.ts b/tests/runner-e2e/braintrust-experiment.ts new file mode 100644 index 00000000000..4b2859e17d8 --- /dev/null +++ b/tests/runner-e2e/braintrust-experiment.ts @@ -0,0 +1,207 @@ +import { createHash } from "node:crypto"; +import { buildCampaignAccounting } from "./attempt-accounting.js"; +import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js"; +import { validateCampaignFeedbackReview } from "./feedback-dispositions.js"; +import { validateEvalOperationsReport } from "./operations-report.js"; +import { html, isPublicReportTimestamp } from "./report-format.js"; +import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +const identifier = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(value); +const modelName = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9._:/@=,\[\]-]{0,255}$/.test(value); +const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0; +const invalid = (): never => { throw new Error("Invalid retained experiment evidence"); }; + +function attemptProjection(result: RunnerE2EResult, projectId: string) { + if (result.schema !== "paperclip.runner-e2e.result/v2" + || ![result.executionId, result.suiteId, result.profileId, result.caseId, result.provider, + result.accountingIdentity?.campaignId, result.accountingIdentity?.attemptId].every(identifier) + || !modelName(result.model) + || !/^[0-9a-f]{64}$/.test(result.suiteDefinitionHash ?? "") || !/^[0-9a-f]{40}$/.test(result.source?.sha ?? "") + || !count(result.attempt) || result.attempt < 1 || !count(result.durationMs) + || !["passed", "failed"].includes(result.status) || !["passed", "failed", "not_started"].includes(result.cleanup) + || !["native", "legacy"].includes(result.runtimeMode) || !["local", "daytona"].includes(result.environmentId) + || !isPublicReportTimestamp(result.startedAt) || !isPublicReportTimestamp(result.finishedAt) + || Date.parse(result.finishedAt) < Date.parse(result.startedAt)) invalid(); + const feedback = result.feedback; + if (feedback && (![feedback.complaints, feedback.suggestions, feedback.invocations?.failed, feedback.invocations?.unknown].every(count) + || !["complete", "partial", "unavailable", "not_started"].includes(feedback.collection) + || !["complete", "partial", "unavailable", "not_started"].includes(feedback.invocationCollection))) invalid(); + const trace = result.tracing; + if (trace) { + exportBraintrustAttemptParent(trace); + if (trace.projectId !== projectId || !["pending", "accepted", "failed"].includes(trace.status)) invalid(); + } + return { + input: { execution_id: result.executionId, suite_id: result.suiteId, profile_id: result.profileId, + case_id: result.caseId, provider: result.provider, model: result.model, environment: result.environmentId, + runtime: result.runtimeMode, source_revision: result.source!.sha, definition_hash: result.suiteDefinitionHash }, + output: { product: result.status, cleanup: result.cleanup }, + scores: { product_pass: Number(result.status === "passed"), cleanup_pass: Number(result.cleanup === "passed") }, + metrics: { start: Date.parse(result.startedAt) / 1000, end: Date.parse(result.finishedAt) / 1000 }, + metadata: { campaign_id: result.accountingIdentity!.campaignId, attempt_id: result.accountingIdentity!.attemptId, + attempt_number: result.attempt, wall_duration_ms: result.durationMs, grader: "tests/runner-e2e/runner.spec.ts", + feedback: feedback ? { collection: feedback.collection, invocation_collection: feedback.invocationCollection, + complaints: feedback.complaints, suggestions: feedback.suggestions, + failed_submissions: feedback.invocations.failed, unresolved_submissions: feedback.invocations.unknown } : null, + trace: trace ? { project_id: trace.projectId, root_span_id: trace.traceId, span_id: trace.spanId, + export_status: trace.status, indexed_verification: "unverified" } : null }, + span_attributes: { name: result.executionId, type: "eval" }, + }; +} + +export function buildBraintrustExperiment(projectId: string, campaign: RunnerE2ECampaign, originals: readonly RunnerE2EResult[]) { + if (!uuid.test(projectId) || campaign.schema !== "paperclip.runner-e2e.campaign/v2" || !identifier(campaign.campaignId) + || campaign.accounting?.scope !== "retained_attempts" || !Array.isArray(campaign.accounting.attempts) + || !count(campaign.accounting.missingAttempts) || !Array.isArray(campaign.expected) + || campaign.expected.length === 0 || !campaign.expected.every(identifier) + || new Set(campaign.expected).size !== campaign.expected.length || !Array.isArray(originals) + || originals.length > 10_000 || campaign.accounting.attempts.length === 0) invalid(); + if (campaign.operations) validateEvalOperationsReport(campaign.operations); + if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview); + const accounting = campaign.accounting!; + const operations = campaign.operations?.status !== "unavailable" ? campaign.operations : undefined; + if (operations && operations.ledgerCampaignId !== campaign.campaignId) invalid(); + const candidates = new Map }>(); + for (const original of originals) { + const projection = attemptProjection(original, projectId); + if (!campaign.expected.includes(original.executionId)) invalid(); + const receipt = buildCampaignAccounting({ results: [original] }).attempts[0]!; + const retained = accounting.attempts.find(attempt => attempt.id === receipt.id); + if (!retained || JSON.stringify(retained) !== JSON.stringify(receipt)) invalid(); + const previous = candidates.get(receipt.id); + if (previous && digest(previous.projection) !== digest(projection)) invalid(); + candidates.set(receipt.id, { original, projection }); + } + if (candidates.size !== accounting.attempts.length) invalid(); + for (const selected of campaign.results.filter(result => result.attempt > 0)) { + const id = buildCampaignAccounting({ results: [selected] }).attempts[0]!.id; + const original = candidates.get(id); + const selectedProjection = attemptProjection(selected, projectId); + if (!original || digest(original.projection.input) !== digest(selectedProjection.input) + || digest(original.projection.metrics) !== digest(selectedProjection.metrics) + || digest(original.projection.metadata) !== digest(selectedProjection.metadata)) invalid(); + } + const events = [...candidates.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([id, { original, projection }]) => { + const cost = operations && original.accountingIdentity!.campaignId === operations.ledgerCampaignId + ? operations.attempts.find(row => row.executionId === original.executionId && row.attemptId === original.accountingIdentity!.attemptId) + : undefined; + return { ...projection, id, span_id: id, root_span_id: id, + metadata: { ...projection.metadata, request_ledger: cost ? { + known_usd: cost.knownUsd, additional_held_usd: cost.heldUsd, requests: cost.requests, + unresolved_requests: cost.unresolvedRequests, journal_hash: operations!.journalHash, + } : null } }; + }); + const metadata = { importer: "paperclip-retained-attempts-v1", campaign_id: campaign.campaignId, + expected_execution_ids: [...campaign.expected].sort(), retained_attempts: events.length, + missing_attempts: accounting.missingAttempts, + request_ledger: operations ? { journal_hash: operations.journalHash, known_usd: operations.knownUsd, + additional_held_usd: operations.heldUsd, committed_usd: operations.committedUsd, + requests: operations.requests, unresolved_requests: operations.unresolvedRequests, breakdown: operations.breakdown } : null, + feedback_review: campaign.feedbackReview ?? null, + accounting_note: "Ledger amounts are separate observations, not additive trace metrics. Held reservations are not charges. Requests outside the gateway and infrastructure require separate accounting." }; + const payloadHash = digest({ projectId, metadata, events }); + return { projectId, payloadHash, name: `${campaign.campaignId}-${payloadHash}`, metadata: { ...metadata, payload_hash: payloadHash }, events }; +} + +export interface BraintrustExperimentReceipt { + version: 1; projectId: string; experimentId: string; name: string; payloadHash: string; + status: "created" | "accepted"; acceptedRows: number; expectedRows: number; verification: "unverified"; + evidenceHash: string; projectUrl?: string; experimentUrl?: string; +} + +export function experimentEvidenceHash(campaign: RunnerE2ECampaign) { + return digest({ campaignId: campaign.campaignId, expected: [...campaign.expected].sort(), + selected: campaign.results.map(result => ({ executionId: result.executionId, accountingIdentity: result.accountingIdentity, + attempt: result.attempt, startedAt: result.attempt === 0 ? null : result.startedAt, sourceRevision: result.source?.sha ?? null, + suiteDefinitionHash: result.suiteDefinitionHash ?? null, suiteId: result.suiteId, profileId: result.profileId, + caseId: result.caseId, provider: result.provider, model: result.model, environmentId: result.environmentId, + runtimeMode: result.runtimeMode })).sort((a, b) => digest(a).localeCompare(digest(b))), + accounting: campaign.accounting, operations: campaign.operations ?? null, feedbackReview: campaign.feedbackReview ?? null }); +} + +function safeBraintrustUrl(value: unknown): value is string { + if (typeof value !== "string") return false; + try { + const url = new URL(value); + return url.protocol === "https:" && ["braintrust.dev", "www.braintrust.dev"].includes(url.host) + && !url.username && !url.password && !url.search && !url.hash && url.pathname.startsWith("/app/"); + } catch { return false; } +} + +export function validateBraintrustExperimentReceipt(campaign: RunnerE2ECampaign, receipt: BraintrustExperimentReceipt): void { + if (!receipt || Object.keys(receipt).some(key => !["version", "projectId", "experimentId", "name", "payloadHash", "status", "acceptedRows", "expectedRows", "verification", "evidenceHash", "projectUrl", "experimentUrl"].includes(key)) + || receipt.version !== 1 || !uuid.test(receipt.projectId) || !uuid.test(receipt.experimentId) + || receipt.evidenceHash !== experimentEvidenceHash(campaign) || !/^[a-f0-9]{64}$/.test(receipt.payloadHash) + || receipt.name !== `${campaign.campaignId}-${receipt.payloadHash}` || !["created", "accepted"].includes(receipt.status) + || !count(receipt.acceptedRows) || receipt.expectedRows !== campaign.accounting?.attempts.length + || receipt.acceptedRows > receipt.expectedRows || receipt.verification !== "unverified" + || receipt.status === "accepted" && receipt.acceptedRows !== receipt.expectedRows + || receipt.experimentUrl !== undefined && !safeBraintrustUrl(receipt.experimentUrl) + || receipt.projectUrl !== undefined && !safeBraintrustUrl(receipt.projectUrl)) invalid(); +} + +export function renderBraintrustExperiment(campaign: RunnerE2ECampaign): string { + const receipt = campaign.braintrustExperiment; + if (!receipt) return ""; + validateBraintrustExperimentReceipt(campaign, receipt); + return `

Braintrust experiment

${receipt.experimentUrl + ? `Open retained attempt scores in Braintrust` + : `Experiment ${html(receipt.experimentId)}`}. ${receipt.acceptedRows}/${receipt.expectedRows} rows accepted. + Original product and cleanup grades are separate. Indexing requires independent verification; no agents were rerun for this import.

`; +} + +async function request(config: EvalBraintrustConfig, endpoint: string, payload: unknown, transport: typeof fetch) { + const response = await transport(`https://api.braintrust.dev/v1/${endpoint}`, { + method: payload === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000), + headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" }, + ...(payload === undefined ? {} : { body: JSON.stringify(payload) }), + }); + const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0; + try { + if (!response.ok) throw new Error("rejected"); + while (reader) { + const next = await reader.read(); if (next.done) break; + size += next.value.length; if (size > 1_048_576) throw new Error("oversized"); + chunks.push(next.value); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; + } finally { await reader?.cancel().catch(() => {}); } +} + +export async function importBraintrustExperiment(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign, + originals: readonly RunnerE2EResult[], persist: (receipt: BraintrustExperimentReceipt) => Promise, transport: typeof fetch = fetch, + previous?: BraintrustExperimentReceipt) { + const plan = buildBraintrustExperiment(config.projectId, campaign, originals); + if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust ingestion credential"); + if (previous) { + validateBraintrustExperimentReceipt(campaign, previous); + if (previous.projectId !== plan.projectId || previous.payloadHash !== plan.payloadHash) invalid(); + } + try { + const created = await request(config, "experiment", { project_id: plan.projectId, name: plan.name, + public: false, ensure_new: false, metadata: plan.metadata }, transport); + if (!uuid.test(String(created?.id)) || created.project_id !== plan.projectId || created.name !== plan.name || created.public !== false + || (created.metadata as Record | undefined)?.payload_hash !== plan.payloadHash) throw new Error("conflicting_experiment"); + if (previous && previous.experimentId !== created.id) throw new Error("conflicting_experiment"); + const receipt: BraintrustExperimentReceipt = previous ? { ...previous } : { version: 1, projectId: plan.projectId, experimentId: String(created.id), + name: plan.name, payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign), + status: "created", acceptedRows: 0, expectedRows: plan.events.length, verification: "unverified" }; + if (!previous) await persist({ ...receipt }); + for (let offset = 0; offset < plan.events.length; offset += 100) { + const events = plan.events.slice(offset, offset + 100); + const inserted = await request(config, `experiment/${receipt.experimentId}/insert`, { events }, transport); + if (!Array.isArray(inserted?.row_ids) || JSON.stringify([...inserted.row_ids].sort()) !== JSON.stringify(events.map(event => event.id).sort())) throw new Error("missing_rows"); + receipt.acceptedRows = Math.max(receipt.acceptedRows, offset + events.length); + if (receipt.acceptedRows === receipt.expectedRows) receipt.status = "accepted"; + await persist({ ...receipt }); + } + const summary = await request(config, `experiment/${receipt.experimentId}/summarize`, undefined, transport).catch(() => undefined); + if (summary?.experiment_name === plan.name && safeBraintrustUrl(summary.project_url) && safeBraintrustUrl(summary.experiment_url)) { + receipt.projectUrl = summary.project_url; receipt.experimentUrl = summary.experiment_url; + await persist({ ...receipt }); + } + return receipt; + } catch { throw new Error("Braintrust experiment import incomplete; retain the receipt and retry identical evidence. Remote details withheld."); } +} diff --git a/tests/runner-e2e/braintrust-indexing.test.ts b/tests/runner-e2e/braintrust-indexing.test.ts new file mode 100644 index 00000000000..65087433ba7 --- /dev/null +++ b/tests/runner-e2e/braintrust-indexing.test.ts @@ -0,0 +1,113 @@ +import { expect, it, vi } from "vitest"; +import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js"; +import { braintrustTraceIndexQuery, verifyBraintrustIndexing } from "./braintrust-indexing.js"; +import { buildRunnerCampaign } from "./history.js"; +import { runnerMatrix } from "./catalog.js"; +import { emptyFeedbackSummary } from "./feedback.js"; +import type { RunnerE2EResult } from "./types.js"; + +const projectId = "11111111-1111-4111-8111-111111111111"; +const config = { projectId, apiKey: "private-key-canary" }; +const aggregate = { records: 2500, roots: 1, finalizations: 1, native_api_records: 2, complaints: 1, suggestions: 1 }; + +function fixture(profile = "runner-codex") { + const cell = runnerMatrix.find(row => row.id === `feedback-persistence.${profile}.local.complaint-and-suggestion`)!; + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, + suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash, + accountingIdentity: { campaignId: "indexed-pilot", attemptId: "first" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null }, + attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id, + environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode, + startedAt: "2026-10-20T00:00:00.000Z", finishedAt: "2026-10-20T00:00:01.000Z", durationMs: 1000, + feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1 }, + tracing: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "accepted", verification: "unverified", nativeConfiguration: "configured", + finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "passed", verification: "unverified" } } }; + const originals = [result]; + const campaign = buildRunnerCampaign({ campaignId: "indexed-pilot", generatedAt: result.finishedAt, expected: [cell.id], results: originals, attempts: originals }); + const plan = buildBraintrustExperiment(projectId, campaign, originals); + const experiment: BraintrustExperimentReceipt = { version: 1, projectId, experimentId: "22222222-2222-4222-8222-222222222222", name: plan.name, + payloadHash: plan.payloadHash, evidenceHash: experimentEvidenceHash(campaign), status: "accepted", acceptedRows: 1, expectedRows: 1, verification: "unverified" }; + return { originals, campaign, plan, experiment }; +} + +it.each(["runner-codex", "runner-acpx-claude"])("verifies %s native records and original experiment grades without fetching content", async profile => { + const { originals, campaign, plan, experiment } = fixture(profile); + const transport = vi.fn().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: plan.events })); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport); + expect(receipt.status).toBe("verified"); + expect(receipt.traces[0]).toMatchObject({ records: 2500, nativeApiRecords: 2, complaints: 1 }); + expect(receipt.experiment.indexedRows).toBe(1); + expect(transport).toHaveBeenCalledTimes(2); + const sql = JSON.parse(String(transport.mock.calls[0]![1]!.body)).query; + expect(sql).toContain(`root_span_id = '${"a".repeat(32)}'`); + expect(sql).toContain(profile === "runner-codex" ? "codex.api_request" : "claude_code.llm_request"); + expect(sql).not.toMatch(/arguments|prompt|response|tool_input|tool_result_size/); + expect(JSON.stringify(receipt)).not.toContain(config.apiKey); +}); + +it.each(["roots", "finalizations", "native_api_records", "complaints", "suggestions"])("does not pass a trace missing %s", async missing => { + const { originals, campaign, plan, experiment } = fixture(); + const transport = vi.fn().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, [missing]: 0 }] })).mockResolvedValueOnce(Response.json({ data: plan.events })); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 }); + expect(receipt.status).toBe("incomplete"); + expect(receipt.experiment.status).toBe("verified"); +}); + +it("retries delayed indexing without re-querying already verified records", async () => { + const { originals, campaign, plan, experiment } = fixture(); + const transport = vi.fn().mockResolvedValueOnce(Response.json({ data: [{ ...aggregate, roots: 0 }] })) + .mockResolvedValueOnce(Response.json({ data: plan.events })).mockResolvedValueOnce(Response.json({ data: [aggregate] })); + const pause = vi.fn(async () => {}); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { pause }); + expect(receipt.status).toBe("verified"); + expect(transport).toHaveBeenCalledTimes(3); + expect(pause).toHaveBeenCalledTimes(1); +}); + +it("treats an empty aggregate as missing indexing, without inventing native activity", async () => { + const { originals, campaign, plan, experiment } = fixture(); + const transport = vi.fn().mockResolvedValueOnce(Response.json({ data: [{ records: 0, roots: null, + finalizations: null, native_api_records: null, complaints: null, suggestions: null }] })).mockResolvedValueOnce(Response.json({ data: plan.events })); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 }); + expect(receipt.status).toBe("incomplete"); + expect(receipt.traces[0]).toMatchObject({ status: "incomplete", records: 0, nativeApiRecords: 0 }); +}); + +it.each(["missing", "extra", "duplicate", "grade-changed", "wrong-revision", "wrong-attempt"])("rejects %s indexed experiment evidence", async fault => { + const { originals, campaign, plan, experiment } = fixture(); + const rows = structuredClone(plan.events); + if (fault === "missing") rows.pop(); + if (fault === "extra") rows.push({ ...rows[0]!, id: "other" }); + if (fault === "duplicate") rows.push(rows[0]!); + if (fault === "grade-changed") rows[0]!.scores.product_pass = 0; + if (fault === "wrong-revision") rows[0]!.input.source_revision = "b".repeat(40); + if (fault === "wrong-attempt") rows[0]!.metadata.attempt_id = "other"; + const transport = vi.fn().mockResolvedValueOnce(Response.json({ data: [aggregate] })).mockResolvedValueOnce(Response.json({ data: rows })); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 }); + expect(receipt.status).toBe("incomplete"); + expect(receipt.traces[0]!.status).toBe("verified"); +}); + +it.each(["rejected", "malformed", "oversized", "invalid-counts"])("retains %s query failures without raw error content", async fault => { + const { originals, campaign, experiment } = fixture(); + const transport = vi.fn().mockImplementation(async () => fault === "rejected" ? new Response("private-response-canary", { status: 401 }) + : fault === "malformed" ? new Response("private-response-canary") + : fault === "oversized" ? new Response("private-response-canary".repeat(100_000)) + : Response.json({ data: [{ ...aggregate, records: -1 }] })); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { rounds: 1 }); + expect(receipt.status).toBe("unavailable"); + expect(JSON.stringify(receipt)).not.toMatch(/private-response-canary|private-key-canary/); +}); + +it("honors cancellation before making a query", async () => { + const { originals, campaign, experiment } = fixture(); + const transport = vi.fn(); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment, transport, { signal: AbortSignal.abort() }); + expect(receipt.status).toBe("unavailable"); + expect(transport).not.toHaveBeenCalled(); +}); + +it("rejects foreign or malformed trace identities before constructing a query", () => { + const { originals } = fixture(); + originals[0]!.tracing!.traceId = "injected'"; + expect(() => braintrustTraceIndexQuery(projectId, originals[0]!)).toThrow(); +}); diff --git a/tests/runner-e2e/braintrust-indexing.ts b/tests/runner-e2e/braintrust-indexing.ts new file mode 100644 index 00000000000..8ca5d9f11b4 --- /dev/null +++ b/tests/runner-e2e/braintrust-indexing.ts @@ -0,0 +1,118 @@ +import { setTimeout as delay } from "node:timers/promises"; +import { buildBraintrustExperiment, experimentEvidenceHash, type BraintrustExperimentReceipt } from "./braintrust-experiment.js"; +import { exportBraintrustAttemptParent, type EvalBraintrustConfig } from "./braintrust.js"; +import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js"; + +type IndexStatus = "verified" | "incomplete" | "unavailable"; +export interface BraintrustIndexReceipt { + version: 1; projectId: string; evidenceHash: string; checkedAt: string; + status: IndexStatus; + traces: Array<{ attemptId: string; traceId: string | null; status: IndexStatus; + records?: number; roots?: number; finalizations?: number; nativeApiRecords?: number; + complaints?: number; suggestions?: number }>; + experiment: { id: string | null; status: IndexStatus; expectedRows: number; indexedRows?: number }; +} + +async function query(config: EvalBraintrustConfig, sql: string, transport: typeof fetch, signal: AbortSignal): Promise[]> { + const response = await transport("https://api.braintrust.dev/btql", { + method: "POST", redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(10_000)]), + headers: { authorization: `Bearer ${config.apiKey}`, "content-type": "application/json" }, + body: JSON.stringify({ query: sql, fmt: "json" }), + }); + const reader = response.body?.getReader(), chunks: Uint8Array[] = []; let size = 0; + try { + if (!response.ok) throw new Error("query_rejected"); + while (reader) { + const next = await reader.read(); if (next.done) break; + size += next.value.length; if (size > 1_048_576) throw new Error("query_oversized"); + chunks.push(next.value); + } + const payload = JSON.parse(Buffer.concat(chunks).toString("utf8")); + if (!payload || !Array.isArray(payload.data) || payload.data.some((row: unknown) => !row || typeof row !== "object" || Array.isArray(row))) throw new Error("invalid_query_result"); + return payload.data; + } finally { await reader?.cancel().catch(() => {}); } +} + +export function braintrustTraceIndexQuery(projectId: string, result: RunnerE2EResult): string { + const trace = result.tracing; + if (!trace || trace.projectId !== projectId || !/^[0-9a-f]{16}$/.test(trace.finalization?.spanId ?? "")) throw new Error("Missing trace identity"); + exportBraintrustAttemptParent(trace); + if (!(result.profileId === "runner-codex" && result.provider === "codex") + && !(result.profileId === "runner-acpx-claude" && result.provider === "acpx")) throw new Error("Unsupported native index contract"); + const nativeRequest = result.profileId === "runner-codex" ? `metadata."event.name" = 'codex.api_request'` + : `span_attributes.name = 'claude_code.llm_request' AND span_attributes.type = 'llm'`; + const tool = (name: string) => result.profileId === "runner-codex" + ? `metadata."event.name" = 'codex.tool_result' AND metadata.tool_name = '${name}' AND metadata.success = 'true'` + : `span_attributes.type = 'tool' AND span_attributes.name = 'mcp__paperclip__${name}'`; + return `SELECT COUNT(*) AS records, + SUM(CASE WHEN span_id = '${trace.spanId}' AND span_attributes.name = 'paperclip.eval.attempt' THEN 1 ELSE 0 END) AS roots, + SUM(CASE WHEN span_id = '${trace.finalization!.spanId}' AND span_attributes.name = 'paperclip.eval.finalization' THEN 1 ELSE 0 END) AS finalizations, + SUM(CASE WHEN ${nativeRequest} THEN 1 ELSE 0 END) AS native_api_records, + SUM(CASE WHEN ${tool("submit_complaint")} THEN 1 ELSE 0 END) AS complaints, + SUM(CASE WHEN ${tool("submit_suggestion")} THEN 1 ELSE 0 END) AS suggestions + FROM project_logs('${projectId}') WHERE root_span_id = '${trace.traceId}'`; +} + +function counts(rows: Record[]) { + if (rows.length !== 1) throw new Error("Invalid trace aggregate"); + const row = rows[0]!; + const fields = ["records", "roots", "finalizations", "native_api_records", "complaints", "suggestions"]; + if (row.records === 0) for (const key of fields.slice(1)) if (row[key] === null) row[key] = 0; + if (fields.some(key => !Number.isSafeInteger(row[key]) || Number(row[key]) < 0)) throw new Error("Invalid trace counts"); + return { records: Number(row.records), roots: Number(row.roots), finalizations: Number(row.finalizations), + nativeApiRecords: Number(row.native_api_records), complaints: Number(row.complaints), suggestions: Number(row.suggestions) }; +} + +export async function verifyBraintrustIndexing(config: EvalBraintrustConfig, campaign: RunnerE2ECampaign, + originals: readonly RunnerE2EResult[], experiment: BraintrustExperimentReceipt | undefined, + transport: typeof fetch = fetch, options: { rounds?: number; pause?: () => Promise; signal?: AbortSignal } = {}): Promise { + const plan = buildBraintrustExperiment(config.projectId, campaign, originals); + const receipt: BraintrustIndexReceipt = { version: 1, projectId: config.projectId, + evidenceHash: experimentEvidenceHash(campaign), checkedAt: new Date().toISOString(), status: "incomplete", + traces: originals.map(result => ({ attemptId: result.accountingIdentity!.attemptId, + traceId: result.tracing?.traceId ?? null, status: "incomplete" })), + experiment: { id: experiment?.experimentId ?? null, status: "incomplete", expectedRows: plan.events.length } }; + if (!config.apiKey?.trim() || /[\r\n]/.test(config.apiKey)) throw new Error("Missing Braintrust query credential"); + const rounds = options.rounds ?? 3; + if (!Number.isInteger(rounds) || rounds < 1 || rounds > 3) throw new Error("Invalid index verification limit"); + const signal = AbortSignal.any([AbortSignal.timeout(45_000), ...(options.signal ? [options.signal] : [])]); + const pause = options.pause ?? (() => delay(2_000, undefined, { signal })); + for (let round = 0; round < rounds && !signal.aborted; round++) { + for (let index = 0; index < originals.length && !signal.aborted; index++) { + const result = originals[index]!, row = receipt.traces[index]!; + if (row.status === "verified") continue; + try { + const values = counts(await query(config, braintrustTraceIndexQuery(config.projectId, result), transport, signal)); + Object.assign(row, values); + row.status = values.roots > 0 && values.finalizations > 0 && values.nativeApiRecords > 0 + && (result.feedback?.complaints ?? 0) <= values.complaints + && (result.feedback?.suggestions ?? 0) <= values.suggestions ? "verified" : "incomplete"; + } catch { row.status = "unavailable"; } + } + if (!signal.aborted && receipt.experiment.status !== "verified" && experiment?.status === "accepted" + && experiment.projectId === config.projectId && /^[0-9a-f-]{36}$/.test(experiment.experimentId) + && experiment.payloadHash === plan.payloadHash && experiment.evidenceHash === receipt.evidenceHash) { + try { + const rows = await query(config, `SELECT id, input, output, scores, metadata FROM experiment('${experiment.experimentId}') LIMIT ${plan.events.length + 1}`, transport, signal); + receipt.experiment.indexedRows = rows.length; + const equal = (left: unknown, right: unknown): boolean => { + if (left === right) return true; + if (!left || !right || typeof left !== "object" || typeof right !== "object" || Array.isArray(left) || Array.isArray(right)) return false; + const a = left as Record, b = right as Record; + return Object.keys(a).length === Object.keys(b).length && Object.keys(a).every(key => equal(a[key], b[key])); + }; + receipt.experiment.status = rows.length === plan.events.length && new Set(rows.map(row => row.id)).size === rows.length + && plan.events.every(expected => rows.some(row => row.id === expected.id && equal(row.input, expected.input) + && equal(row.output, expected.output) && equal(row.scores, expected.scores) + && (row.metadata as Record | undefined)?.attempt_id === expected.metadata.attempt_id)) ? "verified" : "incomplete"; + } catch { receipt.experiment.status = "unavailable"; } + } + if (receipt.experiment.status === "verified" && receipt.traces.every(row => row.status === "verified")) break; + if (round + 1 < rounds && !signal.aborted) { try { await pause(); } catch { break; } } + } + receipt.checkedAt = new Date().toISOString(); + const statuses = [...receipt.traces.map(row => row.status), receipt.experiment.status]; + receipt.status = statuses.every(status => status === "verified") ? "verified" + : statuses.some(status => status === "unavailable") || signal.aborted ? "unavailable" : "incomplete"; + return receipt; +} diff --git a/tests/runner-e2e/braintrust.test.ts b/tests/runner-e2e/braintrust.test.ts new file mode 100644 index 00000000000..118621d3ffe --- /dev/null +++ b/tests/runner-e2e/braintrust.test.ts @@ -0,0 +1,213 @@ +import { readFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { afterEach, expect, it, vi } from "vitest"; +import { EvalAttemptTrace, evalBraintrustConfig, exportBraintrustAttemptParent, finalizeEvalAttemptTrace } from "./braintrust.js"; +import { RunnerApi } from "./api.js"; +import { buildPaperclipServerEnvironment, buildRunnerE2EProcessEnvironment } from "./harness-env.js"; +import { setupLiveFixtures } from "./live-fixtures.js"; +import { runnerMatrix } from "./catalog.js"; +import { buildProviderObservabilityLaunch } from "../../packages/adapter-utils/src/agent-observability.js"; +const require = createRequire(new URL("../../server/package.json", import.meta.url)); +const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js"); +const projectId = "11111111-1111-4111-8111-111111111111", companyId = "22222222-2222-4222-8222-222222222222"; +const config = { projectId, apiKey: "private-ingestion-fixture" }; +const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "hello.codex.local", provider: "codex", model: "fixture-model", environment: "local" }; +const provenance = { suiteId: "core-compatibility", suiteDefinitionHash: "a".repeat(64), profileId: "runner-codex", caseId: "message-marker", sourceRevision: "b".repeat(40) }; +const input = { failed: false, runIds: ["run-1"], cleanup: "passed", complaints: 1, suggestions: 2 }; +afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); + +it("requires explicit export routing and keeps ingestion credentials out of server inheritance", () => { + expect(evalBraintrustConfig({ BRAINTRUST_API_KEY: config.apiKey })).toBeUndefined(); + expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId })).toThrow("API key"); + expect(() => evalBraintrustConfig({ PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "bad", BRAINTRUST_API_KEY: config.apiKey })).toThrow("UUID"); + expect(buildPaperclipServerEnvironment({ BRAINTRUST_API_KEY: config.apiKey, BRAINTRUST_OTHER_KEY: "private", KEEP: "yes" })).toEqual({ KEEP: "yes" }); +}); + +it("accepts only explicit loopback relay destinations and strips inherited relay endpoints", async () => { + const endpointBaseUrl = "http://127.0.0.1:43123/otel"; + const env = { PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: projectId, BRAINTRUST_API_KEY: "attempt-token", PAPERCLIP_E2E_OTLP_RELAY_URL: endpointBaseUrl }; + expect(evalBraintrustConfig(env)).toEqual({ projectId, apiKey: "attempt-token", endpointBaseUrl }); + expect(buildPaperclipServerEnvironment(env).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined(); + expect(buildRunnerE2EProcessEnvironment(env, []).PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined(); + for (const invalid of ["https://foreign.invalid/otel", "http://127.0.0.1:43123/otel?key=secret", "http://localhost:43123/otel", "http://127.0.0.1:99999/otel"]) { + expect(() => evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: invalid })).toThrow(); + } + const trace = new EvalAttemptTrace(evalBraintrustConfig(env)!, identity); + const putSensitive = vi.fn(async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId, + destination: { provider: "otlp", endpointBaseUrl } } })); + await trace.configureCompany({ putSensitive } as unknown as RunnerApi, companyId); + expect(putSensitive).toHaveBeenCalledWith(`/api/companies/${companyId}/observability`, expect.objectContaining({ + destination: { provider: "otlp", endpointBaseUrl }, credentials: { mode: "replace", headers: [ + { name: "authorization", value: "Bearer attempt-token" }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) }, + ] }, + })); +}); + +it("exports the SDK-verified project-log parent format and refuses invalid identifiers", () => { + const receipt = { projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" }; + expect(exportBraintrustAttemptParent(receipt)).toBe("BAIAeyJvYmplY3RfaWQiOiIxMTExMTExMS0xMTExLTQxMTEtODExMS0xMTExMTExMTExMTEiLCJyb3dfaWQiOiIxMjM0NTY3ODkwYWJjZGVmIiwic3Bhbl9pZCI6IjEyMzQ1Njc4OTBhYmNkZWYiLCJyb290X3NwYW5faWQiOiJhYmNkZWYwMTIzNDU2Nzg5YWJjZGVmMDEyMzQ1Njc4OSJ9"); + for (const invalid of [{ projectId: "project" }, { spanId: "0".repeat(16) }, { traceId: "0".repeat(32) }, { spanId: "1234567890abcdeg" }]) { + expect(() => exportBraintrustAttemptParent({ ...receipt, ...invalid })).toThrow("parent identity"); + } +}); + +it.each(["codex", "claude"] as const)("preserves the attempt parent through the production %s launch encoder", kind => { + const trace = new EvalAttemptTrace(config, identity); + const parent = exportBraintrustAttemptParent(trace.snapshot()); + const launch = buildProviderObservabilityLaunch({ + kind, companyId, agentId: projectId, revision: projectId, compatibilityKey: "fixture", enabled: true, + capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, + tracesEndpoint: "https://api.braintrust.dev/otel/v1/traces", logsEndpoint: "https://api.braintrust.dev/otel/v1/logs", + headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": parent }, + }); + if (kind === "codex") { + const exporter = launch.otelConfig!.trace_exporter as { "otlp-http": { headers: Record } }; + expect(exporter["otlp-http"].headers).toEqual({}); + expect(JSON.stringify(launch.otelConfig)).not.toContain(config.apiKey); + } + for (const name of ["OTEL_EXPORTER_OTLP_TRACES_HEADERS", "OTEL_EXPORTER_OTLP_LOGS_HEADERS"]) { + const encoded = launch.environment[name].split(",").find(header => header.startsWith("x-bt-parent="))!; + expect(decodeURIComponent(encoded.slice("x-bt-parent=".length))).toBe(parent); + } + expect(JSON.stringify(trace.snapshot())).not.toContain(config.apiKey); +}); + +it("rejects invalid provenance while retaining an explicitly unknown source revision", () => { + expect(() => new EvalAttemptTrace(config, identity, { ...provenance, suiteDefinitionHash: "unknown" })).toThrow("provenance"); + expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: "master" })).toThrow("provenance"); + expect(() => new EvalAttemptTrace(config, identity, { ...provenance, sourceRevision: null })).not.toThrow(); +}); + +it("sends real protobuf with parented phases, exact project routing, and no duplicate model charges", async () => { + const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root; + const trace = new EvalAttemptTrace(config, identity, provenance); + trace.phase("setup"); trace.phase("execution"); trace.phase("feedback"); trace.phase("cleanup"); + let captured: RequestInit | undefined; + const transport = vi.fn(async (_url, init) => { captured = init; return new Response(null, { status: 200 }); }); + expect(await trace.export(input, transport)).toMatchObject({ status: "accepted", verification: "unverified", nativeConfiguration: "not_started" }); + expect(captured).toMatchObject({ method: "POST", redirect: "error", headers: { authorization: `Bearer ${config.apiKey}`, "x-bt-parent": `project_id:${projectId}`, "content-type": "application/x-protobuf" } }); + const bytes = captured!.body as Buffer; + expect(bytes.includes(config.apiKey)).toBe(false); + const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(bytes).toJSON(); + const spans = decoded.resourceSpans[0].scopeSpans[0].spans; + expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.setup", "paperclip.eval.execution", "paperclip.eval.feedback", "paperclip.eval.cleanup"]); + expect(spans[0].parentSpanId).toBeUndefined(); + for (const span of spans.slice(1)) { + expect(span.traceId).toBe(spans[0].traceId); + expect(span.parentSpanId).toBe(spans[0].spanId); + } + expect(JSON.stringify(decoded)).toContain("paperclip.eval.attempt_id"); + const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue])); + expect(attributes).toMatchObject({ + "paperclip.eval.suite_id": provenance.suiteId, "paperclip.eval.suite_definition_hash": provenance.suiteDefinitionHash, + "paperclip.eval.profile_id": provenance.profileId, "paperclip.eval.case_id": provenance.caseId, + "paperclip.eval.source_revision": provenance.sourceRevision, "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts", + }); + expect(JSON.stringify(decoded)).not.toMatch(/gen_ai\.usage|cost_usd|private-ingestion/); + expect(transport).toHaveBeenCalledTimes(1); +}); + +it.each([ + [() => new Response("private-provider-error", { status: 403 }), "receiver_rejected"], + [() => Response.json({ partialSuccess: { rejectedSpans: "1" } }), "partial_rejection"], + [() => Response.json({ partialSuccess: { rejectedSpans: -1 } }), "export_failed"], + [() => new Response("invalid", { status: 200, headers: { "content-type": "text/html" } }), "export_failed"], + [() => new Response(Buffer.alloc(65_537)), "export_failed"], +])("retains export failure without raw receiver diagnostics", async (response, errorCode) => { + const trace = new EvalAttemptTrace(config, identity); + expect(await trace.export(input, async () => response())).toMatchObject({ status: "failed", verification: "unverified", errorCode }); + expect(JSON.stringify(trace.snapshot())).not.toMatch(/private-provider-error|private-ingestion/); +}); + +it("configures native export through a revision-fenced sensitive PUT before any credential or agent setup", async () => { + const order: string[] = []; + const trace = new EvalAttemptTrace(config, identity); + const api = { + async get() { return [{ id: "local", driver: "local" }]; }, + async putSensitive(path: string, data: unknown) { + order.push("observability"); + expect(path).toBe(`/api/companies/${companyId}/observability`); + expect(data).toMatchObject({ expectedRevision: null, enabled: true, + credentials: { mode: "replace", headers: [{ name: "authorization", value: `Bearer ${config.apiKey}` }, { name: "x-bt-parent", value: exportBraintrustAttemptParent(trace.snapshot()) }] }, + destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } }); + return { settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination: { provider: "otlp", endpointBaseUrl: "https://api.braintrust.dev/otel" } } }; + }, + async postSensitive() { order.push("secret"); return { id: "secret" }; }, + async post(path: string) { order.push(path === "/api/companies" ? "company" : "agent"); return { id: companyId, name: "fixture" }; }, + } as unknown as RunnerApi; + const execution = runnerMatrix.find(cell => cell.profile.id === "runner-codex" && cell.environment.id === "local" && cell.task.flow === "single_turn")!; + expect(execution).toBeDefined(); + const fixtures = await setupLiveFixtures({ api, execution, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { OPENAI_API_KEY: "private-provider" }, onCompanyCreated: id => trace.configureCompany(api, id) }); + await fixtures.teardown(); + expect(order.slice(0, 3)).toEqual(["company", "observability", "secret"]); + expect(trace.snapshot()).toMatchObject({ nativeConfiguration: "configured", companyId, revision: projectId }); +}); + +it.each([ + { provider: "braintrust", projectId }, + { provider: "otlp", endpointBaseUrl: "https://elsewhere.invalid/otel" }, +])("refuses a saved native destination that cannot carry the attempt parent", async destination => { + const trace = new EvalAttemptTrace(config, identity); + const api = { putSensitive: async () => ({ settings: { enabled: true, credentialsConfigured: true, revision: projectId, destination } }) } as unknown as RunnerApi; + await expect(trace.configureCompany(api, companyId)).rejects.toThrow("configuration failed"); + expect(trace.snapshot().nativeConfiguration).toBe("failed"); +}); + +it("never sends sensitive PUT bodies through Playwright and refuses redirects or foreign destinations", async () => { + vi.stubEnv("PAPERCLIP_RUNNER_E2E_PORT", "43123"); + const playwright = { put: vi.fn() }; + const api = new RunnerApi(playwright as never); + api.setBrowserSession("session=private"); + const transport = vi.fn().mockResolvedValue(Response.json({ ok: true })); vi.stubGlobal("fetch", transport); + await api.putSensitive("/api/companies/fixture/observability", { apiKey: config.apiKey }); + expect(playwright.put).not.toHaveBeenCalled(); + expect(transport).toHaveBeenCalledWith(new URL("http://127.0.0.1:43123/api/companies/fixture/observability"), expect.objectContaining({ method: "PUT", redirect: "error", signal: expect.any(AbortSignal), headers: expect.objectContaining({ Cookie: "session=private" }) })); + await expect(api.putSensitive("https://other.invalid/api/collect", { apiKey: config.apiKey })).rejects.toThrow("fixture API"); + transport.mockRejectedValue(new Error(config.apiKey)); + await expect(api.putSensitive("/api/fixture", {})).rejects.toThrow("details withheld"); +}); + +it.each([ + { status: "failed", cleanup: "passed", failureClass: "permanent_infrastructure" }, + { status: "failed", cleanup: "failed", failureClass: "cleanup_failure" }, +] as const)("updates a passing worker trace with the supervisor's $failureClass outcome", async outcome => { + const root = protobuf.parse(await readFile(new URL("../../server/src/__tests__/fixtures/otlp-observability.proto", import.meta.url), "utf8")).root; + const trace = new EvalAttemptTrace(config, identity, provenance); + const original = await trace.export(input, async () => new Response(null, { status: 200 })); + const saved = structuredClone(original); + let captured: RequestInit | undefined; + const finalization = await finalizeEvalAttemptTrace(config, { receipt: original, identity, provenance, + result: { startedAt: new Date(Date.now() - 10_000).toISOString(), ...outcome, runIds: [companyId] }, + }, async (_url, init) => { captured = init; return new Response(null, { status: 200 }); }); + expect(finalization).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" }); + expect(original).toEqual(saved); + const decoded = root.lookupType("paperclip.test.otlp.ExportTraceServiceRequest").decode(captured!.body as Buffer).toJSON(); + const spans = decoded.resourceSpans[0].scopeSpans[0].spans; + expect(spans.map((span: { name: string }) => span.name)).toEqual(["paperclip.eval.attempt", "paperclip.eval.finalization"]); + expect(Buffer.from(spans[0].traceId, "base64").toString("hex")).toBe(original.traceId); + expect(Buffer.from(spans[0].spanId, "base64").toString("hex")).toBe(original.spanId); + expect(spans[1].parentSpanId).toBe(spans[0].spanId); + expect(spans.every((span: { status: { code: number } }) => span.status.code === 2)).toBe(true); + const attributes = Object.fromEntries(spans[0].attributes.map((attribute: { key: string; value: { stringValue: string } }) => [attribute.key, attribute.value.stringValue])); + expect(attributes).toMatchObject({ "paperclip.eval.outcome_stage": "supervisor", "paperclip.eval.outcome": "failed", + "paperclip.eval.failure_class": outcome.failureClass, "paperclip.eval.source_revision": provenance.sourceRevision }); + expect(JSON.stringify(decoded)).not.toMatch(/private-ingestion|gen_ai\.usage|cost_usd/); +}); + +it("retains finalization export failures without changing the recorded product result", async () => { + const trace = new EvalAttemptTrace(config, identity, provenance); + const result = { startedAt: new Date().toISOString(), status: "passed" as const, cleanup: "passed" as const }; + const saved = structuredClone(result); + const receipt = await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance, result }, async () => new Response("private-error", { status: 503 })); + expect(receipt).toMatchObject({ status: "failed", outcome: "passed", errorCode: "receiver_rejected", httpStatus: 503 }); + expect(result).toEqual(saved); + expect(JSON.stringify(receipt)).not.toMatch(/private-error|private-ingestion/); + const transport = vi.fn(); + expect(await finalizeEvalAttemptTrace(config, { receipt: { ...trace.snapshot(), projectId: companyId }, identity, provenance, result }, transport)) + .toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" }); + expect(transport).not.toHaveBeenCalled(); + expect(await finalizeEvalAttemptTrace(config, { receipt: trace.snapshot(), identity, provenance, + result: { ...result, failureClass: "private-target-output" as never }, + }, transport)).toMatchObject({ status: "failed", errorCode: "invalid_finalization_context" }); + expect(transport).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/braintrust.ts b/tests/runner-e2e/braintrust.ts new file mode 100644 index 00000000000..adb5d94e4b9 --- /dev/null +++ b/tests/runner-e2e/braintrust.ts @@ -0,0 +1,218 @@ +import { randomBytes } from "node:crypto"; +import { createRequire } from "node:module"; +import type { RunnerApi } from "./api.js"; +import type { FailureClass, RunnerE2EResult } from "./types.js"; + +export const EVAL_OBSERVABILITY_SECRET_NAMES = ["BRAINTRUST_API_KEY"] as const; +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +export interface EvalBraintrustConfig { projectId: string; apiKey: string; endpointBaseUrl?: string } +function evalOtlpEndpoint(config: EvalBraintrustConfig) { + if (config.endpointBaseUrl !== undefined && !/^http:\/\/127\.0\.0\.1:[1-9][0-9]{0,4}\/otel$/.test(config.endpointBaseUrl)) { + throw new Error("Eval OTLP relay must use the parent-owned loopback endpoint"); + } + if (config.endpointBaseUrl && Number(new URL(config.endpointBaseUrl).port) > 65535) throw new Error("Invalid eval OTLP relay port"); + return config.endpointBaseUrl ?? "https://api.braintrust.dev/otel"; +} +export function evalBraintrustConfig(env: NodeJS.ProcessEnv): EvalBraintrustConfig | undefined { + const projectId = env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID; + if (!projectId) return undefined; + if (!uuid.test(projectId) || !env.BRAINTRUST_API_KEY?.trim() || /[\r\n]/.test(env.BRAINTRUST_API_KEY)) { + throw new Error("Eval Braintrust export requires an explicit project UUID and API key"); + } + const config = { projectId, apiKey: env.BRAINTRUST_API_KEY.trim(), + ...(env.PAPERCLIP_E2E_OTLP_RELAY_URL ? { endpointBaseUrl: env.PAPERCLIP_E2E_OTLP_RELAY_URL } : {}) }; + evalOtlpEndpoint(config); + return config; +} + +type Attributes = Record; +type Phase = "setup" | "execution" | "feedback" | "cleanup" | "finalization"; +interface PhaseSpan { name: Phase; spanId: string; start: number; end?: number; failed?: boolean } +export interface EvalTraceIdentity { campaignId: string; attemptId: string; executionId: string; provider: string; model: string; environment: string } +export interface EvalTraceProvenance { + suiteId: string; suiteDefinitionHash: string; profileId: string; caseId: string; sourceRevision: string | null; +} +export interface EvalTraceReceipt { + projectId: string; traceId: string; spanId: string; + status: "pending" | "accepted" | "failed"; + verification: "unverified"; + nativeConfiguration: "not_started" | "configured" | "failed"; + companyId?: string; revision?: string; errorCode?: string; httpStatus?: number; + finalization?: EvalFinalizationReceipt; +} +export interface EvalFinalizationReceipt { + spanId: string; status: "accepted" | "failed"; outcome: "passed" | "failed"; verification: "unverified"; + errorCode?: string; httpStatus?: number; +} +const require = createRequire(new URL("../../server/package.json", import.meta.url)); +const time = (ms: number): [number, number] => [Math.floor(ms / 1000), (ms % 1000) * 1_000_000]; + +export function exportBraintrustAttemptParent(receipt: Pick): string { + if (!uuid.test(receipt.projectId) || !/^[0-9a-f]{16}$/.test(receipt.spanId) || !/^[0-9a-f]{32}$/.test(receipt.traceId) + || /^0+$/.test(receipt.spanId) || /^0+$/.test(receipt.traceId)) throw new Error("Invalid eval trace parent identity"); + return Buffer.concat([Buffer.from([4, 2, 0]), Buffer.from(JSON.stringify({ + object_id: receipt.projectId, row_id: receipt.spanId, span_id: receipt.spanId, root_span_id: receipt.traceId, + }))]).toString("base64"); +} + +export class EvalAttemptTrace { + private readonly startedAt = Date.now(); + private readonly phases: PhaseSpan[] = []; + private active?: PhaseSpan; + private readonly receipt: EvalTraceReceipt; + private readonly endpointBaseUrl: string; + constructor(private readonly config: EvalBraintrustConfig, private readonly identity: EvalTraceIdentity, private readonly provenance?: EvalTraceProvenance, + context?: { receipt: EvalTraceReceipt; startedAtMs: number }) { + this.endpointBaseUrl = evalOtlpEndpoint(config); + if (Object.values(identity).some(value => typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value))) throw new Error("Invalid eval trace identity"); + if (provenance && (Object.values(provenance).some(value => value !== null && (typeof value !== "string" || !value || value.length > 512 || /[\r\n]/.test(value))) + || !/^[0-9a-f]{64}$/.test(provenance.suiteDefinitionHash) + || provenance.sourceRevision !== null && !/^[0-9a-f]{40}$/.test(provenance.sourceRevision))) throw new Error("Invalid eval trace provenance"); + this.receipt = { projectId: config.projectId, traceId: randomBytes(16).toString("hex"), spanId: randomBytes(8).toString("hex"), + status: "pending", verification: "unverified", nativeConfiguration: "not_started" }; + if (context) { + const { receipt, startedAtMs } = context; + exportBraintrustAttemptParent(receipt); + if (receipt.projectId !== config.projectId || !Number.isFinite(startedAtMs) || startedAtMs < 0 || startedAtMs > Date.now() + || !["not_started", "configured", "failed"].includes(receipt.nativeConfiguration) + || receipt.companyId !== undefined && !uuid.test(receipt.companyId) + || receipt.revision !== undefined && !uuid.test(receipt.revision)) throw new Error("Invalid eval finalization context"); + this.startedAt = startedAtMs; + Object.assign(this.receipt, { traceId: receipt.traceId, spanId: receipt.spanId, nativeConfiguration: receipt.nativeConfiguration, + ...(receipt.companyId ? { companyId: receipt.companyId } : {}), ...(receipt.revision ? { revision: receipt.revision } : {}) }); + } + } + snapshot(): EvalTraceReceipt { return { ...this.receipt }; } + phase(name: Phase, previousFailed = false) { + this.endPhase(previousFailed); + this.active = { name, spanId: randomBytes(8).toString("hex"), start: Date.now() }; + this.phases.push(this.active); + return this.active.spanId; + } + endPhase(failed = false) { + if (this.active) { this.active.end = Date.now(); this.active.failed = failed; this.active = undefined; } + } + async configureCompany(api: Pick, companyId: string) { + if (!uuid.test(companyId)) throw new Error("Invalid eval trace company identity"); + this.receipt.companyId = companyId; + try { + const saved = await api.putSensitive<{ settings: { revision: string; enabled: boolean; credentialsConfigured: boolean; destination: { provider: string; endpointBaseUrl?: string } } }>( + `/api/companies/${companyId}/observability`, { + expectedRevision: null, enabled: true, + destination: { provider: "otlp", endpointBaseUrl: this.endpointBaseUrl }, + credentials: { mode: "replace", headers: [ + { name: "authorization", value: `Bearer ${this.config.apiKey}` }, + { name: "x-bt-parent", value: exportBraintrustAttemptParent(this.receipt) }, + ] }, + capture: { userPrompts: true, assistantResponses: true, toolDetails: true, toolContent: true }, + }); + if (!saved.settings?.enabled || !saved.settings.credentialsConfigured || !uuid.test(saved.settings.revision) + || saved.settings.destination.provider !== "otlp" || saved.settings.destination.endpointBaseUrl !== this.endpointBaseUrl) throw new Error("Unexpected observability setting"); + this.receipt.revision = saved.settings.revision; + this.receipt.nativeConfiguration = "configured"; + } catch { + this.receipt.nativeConfiguration = "failed"; + throw new Error("Eval native observability configuration failed; response withheld"); + } + } + async export(input: { failed: boolean; runIds: string[]; cleanup: string; complaints: number; suggestions: number; + outcomeStage?: "worker" | "supervisor"; failureClass?: FailureClass }, transport: typeof fetch = fetch): Promise { + this.endPhase(input.failed); + const endedAt = Date.now(); + const { resourceFromAttributes } = require("@opentelemetry/resources") as { resourceFromAttributes(attributes: Attributes): unknown }; + const { ProtobufTraceSerializer: serializer } = require("@opentelemetry/otlp-transformer") as { + ProtobufTraceSerializer: { serializeRequest(spans: unknown[]): Uint8Array | undefined; deserializeResponse(bytes: Uint8Array): unknown }; + }; + const attributes: Attributes = { + "paperclip.eval.campaign_id": this.identity.campaignId, "paperclip.eval.attempt_id": this.identity.attemptId, + "paperclip.eval.execution_id": this.identity.executionId, "paperclip.eval.provider": this.identity.provider, + "paperclip.eval.model": this.identity.model, "paperclip.eval.environment": this.identity.environment, + "paperclip.eval.outcome_stage": input.outcomeStage ?? "worker", "paperclip.eval.outcome": input.failed ? "failed" : "passed", + ...(input.failureClass ? { "paperclip.eval.failure_class": input.failureClass } : {}), + "paperclip.eval.cleanup": input.cleanup, "paperclip.eval.run_ids": input.runIds, + "paperclip.eval.feedback.complaints": input.complaints, "paperclip.eval.feedback.suggestions": input.suggestions, + ...(this.provenance ? { + "paperclip.eval.suite_id": this.provenance.suiteId, "paperclip.eval.suite_definition_hash": this.provenance.suiteDefinitionHash, + "paperclip.eval.profile_id": this.provenance.profileId, "paperclip.eval.case_id": this.provenance.caseId, + "paperclip.eval.source_revision": this.provenance.sourceRevision ?? "unknown", + "paperclip.eval.grader": "tests/runner-e2e/runner.spec.ts", + } : {}), + ...(this.receipt.companyId ? { "paperclip.company.id": this.receipt.companyId } : {}), + }; + const resource = resourceFromAttributes({ "service.name": "paperclip-evals" }); + const instrumentationScope = { name: "paperclip.eval-operations", version: "1" }; + const rootContext = { traceId: this.receipt.traceId, spanId: this.receipt.spanId, traceFlags: 1 }; + const span = (name: string, spanId: string, start: number, end: number, failed: boolean, parent = false) => ({ + name, kind: 0, spanContext: () => ({ ...rootContext, spanId }), ...(parent ? { parentSpanContext: rootContext } : {}), + startTime: time(start), endTime: time(end), duration: time(end - start), status: { code: failed ? 2 : 1 }, attributes, + links: [], events: [], ended: true, resource, instrumentationScope, + droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0, + }); + try { + const body = serializer.serializeRequest([ + span("paperclip.eval.attempt", this.receipt.spanId, this.startedAt, endedAt, input.failed), + ...this.phases.map(phase => span(`paperclip.eval.${phase.name}`, phase.spanId, phase.start, phase.end!, phase.failed ?? false, true)), + ]); + if (!body) throw new Error("serialization"); + const response = await transport(`${this.endpointBaseUrl}/v1/traces`, { + method: "POST", redirect: "error", signal: AbortSignal.timeout(10_000), + headers: { authorization: `Bearer ${this.config.apiKey}`, "x-bt-parent": `project_id:${this.config.projectId}`, + "content-type": "application/x-protobuf", accept: "application/x-protobuf, application/json" }, + body: Buffer.from(body), + }); + this.receipt.httpStatus = response.status; + if (!response.ok) { this.receipt.errorCode = "receiver_rejected"; await response.body?.cancel(); throw new Error("rejected"); } + const contentType = response.headers.get("content-type")?.split(";")[0].trim(); + if (contentType && !["application/json", "application/x-protobuf"].includes(contentType)) { await response.body?.cancel(); throw new Error("invalid_response"); } + const reader = response.body?.getReader(); + const chunks: Uint8Array[] = []; let length = 0; + try { + while (reader) { + const next = await reader.read(); if (next.done) break; + length += next.value.length; if (length > 65_536) throw new Error("response_too_large"); + chunks.push(next.value); + } + } finally { await reader?.cancel().catch(() => {}); } + if (length) { + const bytes = Buffer.concat(chunks); + const decoded = contentType === "application/json" ? JSON.parse(bytes.toString("utf8")) + : contentType === "application/x-protobuf" ? serializer.deserializeResponse(bytes) : null; + if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_response"); + const partial = (decoded as { partialSuccess?: { rejectedSpans?: unknown } }).partialSuccess; + if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_response"); + const rejected = partial?.rejectedSpans ?? 0; + if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_response"); + if (Number(rejected) !== 0) { this.receipt.errorCode = "partial_rejection"; throw new Error("partial_rejection"); } + } + this.receipt.status = "accepted"; + } catch { + this.receipt.status = "failed"; + this.receipt.errorCode ??= "export_failed"; + } + return this.snapshot(); + } +} + +export async function finalizeEvalAttemptTrace(config: EvalBraintrustConfig, input: { + receipt: EvalTraceReceipt; identity: EvalTraceIdentity; provenance: EvalTraceProvenance; + result: Pick; +}, transport: typeof fetch = fetch): Promise { + const failed = input.result.status !== "passed" || input.result.cleanup !== "passed"; + const receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed", outcome: failed ? "failed" : "passed", verification: "unverified" }; + try { + if (!["passed", "failed"].includes(input.result.status) || !["not_started", "passed", "failed"].includes(input.result.cleanup) + || input.result.failureClass !== undefined && !["candidate_failure", "provider_variance", "transient_infrastructure", "permanent_infrastructure", "secret_leak", "cleanup_failure"].includes(input.result.failureClass) + || input.result.runIds !== undefined && (!Array.isArray(input.result.runIds) || input.result.runIds.some(id => !uuid.test(id))) + || [input.result.feedback?.complaints ?? 0, input.result.feedback?.suggestions ?? 0].some(count => !Number.isSafeInteger(count) || count < 0)) throw new Error("Invalid eval finalization result"); + const trace = new EvalAttemptTrace(config, input.identity, input.provenance, { receipt: input.receipt, startedAtMs: Date.parse(input.result.startedAt) }); + receipt.spanId = trace.phase("finalization"); + const exported = await trace.export({ failed, outcomeStage: "supervisor", failureClass: input.result.failureClass, + runIds: input.result.runIds ?? [], cleanup: input.result.cleanup, + complaints: input.result.feedback?.complaints ?? 0, suggestions: input.result.feedback?.suggestions ?? 0, + }, transport); + receipt.status = exported.status === "accepted" ? "accepted" : "failed"; + if (exported.httpStatus !== undefined) receipt.httpStatus = exported.httpStatus; + if (exported.errorCode) receipt.errorCode = exported.errorCode; + } catch { receipt.errorCode = "invalid_finalization_context"; } + return receipt; +} diff --git a/tests/runner-e2e/budgeted-setup.test.ts b/tests/runner-e2e/budgeted-setup.test.ts new file mode 100644 index 00000000000..f7ddaf2c5b0 --- /dev/null +++ b/tests/runner-e2e/budgeted-setup.test.ts @@ -0,0 +1,75 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import type { RunnerApi } from "./api.js"; +import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js"; +import { finishBudgetedEvalSetup, type EvalSetupObservation } from "./budgeted-setup.js"; + +const identity = { executionId: "fixture-execution", attemptId: "fixture-attempt" }; +const ready = { id: "agent", companyId: "company", status: "idle", lifecycleState: "ready", lifecycleError: null }; +async function fixture() { + const directory = await mkdtemp(path.join(tmpdir(), "budgeted-setup-")); + const file = path.join(directory, "phase.json"); + await createEvalAttemptPhase(file, identity); + return { directory, file, env: { PAPERCLIP_E2E_GATEWAY_BASE_URL: "http://127.0.0.1:12345", + PAPERCLIP_E2E_GATEWAY_PHASE_PATH: file, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId } }; +} + +it("keeps asynchronous hello probes in setup until the same agent is ready", async () => { + const f = await fixture(); + const get = vi.fn().mockResolvedValueOnce({ ...ready, status: "paused", lifecycleState: "verifying", adapterConfig: { private: true } }).mockResolvedValue(ready); + let saved: EvalSetupObservation[] = []; + try { + await finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, + observe: async observations => { + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); + saved = structuredClone(observations); + } }, f.env); + expect(saved.map(row => row.lifecycleState)).toEqual(["verifying", "ready"]); + expect(saved[0]).not.toHaveProperty("adapterConfig"); + expect(get).toHaveBeenCalledTimes(2); + expect(get.mock.calls.every(([url, options]) => url === "/api/agents/agent" && options.timeout > 0 && options.timeout <= 5_000)).toBe(true); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("agent"); + } finally { await rm(f.directory, { recursive: true, force: true }); } +}); + +it.each([ + [{ ...ready, lifecycleError: "harness_command_unavailable" }, "setup failed"], + [{ ...ready, lifecycleHolds: ["approval"] }, "setup is held"], + [{ ...ready, companyId: "other" }, "scope changed"], + [{ ...ready, lifecycleState: "terminated" }, "setup stopped"], +] as const)("preserves setup attribution and the failed state when readiness is rejected: %s", async (state, reason) => { + const f = await fixture(), observe = vi.fn(async () => {}), get = vi.fn().mockResolvedValue(state); + try { + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + deadlineAt: Date.now() + 5_000, api: { get } as unknown as RunnerApi, observe }, f.env)).rejects.toThrow(reason); + expect(get).toHaveBeenCalledOnce(); + expect(observe).toHaveBeenCalledOnce(); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); + } finally { await rm(f.directory, { recursive: true, force: true }); } +}); + +it("does not start a readiness request after the attempt deadline", async () => { + const f = await fixture(), get = vi.fn(); + try { + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + deadlineAt: Date.now() - 1, api: { get } as unknown as RunnerApi, observe: async () => {} }, f.env)).rejects.toThrow("Timed out"); + expect(get).not.toHaveBeenCalled(); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); + } finally { await rm(f.directory, { recursive: true, force: true }); } +}); + +it("does not advance when saving readiness crosses the deadline", async () => { + const f = await fixture(), get = vi.fn().mockResolvedValue(ready); + const deadlineAt = Date.now() + 5_000; + const now = vi.spyOn(Date, "now"); + try { + await expect(finishBudgetedEvalSetup({ executionId: identity.executionId, agentId: "agent", companyId: "company", + deadlineAt, api: { get } as unknown as RunnerApi, + observe: async () => { now.mockReturnValue(deadlineAt); } }, f.env)).rejects.toThrow("Timed out saving"); + expect(get).toHaveBeenCalledOnce(); + expect((await readEvalAttemptPhase(f.file, identity)).phase).toBe("setup"); + } finally { now.mockRestore(); await rm(f.directory, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/budgeted-setup.ts b/tests/runner-e2e/budgeted-setup.ts new file mode 100644 index 00000000000..42dc216ea50 --- /dev/null +++ b/tests/runner-e2e/budgeted-setup.ts @@ -0,0 +1,44 @@ +import type { Agent } from "../../packages/shared/src/types/agent.js"; +import { pollUntil, type RunnerApi } from "./api.js"; +import { beginEvalAgentPhase } from "./attempt-phase.js"; + +type SetupState = Pick & { lifecycleHolds?: string[] }; +export type EvalSetupObservation = SetupState & { observedAt: string }; + +export async function finishBudgetedEvalSetup(input: { + executionId: string; + agentId: string; + companyId: string; + deadlineAt: number; + api: Pick; + observe: (observations: EvalSetupObservation[]) => Promise; +}, env: NodeJS.ProcessEnv = process.env) { + if (env.PAPERCLIP_E2E_GATEWAY_BASE_URL) { + const deadlineAt = Math.min(input.deadlineAt, Date.now() + 45_000); + const observations: EvalSetupObservation[] = []; + let previous: string | undefined; + const scoped = (agent: SetupState) => agent.id === input.agentId && agent.companyId === input.companyId; + await pollUntil({ label: "budgeted eval agent setup", deadlineAt, intervalMs: 500, + load: async () => { + const agent = await input.api.get(`/api/agents/${input.agentId}`, { timeout: Math.max(1, Math.min(5_000, deadlineAt - Date.now())) }); + const state: SetupState = { id: agent.id, companyId: agent.companyId, status: agent.status, + lifecycleState: agent.lifecycleState, lifecycleError: agent.lifecycleError, lifecycleHolds: agent.lifecycleHolds }; + const current = JSON.stringify(state); + if (current !== previous) { + observations.push({ ...state, observedAt: new Date().toISOString() }); + await input.observe(observations); + previous = current; + } + return state; + }, + accept: agent => scoped(agent) && agent.lifecycleState === "ready" && ["idle", "active"].includes(agent.status) + && !agent.lifecycleError && !agent.lifecycleHolds?.length, + reject: agent => !scoped(agent) ? "Agent scope changed" + : agent.lifecycleError ? `Agent setup failed: ${agent.lifecycleError}` + : agent.lifecycleHolds?.length ? "Agent setup is held" + : ["paused", "rejected", "terminated"].includes(agent.lifecycleState ?? "") ? `Agent setup stopped: ${agent.lifecycleState}` : undefined, + }); + if (Date.now() >= deadlineAt) throw new Error("Timed out saving budgeted eval agent readiness"); + } + await beginEvalAgentPhase(input.executionId, env); +} diff --git a/tests/runner-e2e/campaign-timing.test.ts b/tests/runner-e2e/campaign-timing.test.ts new file mode 100644 index 00000000000..cf7409be76f --- /dev/null +++ b/tests/runner-e2e/campaign-timing.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import { buildCampaignAccounting } from "./attempt-accounting.js"; +import { campaignAttemptTimeLabel, summarizeCampaignTiming } from "./campaign-timing.js"; +import type { RunnerE2EResult } from "./types.js"; + +function result(executionId: string, startMs: number, durationMs: number, attempt = 1): RunnerE2EResult { + return { + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "timing-fixture", + accountingIdentity: { campaignId: "timing-campaign", attemptId: `${executionId}-${attempt}` }, + attempt, status: "passed", profileId: "runner-codex", environmentId: "local", + caseId: executionId, provider: "codex", model: "fixture-model", runtimeMode: "native", + startedAt: new Date(startMs).toISOString(), finishedAt: new Date(startMs + durationMs).toISOString(), + durationMs, cleanup: "passed", + }; +} + +describe("campaign timing evidence", () => { + it("separates overlapping attempt work from the observed window and retains a failed retry", () => { + const failed = { ...result("a", 0, 30_000), status: "failed" as const }; + const selected = [result("a", 30_000, 120_000, 2), result("b", 60_000, 60_000)]; + const accounting = buildCampaignAccounting({ results: selected, attempts: [failed, ...selected] }); + expect(summarizeCampaignTiming(selected, accounting)).toEqual({ + retainedAttempts: 3, missingAttempts: 0, subtotalMs: 210_000, totalMs: 210_000, observedWindowMs: 150_000, + }); + }); + + it("keeps a missing retry unknown while exposing the retained subtotal", () => { + const timing = summarizeCampaignTiming([result("a", 60_000, 30_000, 2)]); + expect(timing).toEqual({ retainedAttempts: 1, missingAttempts: 1, subtotalMs: 30_000, totalMs: null, observedWindowMs: 30_000 }); + expect(campaignAttemptTimeLabel(timing)).toBe("Unknown (30s retained subtotal)"); + const missing = summarizeCampaignTiming([{ ...result("a", 0, 0, 0), status: "failed" }]); + expect(missing.totalMs).toBeNull(); + expect(missing.observedWindowMs).toBeNull(); + expect(campaignAttemptTimeLabel(missing)).toBe("Unknown"); + }); + + it("does not derive an observed window from reversed retained timestamps", () => { + const timing = summarizeCampaignTiming([{ ...result("a", 1000, 1000), finishedAt: new Date(0).toISOString() }]); + expect(timing.totalMs).toBe(1000); + expect(timing.observedWindowMs).toBeNull(); + }); +}); diff --git a/tests/runner-e2e/campaign-timing.ts b/tests/runner-e2e/campaign-timing.ts new file mode 100644 index 00000000000..0958dede052 --- /dev/null +++ b/tests/runner-e2e/campaign-timing.ts @@ -0,0 +1,83 @@ +import { buildCampaignAccounting } from "./attempt-accounting.js"; +import { durationLabel, html } from "./report-format.js"; +import { WORKER_TIMING_PHASES } from "./worker-timing.js"; +import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; + +export const campaignTimingScope = "Attempt durations include setup and cleanup captured inside each attempt. The observed window spans the earliest retained start to the latest retained finish, including gaps and overlap. Carried-forward evidence can span older runs, so this window is not the current workflow's duration. Neither measure is CI queue/setup time or provider-billed runtime."; + +export function summarizeCampaignTiming( + results: readonly RunnerE2EResult[], + retained?: RunnerE2ECampaignAccounting, +) { + const accounting = retained ?? buildCampaignAccounting({ results }); + const attempts = accounting.attempts; + const subtotal = attempts.reduce((total, attempt) => total + attempt.durationMs, 0); + const subtotalMs = Number.isFinite(subtotal) ? subtotal : null; + const starts = attempts.map(attempt => Date.parse(attempt.startedAt)); + const finishes = attempts.map(attempt => Date.parse(attempt.finishedAt)); + const validWindows = attempts.length > 0 && starts.every((start, index) => + Number.isFinite(start) && Number.isFinite(finishes[index]) && finishes[index]! >= start); + const observedWindowMs = validWindows ? Math.max(...finishes) - Math.min(...starts) : null; + const complete = attempts.length > 0 && accounting.missingAttempts === 0 && subtotalMs !== null; + return { + retainedAttempts: attempts.length, + missingAttempts: accounting.missingAttempts, + subtotalMs, + totalMs: complete ? subtotalMs : null, + observedWindowMs, + }; +} + +export function campaignAttemptTimeLabel(timing: ReturnType) { + if (timing.totalMs !== null) return durationLabel(timing.totalMs); + return timing.retainedAttempts > 0 && timing.subtotalMs !== null + ? `Unknown (${durationLabel(timing.subtotalMs)} retained subtotal)` + : "Unknown"; +} + +export function campaignTimingText(timing: ReturnType) { + return [ + `Summed attempt time: ${campaignAttemptTimeLabel(timing)}; ${timing.retainedAttempts} retained attempts; ${timing.missingAttempts} missing attempts.`, + `Observed attempt window: ${durationLabel(timing.observedWindowMs)}.`, + campaignTimingScope, + ]; +} + +export function summarizeWorkerPhaseTiming(results: readonly RunnerE2EResult[], retained?: RunnerE2ECampaignAccounting) { + const accounting = retained ?? buildCampaignAccounting({ results }); + const receipts = accounting.attempts.map(attempt => attempt.workerTiming); + const completeAttempts = receipts.filter(receipt => receipt?.status === "complete").length; + const partialAttempts = receipts.filter(receipt => receipt?.status === "partial").length; + const complete = receipts.length > 0 && completeAttempts === receipts.length && accounting.missingAttempts === 0; + return { + retainedAttempts: receipts.length, completeAttempts, partialAttempts, missingAttempts: accounting.missingAttempts, + phases: WORKER_TIMING_PHASES.map(phase => { + const segments = receipts.flatMap(receipt => receipt && receipt.status !== "unavailable" + ? receipt.segments.filter(segment => segment.phase === phase && segment.finishedAt !== null) : []); + const subtotalMs = segments.reduce((total, segment) => total + Date.parse(segment.finishedAt!) - Date.parse(segment.startedAt), 0); + return { phase, totalMs: complete ? subtotalMs : null, subtotalMs, observedSegments: segments.length }; + }), + }; +} + +export const workerTimingScope = "Worker phases cover the test callback through finalization before result publication. Browser/server startup, parent cleanup, CI queue/build time and provider-billed runtime are outside these measurements. An interrupted phase has no inferred end time."; + +function workerPhaseLabel(phase: ReturnType["phases"][number]) { + return phase.totalMs !== null ? durationLabel(phase.totalMs) + : phase.observedSegments ? `Unknown (${durationLabel(phase.subtotalMs)} observed)` : "Unknown"; +} + +export function workerPhaseTimingText(timing: ReturnType) { + return [ + `Worker phase coverage: ${timing.completeAttempts}/${timing.retainedAttempts} complete receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts.`, + ...timing.phases.map(phase => `Worker ${phase.phase} time: ${workerPhaseLabel(phase)}.`), + workerTimingScope, + ]; +} + +export function renderWorkerPhaseTiming(timing: ReturnType) { + return `
+ ${timing.phases.map(phase => `
${html(workerPhaseLabel(phase))}Worker ${phase.phase} time
`).join("")} +

${timing.completeAttempts}/${timing.retainedAttempts} complete worker receipts; ${timing.partialAttempts} partial; ${timing.missingAttempts} missing attempts. ${html(workerTimingScope)}

+
`; +} diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index b06ccb8b1fd..8fe1d5957fe 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -219,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 48, 16, 10, 2, 1, 1, 4, 116, 2, + 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 28, 18, 2, 6, 6, 13, 10, 2, 48, 16, 10, 2, 1, 1, 4, 116, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(761); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(761); + expect(validateRunnerCatalog()).toHaveLength(763); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(763); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 8f33bc2ad3c..c6bc7b4d292 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -1,3 +1,4 @@ +import { FEEDBACK_PERSISTENCE_SUITE, feedbackPersistenceTask, feedbackPersistenceDigest } from "./feedback-persistence.js"; import { followupStopTask, followupStopDigest } from "./followup-stop-resume.js"; import { finalizationRecoveryTask, finalizationRecoveryDigest } from "./finalization-recovery.js"; import { necessaryQuestionTask, necessaryQuestionDigest } from "./necessary-question.js"; @@ -1664,6 +1665,17 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), + { + id: FEEDBACK_PERSISTENCE_SUITE, label: "Feedback Persistence", manualOnly: true, + description: "Real complaint and suggestion submissions, durable bodies, and retained tool receipts.", + groups: ["native"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)), + environments: [localEnvironment], tasks: [feedbackPersistenceTask(runnerTasks.find(task => task.id === "message-marker")!)], expectedMatrixSize: 2, + definitionMetadata: { version: 1, owner: "runner-e2e-maintainers", scheduling: "explicit-only", + sourceDigest: feedbackPersistenceDigest(runnerTasks.find(task => task.id === "message-marker")!), expectedWorkRuns: 1, automaticRetryPolicy: "single_attempt", + budgetCents: 1_000, forecast: "unmeasured; gateway manifest required for bounded qualification", + grading: "retained-exact-bodies-and-run-attributed-completed-tools", feedback: "synthetic-disposable-items" }, + }, { id: "core-compatibility", label: "Core Runner Compatibility", diff --git a/tests/runner-e2e/cleanup-verification.ts b/tests/runner-e2e/cleanup-verification.ts index d21191e99fc..2e935b226b0 100644 --- a/tests/runner-e2e/cleanup-verification.ts +++ b/tests/runner-e2e/cleanup-verification.ts @@ -22,7 +22,7 @@ export function mustPreserveRecoveryState(input: { resourceAdmissionStarted?: boolean; results: readonly { cleanup: string; synthetic?: boolean }[]; }) { - return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed") + return input.processCleanupFailed || input.results.some(result => result.cleanup === "failed" || result.synthetic) || (input.resourceAdmissionStarted === true && (input.results.length === 0 || input.results.some(result => result.synthetic))); } diff --git a/tests/runner-e2e/codex-ci-sandbox.test.ts b/tests/runner-e2e/codex-ci-sandbox.test.ts index 05942345988..94b043ed204 100644 --- a/tests/runner-e2e/codex-ci-sandbox.test.ts +++ b/tests/runner-e2e/codex-ci-sandbox.test.ts @@ -34,7 +34,7 @@ describe("Codex CI sandbox trust boundary", () => { /- name: Provision Codex sandbox on the disposable trusted runner\n\s+if: ([^\n]+)/, )?.[1]; expect(condition).toMatch( - /^matrix\.environmentId == 'local' && \(matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/, + /^matrix\.environmentId == 'local' && \(needs\.catalog\.outputs\.operations_cadence == 'weekly' \|\| matrix\.profileId == '[^']+'(?: \|\| matrix\.profileId == '[^']+')*\)$/, ); const provisionedProfiles = new Set( [...condition!.matchAll(/matrix\.profileId == '([^']+)'/g)].map((match) => match[1]), @@ -77,7 +77,7 @@ describe("Codex CI sandbox trust boundary", () => { const setup = workflow.indexOf( "- name: Provision Codex sandbox on the disposable trusted runner", ); - const paid = workflow.indexOf("- name: Run paid cell"); + const paid = workflow.indexOf("- name: Run bounded campaign and retain feedback review"); expect(setup).toBeGreaterThan( workflow.indexOf( "- name: Reauthorize paid execution before provider access", diff --git a/tests/runner-e2e/cost-ledger.test.ts b/tests/runner-e2e/cost-ledger.test.ts new file mode 100644 index 00000000000..c4c0346775a --- /dev/null +++ b/tests/runner-e2e/cost-ledger.test.ts @@ -0,0 +1,197 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalCostObservation, type EvalSpendLimits } from "./cost-ledger.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const limits = (overrides: Partial = {}): EvalSpendLimits => ({ campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2, ...overrides }); +const identity = { executionId: "codex.local.fixture", attemptId: "attempt-1", phase: "setup" as const }; +const observation = (overrides: Partial = {}): EvalCostObservation => ({ observationId: "receipt", requestId: "request", source: "provider", provider: "openai", model: "fixture", providerRequestId: "provider-request", currency: "USD", route: "direct", classification: "reported", costUsd: "0.000000123", final: true, ...overrides }); +async function fixture(overrides: Partial = {}) { + const root = await mkdtemp(path.join(os.tmpdir(), "eval-cost-ledger-")); roots.push(root); + const file = path.join(root, "costs.jsonl"); + return { file, ledger: await openEvalCostLedger(file, limits(overrides)) }; +} + +it("persists a reservation before dispatch and counts setup spending with nanodollar precision", async () => { + const { file, ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "0.1"); + const beforeDispatch = (await readEvalCostLedger(file)).state.snapshot(); + expect(beforeDispatch).toMatchObject({ committedUsd: "0.100000000", heldUsd: "0.100000000", unknownRequests: 1 }); + await ledger.observe(observation()); + await ledger.finish("request", "succeeded"); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.000000123", committedUsd: "0.000000123", heldUsd: "0.000000000", + requests: [{ phase: "setup", receiptComplete: true }] }); + } finally { await ledger.close(); } + expect((await readEvalCostLedger(file)).state.snapshot().knownUsd).toBe("0.000000123"); +}); + +it("serializes competing admissions so concurrent requests cannot oversubscribe a cap", async () => { + const { ledger } = await fixture(); + try { + const admitted = await Promise.allSettled([ledger.reserve("one", identity, "0.6"), ledger.reserve("two", identity, "0.6")]); + expect(admitted.map(value => value.status)).toEqual(["fulfilled", "rejected"]); + expect(ledger.snapshot().committedUsd).toBe("0.600000000"); + } finally { await ledger.close(); } +}); + +it("durably caps attempts including zero-request failures and rejects unadmitted requests", async () => { + const cap = limits({ maxAttemptsPerExecution: 2 }); + const { file, ledger } = await fixture({ maxAttemptsPerExecution: 2 }); + await expect(ledger.reserve("request", identity, "0.1")).rejects.toThrow("no admitted attempt"); + await ledger.admitAttempt(identity.executionId, "attempt-1"); + await ledger.close(); + const resumed = await openEvalCostLedger(file, cap); + try { + await expect(resumed.admitAttempt(identity.executionId, "attempt-1")).rejects.toThrow("already admitted"); + const competing = await Promise.allSettled([ + resumed.admitAttempt(identity.executionId, "attempt-2"), + resumed.admitAttempt(identity.executionId, "attempt-3"), + ]); + expect(competing.map(result => result.status)).toEqual(["fulfilled", "rejected"]); + await resumed.reserve("request", { ...identity, attemptId: "attempt-2" }, "0.1"); + await resumed.admitAttempt("another.cell", "another-attempt"); + await expect(resumed.reserve("wrong-cell", { ...identity, attemptId: "another-attempt" }, "0.1")).rejects.toThrow("no admitted attempt"); + expect((await readEvalCostLedger(file)).state.snapshot().admittedAttempts).toHaveLength(3); + } finally { await resumed.close(); } + await expect(openEvalCostLedger(file, limits({ maxAttemptsPerExecution: 3 }))).rejects.toThrow("limits changed"); +}); + +it("holds unknown/cancelled charges after restart and counts failed retries independently", async () => { + const { file, ledger } = await fixture(); + await ledger.reserve("request", identity, "0.5"); + await ledger.finish("request", "cancelled"); + await ledger.close(); + const resumed = await openEvalCostLedger(file, limits()); + try { + expect(resumed.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, committedUsd: "0.500000000" }); + await resumed.reserve("retry", { ...identity, attemptId: "attempt-2" }, "0.5"); + await resumed.observe(observation({ observationId: "retry-cost", requestId: "retry", providerRequestId: "provider-retry", costUsd: "0.2" })); + await resumed.finish("retry", "failed"); + expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.500000000", committedUsd: "0.700000000" }); + await resumed.observe(observation({ costUsd: "0.3" })); + expect(resumed.snapshot()).toMatchObject({ knownUsd: "0.500000000", heldUsd: "0.000000000" }); + } finally { await resumed.close(); } +}); + +it("deduplicates provider/gateway/native observations and preserves invoice adjustments", async () => { + const { ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "1"); + await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" })); + await ledger.observe(observation({ source: "gateway", classification: "estimated", pricingVersion: "2026-10-20", costUsd: "0.3" })); + await ledger.observe(observation({ observationId: "native", source: "native_otlp", costUsd: "0.2" })); + await ledger.observe(observation({ observationId: "invoice", source: "invoice", classification: "reconciled", costUsd: "0.15" })); + await ledger.finish("request", "succeeded"); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.150000000", breakdown: { estimated: "0.000000000", reported: "0.000000000", reconciled: "0.150000000" } }); + await expect(ledger.observe(observation({ costUsd: "0.7" }))).rejects.toThrow("identity conflict"); + } finally { await ledger.close(); } +}); + +it("retains partial costs and contradictory receipts without freeing their reservations", async () => { + const { ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "0.5"); + await ledger.observe(observation({ final: false, costUsd: "0.2" })); + await ledger.finish("request", "failed"); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.200000000", heldUsd: "0.300000000", unknownRequests: 1 }); + await ledger.observe(observation({ observationId: "final1", costUsd: "0.3" })); + await ledger.observe(observation({ observationId: "final2", costUsd: "0.4" })); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.400000000", heldUsd: "0.100000000", conflictingRequests: 1 }); + await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("receipts conflict"); + } finally { await ledger.close(); } +}); + +it("rejects ambiguous request joins and stops on measured overshoot", async () => { + const { ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "0.1"); + await ledger.reserve("another", identity, "0.1"); + await ledger.observe(observation({ costUsd: "1.1" })); + await expect(ledger.observe(observation({ observationId: "duplicate-request", requestId: "another" }))).rejects.toThrow("another reservation"); + expect(ledger.snapshot().overBudget).toBe(true); + await ledger.finish("request", "succeeded"); + await expect(ledger.reserve("later", identity, "0.1")).rejects.toThrow("spending cap"); + } finally { await ledger.close(); } +}); + +it("enforces deadlines, request count, concurrency, and explicit pricing/usage invariants", async () => { + const expired = await fixture({ deadlineAt: "2000-01-01T00:00:00Z" }); + await expect(expired.ledger.reserve("request", identity, "0.1")).rejects.toThrow("deadline"); + await expired.ledger.close(); + const { ledger } = await fixture({ maxRequests: 2, maxConcurrent: 1 }); + try { + await ledger.reserve("request", identity, "0.1"); + await expect(ledger.reserve("two", identity, "0.1")).rejects.toThrow("concurrency"); + await expect(ledger.observe(observation({ classification: "estimated" }))).rejects.toThrow("pricing provenance"); + await expect(ledger.observe(observation({ tokens: { output: 1, reasoning: 2 } }))).rejects.toThrow("token usage"); + await ledger.finish("request", "failed"); + await ledger.reserve("two", identity, "0.1"); + await ledger.finish("two", "failed"); + await expect(ledger.reserve("three", identity, "0.1")).rejects.toThrow("request cap"); + } finally { await ledger.close(); } +}); + +it("fails closed on concurrent writers, changed limits, incomplete writes, and tampering", async () => { + const { file, ledger } = await fixture(); + await expect(openEvalCostLedger(file, limits())).rejects.toThrow("writer lock"); + await ledger.reserve("request", identity, "0.1"); + await ledger.close(); + await expect(openEvalCostLedger(file, limits({ maxUsd: "2" }))).rejects.toThrow("limits changed"); + const contents = await readFile(file, "utf8"); + await writeFile(file, contents.slice(0, -1)); + await expect(readEvalCostLedger(file)).rejects.toThrow("incomplete"); + await writeFile(file, contents.replace('"maxUsd":"1"', '"maxUsd":"2"')); + await expect(readEvalCostLedger(file)).rejects.toThrow("integrity"); +}); + +it("rejects accidental raw payload fields and accepts equivalent receipt key order", async () => { + const { file, ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "0.1"); + await expect(ledger.observe({ ...observation(), rawResponse: "private response" } as EvalCostObservation)).rejects.toThrow("Unknown ledger field"); + await ledger.observe(observation()); + await ledger.observe(Object.fromEntries(Object.entries(observation()).reverse()) as unknown as EvalCostObservation); + expect(await readFile(file, "utf8")).not.toContain("private response"); + } finally { await ledger.close(); } +}); + +it("does not let a terminal estimate erase a stronger partial provider charge", async () => { + const { ledger } = await fixture(); + try { + await ledger.reserve("request", identity, "0.8"); + await ledger.observe(observation({ observationId: "provider-partial", final: false, costUsd: "0.6" })); + await ledger.observe(observation({ observationId: "gateway-final", source: "gateway", classification: "estimated", pricingVersion: "fixture", costUsd: "0.1" })); + await ledger.finish("request", "succeeded"); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.600000000", heldUsd: "0.200000000", unknownRequests: 1, breakdown: { reported: "0.600000000", estimated: "0.000000000" } }); + await expect(ledger.reserve("next", identity, "0.3")).rejects.toThrow("spending cap"); + await ledger.observe(observation({ observationId: "provider-final", costUsd: "0.65" })); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "0.650000000", heldUsd: "0.000000000", unknownRequests: 0 }); + } finally { await ledger.close(); } +}); + +it("replays admissions when the deadline crosses between validation and journal writing", async () => { + const admittedAt = Date.parse("2098-12-31T23:59:59.999Z"); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(admittedAt); + const { file, ledger } = await fixture(); + const validate = EvalSpendState.prototype.validate; + const validation = vi.spyOn(EvalSpendState.prototype, "validate").mockImplementation(function (this: EvalSpendState, event, now) { + validate.call(this, event, now); + vi.setSystemTime(admittedAt + 2); + }); + try { + await ledger.reserve("request", identity, "0.1"); + validation.mockRestore(); + expect((await readEvalCostLedger(file)).state.snapshot().committedUsd).toBe("0.100000000"); + const records = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + expect(records[1].at).toBe(new Date(admittedAt).toISOString()); + } finally { + validation.mockRestore(); + vi.useRealTimers(); + await ledger.close(); + } +}); diff --git a/tests/runner-e2e/cost-ledger.ts b/tests/runner-e2e/cost-ledger.ts new file mode 100644 index 00000000000..c132a6257fa --- /dev/null +++ b/tests/runner-e2e/cost-ledger.ts @@ -0,0 +1,293 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { mkdir, open, readFile, rm, type FileHandle } from "node:fs/promises"; +import path from "node:path"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; + +export interface EvalSpendLimits { + campaignId: string; + maxUsd: string; + deadlineAt: string; + maxRequests: number; + maxConcurrent: number; + maxAttemptsPerExecution?: number; + overhead?: EvalOperatingOverhead; +} +export interface EvalChargeIdentity { + executionId: string; + attemptId: string; + phase: "setup" | "agent" | "judge" | "feedback_review" | "tool" | "infrastructure"; + runId?: string; +} +export interface EvalLegacyAttemptAudit { + previousJournalHash: string; + evidenceSha256: string; + maxAttemptsPerExecution: number; + priorAttempts: Array<{ executionId: string; attemptId: string }>; +} +export interface EvalCostObservation { + observationId: string; + requestId: string; + source: "provider" | "gateway" | "native_otlp" | "heartbeat" | "infrastructure" | "invoice"; + provider: string; + model?: string; + providerRequestId?: string; + currency: "USD"; + route: "direct" | "braintrust" | "other_gateway" | "subscription" | "infrastructure"; + classification: "reported" | "estimated" | "allocated" | "reconciled" | "unknown"; + costUsd?: string; + pricingVersion?: string; + tokens?: { input?: number; cachedRead?: number; cachedWrite?: number; output?: number; reasoning?: number }; + final: boolean; +} + +type Event = + | { type: "campaign"; limits: EvalSpendLimits } + | ({ type: "attempt_policy" } & EvalLegacyAttemptAudit) + | { type: "attempt"; executionId: string; attemptId: string } + | { type: "reserve"; requestId: string; identity: EvalChargeIdentity; maxUsd: string } + | { type: "observe"; observation: EvalCostObservation } + | { type: "finish"; requestId: string; outcome: "succeeded" | "failed" | "cancelled" }; +interface Line { seq: number; at: string; previous: string; event: Event; hash: string } +interface RequestState { identity: EvalChargeIdentity; reserved: bigint; observations: EvalCostObservation[]; finished: boolean } +const usd = (units: bigint) => `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`; +export const validEvalLedgerId = (value: unknown): value is string => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value); +const validId = validEvalLedgerId; +const amount = (value: string) => { + if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(value)) throw new Error("Invalid ledger USD amount"); + return usdToUnits(value); +}; +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value, (_key, item) => + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries(Object.keys(item).sort().map(key => [key, item[key]])) : item)).digest("hex"); +const fields = (value: object, allowed: string[]) => { + if (Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Unknown ledger field; refusing unreviewed data"); +}; +const ranks = { unknown: 0, allocated: 1, estimated: 2, reported: 3, reconciled: 4 }; + +export class EvalAdmissionError extends Error {} + +export class EvalSpendState { + readonly limits: EvalSpendLimits; + readonly requests = new Map(); + readonly observations = new Map(); + readonly attempts = new Map(); + private providerRequests = new Map(); + constructor(limits: EvalSpendLimits) { + this.limits = structuredClone(limits); + fields(limits, ["campaignId", "maxUsd", "deadlineAt", "maxRequests", "maxConcurrent", "maxAttemptsPerExecution", "overhead"]); + if (!validId(limits.campaignId) || amount(limits.maxUsd) <= 0n || !Number.isFinite(Date.parse(limits.deadlineAt)) + || !Number.isSafeInteger(limits.maxRequests) || limits.maxRequests < 1 + || !Number.isSafeInteger(limits.maxConcurrent) || limits.maxConcurrent < 1 + || limits.maxAttemptsPerExecution !== undefined && (!Number.isSafeInteger(limits.maxAttemptsPerExecution) || limits.maxAttemptsPerExecution < 1)) throw new Error("Invalid eval spend limits"); + if (this.limits.overhead !== undefined) { + validateOperatingOverhead(this.limits.overhead, this.limits.maxUsd); + if (this.limits.maxAttemptsPerExecution === undefined || this.limits.maxRequests <= this.limits.overhead.components.length) { + throw new Error("Operating overhead requires bounded attempts and room for model requests"); + } + for (const component of this.limits.overhead.components) this.requests.set(`overhead:${component.component}`, { + identity: { executionId: "campaign-overhead", attemptId: component.component, phase: "infrastructure" }, + reserved: amount(component.maxUsd), observations: [], finished: true, + }); + } + } + private authoritative(request: RequestState) { + const priced = request.observations.filter(value => value.costUsd !== undefined); + const bestRank = Math.max(-1, ...priced.map(value => ranks[value.classification])); + const strongest = priced.filter(value => ranks[value.classification] === bestRank); + const finalized = strongest.filter(value => value.final); + const best = finalized.length ? finalized : strongest; + if (!best.length) return { cost: null, classification: "unknown" as const, conflict: false, resolved: false }; + const amounts = new Set(best.map(value => amount(value.costUsd!).toString())); + const cost = best.reduce((maximum, value) => amount(value.costUsd!) > maximum ? amount(value.costUsd!) : maximum, 0n); + return { cost, classification: best[0]!.classification, conflict: finalized.length > 0 && amounts.size !== 1, + resolved: finalized.length > 0 && amounts.size === 1 }; + } + snapshot() { + let known = 0n, held = 0n, committed = 0n, inflight = 0, unknown = 0, conflicts = 0; + const breakdown: Record<"reported" | "estimated" | "allocated" | "reconciled", bigint> = { reported: 0n, estimated: 0n, allocated: 0n, reconciled: 0n }; + const requests = [...this.requests].map(([requestId, request]) => { + const receipt = this.authoritative(request); + if (!request.finished) inflight++; + if (receipt.conflict) conflicts++; + if (!receipt.resolved) unknown++; + const cost = receipt.cost ?? 0n; + known += cost; + if (receipt.classification !== "unknown") breakdown[receipt.classification] += cost; + const commitment = !request.finished || !receipt.resolved + ? (request.reserved > cost ? request.reserved : cost) : cost; + committed += commitment; + held += commitment - cost; + return { requestId, ...request.identity, reservedUsd: usd(request.reserved), costUsd: receipt.cost === null ? null : usd(cost), + classification: receipt.classification, conflict: receipt.conflict, receiptComplete: receipt.resolved, finished: request.finished }; + }); + return { campaignId: this.limits.campaignId, limitUsd: usd(amount(this.limits.maxUsd)), knownUsd: usd(known), heldUsd: usd(held), committedUsd: usd(committed), + overBudget: committed > amount(this.limits.maxUsd), unknownRequests: unknown, conflictingRequests: conflicts, inflight, + breakdown: Object.fromEntries(Object.entries(breakdown).map(([key, value]) => [key, usd(value)])), requests, + admittedAttempts: [...this.attempts].map(([attemptId, executionId]) => ({ executionId, attemptId })) }; + } + validate(event: Event, now = Date.now()) { + if ((event.type === "attempt" || event.type === "reserve") && this.limits.overhead + && (now < Date.parse(this.limits.overhead.verifiedAt) || now >= Date.parse(this.limits.overhead.expiresAt))) { + throw new EvalAdmissionError("Operating overhead allowance needs review before paid admission"); + } + if (event.type === "campaign") throw new Error("Duplicate campaign event"); + if (event.type === "attempt_policy") { + fields(event, ["type", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"]); + if (this.limits.maxAttemptsPerExecution !== undefined || this.attempts.size) throw new Error("Eval attempt policy already established"); + if (!/^[0-9a-f]{64}$/.test(event.previousJournalHash) || !/^[0-9a-f]{64}$/.test(event.evidenceSha256) + || !Number.isSafeInteger(event.maxAttemptsPerExecution) || event.maxAttemptsPerExecution < 1 + || !Array.isArray(event.priorAttempts)) throw new Error("Invalid legacy attempt audit"); + const audited = new Map(); + for (const attempt of event.priorAttempts) { + fields(attempt, ["executionId", "attemptId"]); + if (!validId(attempt.executionId) || !validId(attempt.attemptId) || audited.has(attempt.attemptId)) throw new Error("Invalid legacy attempt identity"); + audited.set(attempt.attemptId, attempt.executionId); + } + for (const request of this.requests.values()) { + if (!request.finished) throw new Error("Legacy attempt audit requires stopped requests"); + if (audited.get(request.identity.attemptId) !== request.identity.executionId) throw new Error("Legacy attempt audit omits a recorded request"); + } + } else if (event.type === "attempt") { + fields(event, ["type", "executionId", "attemptId"]); + if (!validId(event.executionId) || !validId(event.attemptId) || this.limits.maxAttemptsPerExecution === undefined) throw new Error("Invalid attempt admission"); + if (this.attempts.has(event.attemptId)) throw new EvalAdmissionError("Eval attempt already admitted"); + if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached"); + if ([...this.attempts.values()].filter(id => id === event.executionId).length >= this.limits.maxAttemptsPerExecution) throw new EvalAdmissionError("Eval campaign attempt cap reached"); + } else if (event.type === "reserve") { + fields(event.identity, ["executionId", "attemptId", "phase", "runId"]); + if (!validId(event.requestId) || !validId(event.identity.executionId) || !validId(event.identity.attemptId) + || event.identity.runId !== undefined && !validId(event.identity.runId) + || !["setup", "agent", "judge", "feedback_review", "tool", "infrastructure"].includes(event.identity.phase)) throw new Error("Invalid request identity"); + if (this.limits.maxAttemptsPerExecution !== undefined && this.attempts.get(event.identity.attemptId) !== event.identity.executionId) throw new EvalAdmissionError("Eval request has no admitted attempt"); + if (this.requests.has(event.requestId)) throw new Error("Request already reserved"); + if (amount(event.maxUsd) <= 0n) throw new Error("Request reservation must be positive"); + const current = this.snapshot(); + if (now >= Date.parse(this.limits.deadlineAt)) throw new EvalAdmissionError("Eval campaign deadline reached"); + if (this.requests.size >= this.limits.maxRequests) throw new EvalAdmissionError("Eval campaign request cap reached"); + if (current.inflight >= this.limits.maxConcurrent) throw new EvalAdmissionError("Eval campaign concurrency cap reached"); + if (current.conflictingRequests) throw new EvalAdmissionError("Eval cost receipts conflict; admission stopped"); + if (amount(current.committedUsd) + amount(event.maxUsd) > amount(this.limits.maxUsd)) throw new EvalAdmissionError("Eval campaign spending cap reached"); + } else if (event.type === "observe") { + const o = event.observation; + fields(o, ["observationId", "requestId", "source", "provider", "model", "providerRequestId", "currency", "route", "classification", "costUsd", "pricingVersion", "tokens", "final"]); + if (o.tokens) fields(o.tokens, ["input", "cachedRead", "cachedWrite", "output", "reasoning"]); + if (!this.requests.has(o.requestId) || !validId(o.observationId) || !validId(o.provider) + || o.model !== undefined && !validId(o.model) || o.providerRequestId !== undefined && !validId(o.providerRequestId) + || o.currency !== "USD" || !Object.hasOwn(ranks, o.classification) || typeof o.final !== "boolean" + || !["provider", "gateway", "native_otlp", "heartbeat", "infrastructure", "invoice"].includes(o.source) + || !["direct", "braintrust", "other_gateway", "subscription", "infrastructure"].includes(o.route)) throw new Error("Invalid cost observation"); + if (o.classification === "unknown" ? o.costUsd !== undefined : o.costUsd === undefined) throw new Error("Cost classification requires an explicit amount or unknown"); + if (o.costUsd !== undefined) amount(o.costUsd); + if ((o.classification === "estimated" || o.classification === "allocated") && !validId(o.pricingVersion)) throw new Error("Estimate requires pricing provenance"); + if (o.tokens && (Object.values(o.tokens).some(value => !Number.isSafeInteger(value) || Number(value) < 0) + || o.tokens.reasoning !== undefined && o.tokens.output !== undefined && o.tokens.reasoning > o.tokens.output)) throw new Error("Invalid normalized token usage"); + const previous = this.observations.get(o.observationId); + if (previous && digest(previous) !== digest(o)) throw new Error("Observation identity conflict"); + if (o.providerRequestId) { + const request = this.providerRequests.get(`${o.provider}:${o.providerRequestId}`); + if (request && request !== o.requestId) throw new Error("Provider request belongs to another reservation; reconcile identities before import"); + } + } else if (event.type === "finish") { + if (!this.requests.has(event.requestId) || !["succeeded", "failed", "cancelled"].includes(event.outcome)) throw new Error("Invalid request completion"); + } else throw new Error("Unknown ledger event"); + } + apply(event: Event) { + if (event.type === "attempt_policy") { + this.limits.maxAttemptsPerExecution = event.maxAttemptsPerExecution; + for (const attempt of event.priorAttempts) this.attempts.set(attempt.attemptId, attempt.executionId); + } else if (event.type === "attempt") this.attempts.set(event.attemptId, event.executionId); + else if (event.type === "reserve") this.requests.set(event.requestId, { identity: event.identity, reserved: amount(event.maxUsd), observations: [], finished: false }); + else if (event.type === "observe") { + const o = event.observation; + if (this.observations.has(o.observationId)) return; + this.observations.set(o.observationId, o); + this.requests.get(o.requestId)!.observations.push(o); + if (o.providerRequestId) this.providerRequests.set(`${o.provider}:${o.providerRequestId}`, o.requestId); + } else if (event.type === "finish") this.requests.get(event.requestId)!.finished = true; + } +} + +export async function readEvalCostLedger(file: string) { + const raw = await readFile(file, "utf8"); + if (!raw.endsWith("\n")) throw new Error("Eval ledger has an incomplete durable record"); + const lines = raw.trimEnd().split("\n").map(value => JSON.parse(value) as Line); + let previous = "", state: EvalSpendState | undefined; + for (const [index, line] of lines.entries()) { + const { hash, ...body } = line; + if (line.seq !== index || line.previous !== previous || digest(body) !== hash || !Number.isFinite(Date.parse(line.at))) throw new Error("Eval ledger integrity check failed"); + if (index === 0) { + if (line.event.type !== "campaign") throw new Error("Eval ledger lacks campaign limits"); + state = new EvalSpendState(line.event.limits); + } else { + if (line.event.type === "attempt_policy" && line.event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed"); + state!.validate(line.event, Date.parse(line.at)); + state!.apply(line.event); + } + previous = hash; + } + return { state: state!, sequence: lines.length, previous }; +} + +export async function openEvalCostLedger(file: string, limits: EvalSpendLimits) { + await mkdir(path.dirname(file), { recursive: true, mode: 0o700 }); + const lockPath = `${file}.lock`; + let lock: FileHandle; + try { lock = await open(lockPath, "wx", 0o600); } + catch { throw new Error("Eval ledger writer lock unavailable; inspect prior writer before recovery"); } + let handle: FileHandle | undefined; + try { + await lock.writeFile(`${process.pid}\n`); + await lock.sync(); + let state = new EvalSpendState(limits), sequence = 0, previous = ""; + try { + const loaded = await readEvalCostLedger(file); + if (digest(loaded.state.limits) !== digest(limits)) throw new Error("Eval campaign limits changed during resume"); + ({ state, sequence, previous } = loaded); + } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + handle = await open(file, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); + const write = async (event: Event, at = Date.now()) => { + const body = { seq: sequence, at: new Date(at).toISOString(), previous, event }; + const hash = digest(body); + await handle!.appendFile(`${JSON.stringify({ ...body, hash })}\n`); + await handle!.sync(); + previous = hash; + sequence++; + }; + if (!sequence) await write({ type: "campaign", limits }); + let queue = Promise.resolve(), closed = false, poisoned = false; + const append = (event: Event) => { + if (closed) return Promise.reject(new Error("Eval ledger writer unavailable")); + const operation = queue.then(async () => { + if (poisoned) throw new Error("Eval ledger writer unavailable"); + const admittedAt = Date.now(); + if (event.type === "attempt_policy" && event.previousJournalHash !== previous) throw new Error("Legacy attempt audit checkpoint changed"); + state.validate(event, admittedAt); + try { await write(event, admittedAt); } catch (error) { poisoned = true; throw error; } + state.apply(event); + }); + queue = operation.catch(() => {}); + return operation; + }; + return { + adoptLegacyAttemptPolicy: (audit: EvalLegacyAttemptAudit) => append({ ...structuredClone(audit), type: "attempt_policy" }), + admitAttempt: (executionId: string, attemptId: string) => append({ type: "attempt", executionId, attemptId }), + reserve: (requestId: string, identity: EvalChargeIdentity, maxUsd: string) => append({ type: "reserve", requestId, identity: structuredClone(identity), maxUsd }), + observe: (observation: EvalCostObservation) => append({ type: "observe", observation: structuredClone(observation) }), + finish: (requestId: string, outcome: "succeeded" | "failed" | "cancelled") => append({ type: "finish", requestId, outcome }), + snapshot: () => state.snapshot(), + async close() { + if (closed) return; + closed = true; + await queue; + try { await handle!.close(); } finally { await lock.close(); await rm(lockPath); } + }, + }; + } catch (error) { + await handle?.close(); + await lock.close(); + await rm(lockPath); + throw error; + } +} diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 96976b8b9b7..2913bd96459 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -2,6 +2,7 @@ import path from "node:path"; import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { runnerMatrix } from "./catalog.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { campaignFeedbackReviewText } from "./feedback-dispositions.js"; import { buildRunnerCampaign, canonicalExecutionId, @@ -25,6 +26,10 @@ interface PublishedCampaign { generatedAt: string; expected: string[]; results: PublishedResult[]; + accounting?: RunnerE2ECampaign["accounting"]; + operations?: RunnerE2ECampaign["operations"]; + feedbackReview?: RunnerE2ECampaign["feedbackReview"]; + braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"]; } async function relativeFiles(root: string, current = root): Promise { @@ -57,6 +62,7 @@ export async function regenerateRunnerDashboard(input: { historyFile?: string | null; outputDirectory?: string; evidenceHrefPrefix?: string; + feedbackReviewFile?: string; }) { const bundle = path.resolve(input.bundle); const outputDirectory = path.resolve(input.outputDirectory ?? bundle); @@ -93,6 +99,12 @@ export async function regenerateRunnerDashboard(input: { generatedAt: normalized.generatedAt, expected, results, + accounting: normalized.accounting, + operations: normalized.operations, + braintrustExperiment: normalized.braintrustExperiment, + feedbackReview: input.feedbackReviewFile + ? JSON.parse(await readFile(input.feedbackReviewFile, "utf8")) + : normalized.feedbackReview, }); // A rendering refresh must not inherit the renderer's checkout or CI event. const retainedSource = results.find((result) => @@ -168,10 +180,21 @@ export async function regenerateRunnerDashboard(input: { evidenceErrors: normalized.results[index]?.evidenceErrors ?? [], })), }; + const summary = await readFile(path.join(bundle, "summary.md"), "utf8").then(text => text.replace( + /^(Feedback across retained attempts:.*?); [0-9]+ distinct items awaiting review;/m, "$1;", + )).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); + const reviewStatus = /^(?:[0-9]+ distinct feedback items await review\.|Feedback reviewed by (?:human|agent) at ).*$/m; + const refreshedSummary = summary === undefined ? undefined : reviewStatus.test(summary) + ? summary.replace(reviewStatus, () => campaignFeedbackReviewText(campaign)) + : `${summary.trimEnd()}\n\n## Campaign feedback review\n\n${campaignFeedbackReviewText(campaign)}\n`; await mkdir(outputDirectory, { recursive: true }); await Promise.all([ writeFile(path.join(outputDirectory, "index.html"), dashboard, "utf8"), writeFile(path.join(outputDirectory, "dashboard.html"), dashboard, "utf8"), + ...(refreshedSummary === undefined ? [] : [writeFile(path.join(outputDirectory, "summary.md"), refreshedSummary, "utf8")]), writeFile( path.join(outputDirectory, "normalized-results.json"), `${JSON.stringify(upgraded, null, 2)}\n`, @@ -192,7 +215,7 @@ async function main() { ); if (!bundleArgument) { throw new Error( - "Usage: pnpm test:e2e:runner:dashboard -- [--history ]", + "Usage: pnpm test:e2e:runner:dashboard -- [--history ] [--feedback-review ]", ); } const historyIndex = arguments_.indexOf("--history"); @@ -200,7 +223,10 @@ async function main() { historyIndex >= 0 ? arguments_[historyIndex + 1] : undefined; if (historyIndex >= 0 && !historyFile) throw new Error("--history requires a path"); - await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile }); + const reviewIndex = arguments_.indexOf("--feedback-review"); + const feedbackReviewFile = reviewIndex >= 0 ? arguments_[reviewIndex + 1] : undefined; + if (reviewIndex >= 0 && !feedbackReviewFile) throw new Error("--feedback-review requires a path"); + await regenerateRunnerDashboard({ bundle: bundleArgument, historyFile, feedbackReviewFile }); } if ( diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index c949f6a07b1..b8cb4ac92b0 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -1,3 +1,4 @@ +import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js"; import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; import { reportBrowserScript } from "./report-browser.js"; @@ -8,6 +9,10 @@ import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import { journeyFeatureStyles, renderJourneyFeatureCoverage, renderJourneyFeatureOptions } from "./feature-coverage.js"; import { STARTUP_REUSE_LAYERS, summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; import { renderFirstTaskDetails } from "./first-task-report.js"; +import { summarizeCampaignFeedback } from "./feedback.js"; +import { renderCampaignFeedbackReview } from "./feedback-dispositions.js"; +import { renderEvalOperationsReport } from "./operations-report.js"; +import { renderBraintrustExperiment } from "./braintrust-experiment.js"; import { discoverReportCatalog, type ReportExecution, @@ -20,6 +25,7 @@ import type { } from "./types.js"; import { aggregateCampaignBilling, + billingCoverageLabel, summarizeExecutionBilling, } from "./billing.js"; @@ -269,6 +275,7 @@ function renderCase( ? `
Tokens${html(tokenLabel(billing.llm.inputTokens, billing.llm))} in · ${html(tokenLabel(billing.llm.outputTokens, billing.llm))} out${html(tokenLabel(billing.llm.cachedInputTokens, billing.llm))} cached · ${billing.llm.runsWithTokenUsage}/${billing.llm.runCount} runs covered
LLM spend${html(llmCostLabel(billing.reportedCostUsd, billing.llm))}${billing.llm.runsWithReportedCost}/${billing.llm.runCount} runs provider-priced
+ ${(billing.llm.runsWithEstimatedCost ?? 0) > 0 ? `
LLM estimate${html(billingCoverageLabel(usdLabel(billing.llm.estimatedCostUsd ?? 0), billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount))}Estimated from captured usage
` : ""}
Execution${billing.runtime.estimatedListCostUsd === undefined ? html(billing.runtime.costStatus === "not_metered" ? "Local · not metered" : "Cost unavailable") : `${html(usdLabel(billing.runtime.estimatedListCostUsd))} est.`}${html(durationLabel(billing.runtime.agentRunDurationMs))} agent${billing.runtime.leaseDurationMs === null ? "" : ` · ${html(durationLabel(billing.runtime.leaseDurationMs))} lease`}
${billing.assistant ? `
External assistant${html(usdLabel(billing.assistant.estimatedCostUsd))} est.${html(billing.assistant.model)} · ${billing.assistant.requests} requests · ${html(tokenLabel(billing.assistant.inputTokens + billing.assistant.cachedInputTokens))} in / ${html(tokenLabel(billing.assistant.outputTokens))} out
` : ""}
` @@ -301,6 +308,9 @@ function renderCase( Test details${matcherResults.length ? `${passedMatchers}/${matcherResults.length} checks${notReached ? ` · ${notReached} not reached` : ""}` : "No checks recorded"}${entry ? ` · ${html(durationLabel(entry.result.durationMs))}` : ""} ${passed ? outcome : ""} ${billingStrip} + ${entry ? `

Worker phase timing: ${entry.result.workerTiming ? html(entry.result.workerTiming.status === "unavailable" ? `unavailable (${entry.result.workerTiming.reason})` : entry.result.workerTiming.status) : "not captured"}. ${entry.result.workerTiming?.status === "partial" ? "Only completed phases have observed durations." : ""}

` : ""} + ${entry?.result.feedback ? `

Feedback: ${entry.result.feedback.complaints} complaints · ${entry.result.feedback.suggestions} suggestions · ${entry.result.feedback.review.length} distinct items; see campaign feedback review. Saved-feedback collection: ${html(entry.result.feedback.collection)}; invocation coverage: ${html(entry.result.feedback.invocationCollection)}. Tool receipts: ${entry.result.feedback.invocations.failed} failed · ${entry.result.feedback.invocations.unknown} unresolved. Full bodies: access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.

` : entry ? `

Feedback was not captured for this attempt.

` : ""} + ${entry?.result.tracing ? `

Braintrust worker export: ${html(entry.result.tracing.status)}; native configuration: ${html(entry.result.tracing.nativeConfiguration)}. ${entry.result.tracing.finalization ? `Supervisor outcome: ${html(entry.result.tracing.finalization.outcome)}; finalization export: ${html(entry.result.tracing.finalization.status)}.` : "Supervisor finalization was not captured."} Trace ${html(entry.result.tracing.traceId)} in project ${html(entry.result.tracing.projectId)}. Indexing and native-span correlation require independent verification.

` : ""} ${links} ${renderFirstTaskDetails(entry?.result)} ${html(execution.id)} @@ -324,7 +334,7 @@ function renderCase( ${latencyRows ? `
${latencyRows}
EventTiming verdictPossible latencyTargetResolution
` : ""} ${startupReuseRows ? `
${startupReuseRows}
Startup reuse evidence
RunWarmthEnvironment/leaseWorkspaceRunnerProvider processProvider sessionProvider launches
` : ""} ${matcherRows ? `
${matcherRows}
ResultMatcherExpectationDetail
` : `

No matcher result was recorded.

`} - ${entry ? `
Usage and billing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null }, null, 2))}
` : ""} + ${entry ? `
Usage, billing and timing metadata
${html(JSON.stringify({ billing, rawUsage: entry.result.usage ?? null, workerTiming: entry.result.workerTiming ?? null }, null, 2))}
` : ""} `; } @@ -342,7 +352,7 @@ function renderSuiteResults(input: { ? `
Pass rate${summary.selected > 0 ? ((summary.passed / summary.selected) * 100).toFixed(1) : "0.0"}%${summary.passed}/${summary.selected} passed${summary.incomplete ? ` · ${summary.incomplete} incomplete` : ""}
Tokens${html(tokenLabel(summary.billing.llm.totalTokens, summary.billing.llm))}${html(tokenLabel(summary.billing.llm.inputTokens, summary.billing.llm))} input · ${html(tokenLabel(summary.billing.llm.outputTokens, summary.billing.llm))} output
-
Known spend${html(summary.billing.llm.runsWithReportedCost > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}reported LLM + runtime${summary.billing.judge ? " + judge" : ""} estimate
+
Known spend${html(summary.billing.llm.runsWithReportedCost > 0 || (summary.billing.llm.runsWithEstimatedCost ?? 0) > 0 || summary.billing.estimatedRuntimeCostUsd > 0 || summary.billing.judge || summary.billing.assistant ? `${usdLabel(summary.billing.observedAndEstimatedCostUsd)}${summary.billing.testsWithCompleteBilling < summary.billing.testCount ? " (partial)" : ""}` : "Unavailable")}reported LLM + model/runtime estimates${summary.billing.judge ? " + judge" : ""} estimate
Agent time${html(durationLabel(summary.billing.agentRunDurationMs))}${html(durationLabel(summary.billing.leaseDurationMs))} lease
Execution${summary.executed}/${summary.selected}${summary.retries} retries · cleanup ${summary.cleanupPassed ? "passed" : "failed"}
` @@ -386,15 +396,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { !entry.valid && isIncompleteFirstTaskResult(entry.result, entry.errors), ).length; const failed = input.expected.length - passed - incomplete; - const totalDuration = selectedEntries.reduce( - (total, entry) => total + entry.result.durationMs, - 0, - ); + const timing = summarizeCampaignTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting); const screenshotCount = selectedEntries.reduce( (total, entry) => total + resolveScreenshots(entry).length, 0, ); - const campaignBilling = aggregateCampaignBilling( + const campaignFeedback = summarizeCampaignFeedback(input.campaign?.accounting?.attempts ?? input.entries.map(entry => entry.result)); + const campaignBilling = input.campaign?.billing ?? aggregateCampaignBilling( selectedEntries.map((entry) => entry.result), ); const suites = [ @@ -834,7 +842,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${passed}/${input.expected.length}Passed
${failed}Failed
${incomplete ? `
${incomplete}Incomplete
` : ""} -
${html(durationLabel(totalDuration))}Test time
+
${html(campaignAttemptTimeLabel(timing))}Summed attempt time
@@ -842,18 +850,23 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { ${renderCampaignSuiteSummary(input.campaign?.suites ?? [], catalog, "", false)}
Campaign usage and spend
+
${campaignFeedback.complaints} / ${campaignFeedback.suggestions}Complaints / suggestions · retained attempts${campaignFeedback.review.length} distinct items · ${campaignFeedback.incomplete} attempts with incomplete collection · ${campaignFeedback.failedSubmissions} failed submissions · ${campaignFeedback.unknownSubmissions} unresolved submissions
+ ${input.campaign?.accounting ? `
${input.campaign.accounting.attempts.length}${input.campaign.accounting.scope === "retained_attempts" ? "Retained attempts" : "Selected results only"}${input.campaign.accounting.missingAttempts} missing attempts · ${input.campaign.accounting.excludedArtifacts} out-of-scope artifacts
` : ""}
${html(tokenLabel(campaignBilling.llm.inputTokens, campaignBilling.llm))}Input tokens
${html(tokenLabel(campaignBilling.llm.outputTokens, campaignBilling.llm))}Output tokens
${html(tokenLabel(campaignBilling.llm.cachedInputTokens, campaignBilling.llm))}Cached tokens
${html(llmCostLabel(campaignBilling.reportedLlmCostUsd, campaignBilling.llm))}LLM reported subtotal
+ ${(campaignBilling.llm.runsWithEstimatedCost ?? 0) > 0 ? `
${html(billingCoverageLabel(usdLabel(campaignBilling.estimatedLlmCostUsd ?? 0), campaignBilling.llm.runsWithEstimatedCost ?? 0, campaignBilling.llm.runCount))}LLM estimated subtotal
` : ""}
${html(usdLabel(campaignBilling.estimatedRuntimeCostUsd))}Daytona list estimate
${campaignBilling.judge ? `
${html(usdLabel(campaignBilling.judge.estimatedCostUsd))}Judge estimate · ${campaignBilling.judge.attempts} attempts · ${campaignBilling.judge.attemptsWithUnknownUsage} unknown usage${html(tokenLabel(campaignBilling.judge.inputTokens))} in / ${html(tokenLabel(campaignBilling.judge.outputTokens))} out · $${campaignBilling.judge.reservedCostUsd.toFixed(6)} reserved
` : ""} ${campaignBilling.assistant ? `
${html(usdLabel(campaignBilling.assistant.estimatedCostUsd))}External assistant estimate · ${campaignBilling.assistant.requests} requests${html(tokenLabel(campaignBilling.assistant.inputTokens + campaignBilling.assistant.cachedInputTokens))} in / ${html(tokenLabel(campaignBilling.assistant.outputTokens))} out
` : ""} +
${html(durationLabel(timing.observedWindowMs))}Observed attempt window${timing.retainedAttempts} retained attempts · ${timing.missingAttempts} missing attempts
+

${html(campaignTimingScope)}

${html(durationLabel(campaignBilling.agentRunDurationMs))}Agent execution time
${html(durationLabel(campaignBilling.leaseDurationMs))}Daytona lease time
${campaignBilling.llm.runsWithReportedCost}/${campaignBilling.llm.runCount}Runs provider-priced
-

Model spend is the provider-reported subtotal; unpriced or unavailable runs are excluded, never counted as free. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

-
+

Campaign costs include retained attempts, including failed retries. Result cards show the selected attempt. Provider-reported amounts and model estimates are separate; unpriced or unavailable runs are excluded, never counted as free. Coverage describes captured usage; unobserved setup, subscriptions, tools, CI and storage require additional accounting. Daytona runtime is a public-list-price estimate from captured lease time and pinned resources, before credits, discounts, storage allowance, or invoice adjustments. Local execution has no external runtime meter.

+ ${renderWorkerPhaseTiming(summarizeWorkerPhaseTiming(selectedEntries.map(entry => entry.result), input.campaign?.accounting))}${input.campaign?.operations ? renderEvalOperationsReport(input.campaign.operations) : ""}${input.campaign ? renderCampaignFeedbackReview(input.campaign) + renderBraintrustExperiment(input.campaign) : ""}

All results

${renderJourneyFeatureCoverage(catalog, entryById)}
diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index 4ca6a41d376..8d3daf45bef 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -58,8 +58,12 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner/test/fixtures/copilot-profile-v47-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v48-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v49-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v50-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v51-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v52-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v54-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v55-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v57-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json", "packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs", "packages/paperclip-runner/scripts/cursor-runtime-patch.mjs", diff --git a/tests/runner-e2e/eval-finalization-relay.test.ts b/tests/runner-e2e/eval-finalization-relay.test.ts new file mode 100644 index 00000000000..d2453eb78e8 --- /dev/null +++ b/tests/runner-e2e/eval-finalization-relay.test.ts @@ -0,0 +1,40 @@ +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import { EVAL_FINALIZATION_LIMITS, finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js"; + +it("meters finalization after the worker has closed without changing the original trace or failed outcome", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-finalization-ingestion-")); + try { + const projectId = "11111111-1111-4111-8111-111111111111"; + const input = { receipt: { projectId, traceId: "a".repeat(32), spanId: "b".repeat(16), status: "pending" as const, + verification: "unverified" as const, nativeConfiguration: "configured" as const }, + identity: { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" }, + provenance: { suiteId: "fixture", suiteDefinitionHash: "c".repeat(64), profileId: "fixture", caseId: "case", sourceRevision: "d".repeat(40) }, + result: { startedAt: new Date(Date.now() - 60_000).toISOString(), status: "failed" as const, cleanup: "failed" as const } }; + const before = structuredClone(input); + let sentBytes = 0; + const upstream = vi.fn(async (url, init) => { + expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces"); + expect(init!.headers).toMatchObject({ authorization: "Bearer upstream-private" }); + const body = Buffer.from(init!.body as Uint8Array); sentBytes = body.length; + expect(body.toString()).toContain("paperclip.eval.finalization"); + expect(body.includes(Buffer.from(input.receipt.traceId, "hex"))).toBe(true); + return new Response(null, { status: 200, headers: { "content-type": "application/x-protobuf" } }); + }); + const file = path.join(root, "finalization.jsonl"); + const result = await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream); + expect(result).toMatchObject({ status: "accepted", outcome: "failed" }); + expect(input).toEqual(before); + const raw = await readFile(file, "utf8"), records = raw.trim().split("\n").map(line => JSON.parse(line)); + expect(records[0]).toMatchObject({ campaignId: "campaign", attemptId: "attempt", limits: EVAL_FINALIZATION_LIMITS }); + expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 1, admittedBytes: sentBytes, acceptedRequests: 1 }); + expect(raw).not.toContain("upstream-private"); + expect((await stat(file)).mode & 0o777).toBe(0o600); + expect(await finalizeEvalAttemptThroughRelay({ projectId, apiKey: "upstream-private" }, input, file, upstream)) + .toMatchObject({ status: "failed", errorCode: "finalization_ingestion_unavailable" }); + expect(upstream).toHaveBeenCalledTimes(1); + expect(await readFile(file, "utf8")).toBe(raw); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/eval-finalization-relay.ts b/tests/runner-e2e/eval-finalization-relay.ts new file mode 100644 index 00000000000..db0c2119f84 --- /dev/null +++ b/tests/runner-e2e/eval-finalization-relay.ts @@ -0,0 +1,24 @@ +import { randomBytes } from "node:crypto"; +import { finalizeEvalAttemptTrace, type EvalBraintrustConfig, type EvalFinalizationReceipt } from "./braintrust.js"; +import { startEvalOtlpRelay } from "./eval-otlp-relay.js"; + +export const EVAL_FINALIZATION_LIMITS = { maxRequests: 1, maxRequestBytes: 64 * 1024, + maxTotalBytes: 64 * 1024, maxConcurrent: 1, maxExpandedBytes: 1024 * 1024, maxRecords: 256 } as const; + +export async function finalizeEvalAttemptThroughRelay(config: EvalBraintrustConfig, + input: Parameters[1], receiptPath: string, transport?: typeof fetch): Promise { + let relay: Awaited> | undefined; + let receipt: EvalFinalizationReceipt = { spanId: randomBytes(8).toString("hex"), status: "failed", + outcome: input.result.status === "passed" && input.result.cleanup === "passed" ? "passed" : "failed", verification: "unverified" }; + try { + relay = await startEvalOtlpRelay({ projectId: config.projectId, apiKey: config.apiKey, + campaignId: input.identity.campaignId, attemptId: input.identity.attemptId, receiptPath, + deadlineAt: Date.now() + 30_000, limits: EVAL_FINALIZATION_LIMITS, transport }); + receipt = await finalizeEvalAttemptTrace({ ...config, apiKey: relay.token, endpointBaseUrl: relay.baseUrl }, input); + } catch { receipt.errorCode = "finalization_ingestion_unavailable"; } + finally { + try { await relay?.close(); } + catch { receipt.status = "failed"; receipt.errorCode = "finalization_ingestion_persistence_failed"; } + } + return receipt; +} diff --git a/tests/runner-e2e/eval-gateway.test.ts b/tests/runner-e2e/eval-gateway.test.ts new file mode 100644 index 00000000000..4fa02239ca9 --- /dev/null +++ b/tests/runner-e2e/eval-gateway.test.ts @@ -0,0 +1,315 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { connect } from "node:net"; +import { afterEach, expect, it, vi } from "vitest"; +import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js"; +import { beginEvalAgentPhase, createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js"; +import { evalRequestReservation, EvalUsageStream, forwardEvalRequest, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js"; +const directories: string[] = [], closes: Array<() => Promise> = []; +afterEach(async () => { for (const close of closes.splice(0).reverse()) await close(); for (const dir of directories.splice(0)) await rm(dir, { recursive: true, force: true }); }); +const identity = { executionId: "hello", attemptId: "attempt", phase: "agent" as const }; +const route: EvalGatewayRoute = { provider: "openai", mode: "braintrust", apiKey: "private-upstream-fixture", projectId: "11111111-1111-4111-8111-111111111111", endpointName: "OPENAI_API_KEY", + maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 2000, + price: { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } }; +const usage = { object: "response", status: "completed", id: "provider-response", model: "fixture", usage: { input_tokens: 10, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 5, output_tokens_details: { reasoning_tokens: 3 } } }; +const response = () => Response.json(usage, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS", "x-bt-request-id": "gateway-request", "x-bt-span-id": "gateway-span" } }); +async function fixture(maxUsd = "1", maxConcurrent = 2) { + const dir = await mkdtemp(path.join(tmpdir(), "eval-forward-test-")); directories.push(dir); + const file = path.join(dir, "ledger.jsonl"), deadlineAt = Date.now() + 30_000; + const ledger = await openEvalCostLedger(file, { campaignId: "campaign", maxUsd, deadlineAt: new Date(deadlineAt).toISOString(), maxRequests: 10, maxConcurrent }); + closes.push(() => ledger.close()); + const receipts: EvalGatewayReceipt[] = []; + return { ledger, file, deadlineAt, identity, route, receipts, payload: { model: "fixture", input: "hello", max_output_tokens: 500 }, + persist: async (receipt: EvalGatewayReceipt) => { receipts.push(structuredClone(receipt)); }, onHeaders: vi.fn(), onChunk: vi.fn(async () => {}) }; +} +it("fsyncs reservation before dispatch, pins gateway routing, and replaces reservation with one estimate", async () => { + const f = await fixture(); + let captured: RequestInit | undefined; + let reservedBeforeDispatch = false; + const transport = vi.fn(async (_url, init) => { captured = init; reservedBeforeDispatch = (await readEvalCostLedger(f.file)).state.snapshot().inflight === 1; return response(); }); + const result = await forwardEvalRequest({ ...f, transport }); + expect(reservedBeforeDispatch).toBe(true); + expect(transport.mock.calls[0]![0]).toBe("https://gateway.braintrust.dev/responses"); + expect(captured).toMatchObject({ redirect: "error", headers: { authorization: `Bearer ${route.apiKey}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", "x-bt-project-id": route.projectId, "x-bt-endpoint-name": route.endpointName, "x-bt-parent": `project_id:${route.projectId}` } }); + expect(JSON.parse(captured!.body as string)).toMatchObject({ max_output_tokens: 100, service_tier: "default" }); + expect(result).toMatchObject({ outcome: "succeeded", costUsd: "0.000068000", final: true, usageComplete: true, gatewaySpanId: "gateway-span" }); + expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000068000", heldUsd: "0.000000000", inflight: 0 }); + expect(await readFile(f.file, "utf8")).not.toContain(route.apiKey); +}); +it("stops at the campaign cap before any upstream request", async () => { + const f = await fixture("0.002"), transport = vi.fn(); + expect(evalRequestReservation(route)).toBe("0.003000000"); + await expect(forwardEvalRequest({ ...f, transport })).rejects.toThrow("spending cap"); + expect(transport).not.toHaveBeenCalled(); +}); +it.each(["direct", "braintrust"] as const)("retains request-policy cost provenance without changing the %s cache policy", async mode => { + const f = await fixture(); + const pricedRoute = { ...route, mode, price: { ...route.price, model: "gpt-5.6-sol", inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", + outputUsdPerMillion: "20", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10" } }; + const payload = { ...f.payload, model: pricedRoute.price.model, prompt_cache_retention: "24h", stream: true }; + const body = { ...usage, model: pricedRoute.price.model, usage: { ...usage.usage, input_tokens_details: { cache_write_tokens: 6, cached_tokens: 2 } } }; + const transport = vi.fn(async () => new Response(`data: ${JSON.stringify({ type: "response.completed", response: body })}\n\n`, { + headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY", "x-bt-cached": "MISS" }, + })); + const receipt = await forwardEvalRequest({ ...f, route: pricedRoute, payload, transport }); + expect(JSON.parse(transport.mock.calls[0]![1]!.body as string)).toEqual({ ...payload, max_output_tokens: 100, service_tier: "default" }); + expect(receipt).toMatchObject({ outcome: "succeeded", final: true, costUsd: "0.000138800", usage: { cacheWrite30m: 6, cacheWrite12h: 0, + cacheWriteAttribution: { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: "30m" } } }); + expect(f.receipts.at(-1)).toEqual(receipt); + expect(f.ledger.snapshot()).toMatchObject({ knownUsd: "0.000138800", heldUsd: "0.000000000", inflight: 0 }); +}); +it("retains cancelled and provider-error requests as unknown, including repeated retries", async () => { + const f = await fixture(); + const first = await forwardEvalRequest({ ...f, transport: async () => { throw new Error(route.apiKey); } }); + const second = await forwardEvalRequest({ ...f, transport: async () => Response.json({ error: "private-provider-message" }, { status: 429 }) }); + expect(first).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null }); + expect(second).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null }); + expect(first.requestId).not.toBe(second.requestId); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.006000000", knownUsd: "0.000000000", unknownRequests: 2 }); + expect(JSON.stringify(f.receipts)).not.toMatch(/private-upstream|private-provider-message/); +}); +it.each([ + { model: "other" }, { model: "fixture", background: true }, { model: "fixture", service_tier: "priority" }, + { model: "fixture", tools: [{ type: "web_search" }] }, { model: "fixture", input: [{ type: "input_image", image_url: "https://image.invalid" }] }, + { model: "fixture", speed: "fast" }, { model: "fixture", inference_geo: "us" }, +])("rejects unpriced request shapes before admission", async payload => { + const f = await fixture(), transport = vi.fn(); + await expect(forwardEvalRequest({ ...f, payload, transport })).rejects.toThrow(); + expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0); +}); +it("does not dispatch when the private pre-request receipt cannot be persisted", async () => { + const f = await fixture(), transport = vi.fn(); + await expect(forwardEvalRequest({ ...f, persist: async () => { throw new Error("disk unavailable"); }, transport })).rejects.toThrow("disk unavailable"); + expect(transport).not.toHaveBeenCalled(); + expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 1, heldUsd: "0.003000000" }); +}); +it("parses actual fragmented SSE without summing cumulative counters", async () => { + const f = await fixture(); + const wire = `data: ${JSON.stringify({ type: "response.output_text.delta", delta: "hello 👋" })}\r\n\r\ndata: ${JSON.stringify({ type: "response.completed", response: usage })}\r\n\r\ndata: [DONE]\r\n\r\n`; + const bytes = Buffer.from(wire); + const body = new ReadableStream({ start(controller) { for (let i = 0; i < bytes.length; i += 3) controller.enqueue(bytes.subarray(i, i + 3)); controller.close(); } }); + const result = await forwardEvalRequest({ ...f, transport: async () => new Response(body, { headers: { "content-type": "text/event-stream", "x-bt-used-endpoint": "OPENAI_API_KEY" } }) }); + expect(result).toMatchObject({ costUsd: "0.000068000", final: true }); + expect(f.onChunk.mock.calls.length).toBeGreaterThan(1); + const truncated = new EvalUsageStream(() => {}); + expect(() => truncated.push(Buffer.from('data: {"type":"response.completed"}'), true)).toThrow("incomplete_usage_stream"); +}); +it.each([{ "x-bt-cached": "HIT" }, { "x-bt-used-endpoint": "FOREIGN" }, { "x-bt-failover-to": "fixture/FOREIGN" }])("rejects cached/failover/foreign route evidence without releasing the reservation", async headers => { + const f = await fixture(); + const result = await forwardEvalRequest({ ...f, transport: async () => Response.json(usage, { headers: Object.fromEntries(Object.entries({ "x-bt-used-endpoint": "OPENAI_API_KEY", ...headers }).filter((pair): pair is [string, string] => pair[1] !== undefined)) }) }); + expect(result).toMatchObject({ outcome: "failed", errorCode: "gateway_route_not_qualified", costUsd: null }); + expect(f.onChunk).not.toHaveBeenCalled(); + expect(f.ledger.snapshot().heldUsd).toBe("0.003000000"); +}); +it("uses the loopback gateway through real HTTP while withholding upstream credentials from the client", async () => { + const f = await fixture(); + const transport = vi.fn(async () => response()); + const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close()); + const denied = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", body: "{}" }); + expect(denied.status).toBe(401); expect(transport).not.toHaveBeenCalled(); + const accepted = await fetch(`${gateway.baseUrl}/v1/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}`, "content-type": "application/json" }, body: JSON.stringify(f.payload) }); + expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(usage); + expect(gateway.token).not.toBe(route.apiKey); + expect(accepted.headers.get("x-paperclip-eval-request-id")).toBeTruthy(); + expect([...accepted.headers.values()].join(" ")).not.toContain(route.apiKey); +}); +it.each([ + { status: 400, type: "text/plain", body: "private fixture rejection" }, + { status: 503, type: "text/html", body: "private fixture rejection" }, + { status: 400, type: "application/json", body: "invalid JSON fixture rejection" }, +])("finishes non-JSON upstream errors without truncating the client response: $status $type", async error => { + const f = await fixture(); + const transport = vi.fn().mockResolvedValueOnce(new Response(error.body, { status: error.status, headers: { "content-type": error.type } })) + .mockImplementation(async () => response()); + const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close()); + const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) }); + const rejected = await request(); + expect(rejected.status).toBe(error.status); + expect(await rejected.text()).toBe(error.body); + expect(f.receipts.at(-1)).toMatchObject({ outcome: "failed", errorCode: "upstream_rejected", costUsd: null, usageComplete: false }); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 }); + expect(JSON.stringify(f.receipts)).not.toContain(error.body); + expect(await (await request()).json()).toEqual(usage); + expect(transport).toHaveBeenCalledTimes(2); +}); +it("still fails successful responses with invalid usage JSON and holds their reservation", async () => { + const f = await fixture(); + const result = await forwardEvalRequest({ ...f, transport: async () => new Response("invalid successful response", { + headers: { "content-type": "application/json", "x-bt-used-endpoint": "OPENAI_API_KEY" }, + }) }); + expect(result).toMatchObject({ outcome: "failed", errorCode: "request_failed", costUsd: null }); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 }); +}); + +it("reserves the highest cache-write and long-context rates before dispatch", () => { + expect(evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: "5", + longContext: { aboveInputTokens: 100, rates: { inputUsdPerMillion: "4", cachedReadUsdPerMillion: "2", cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10", outputUsdPerMillion: "15" } } } })).toBe("0.011500000"); + expect(() => evalRequestReservation({ ...route, price: { ...route.price, cacheWrite12hUsdPerMillion: undefined } })).toThrow("both provider cache-write"); +}); +it("propagates cancellation to an in-flight upstream request and retains its reservation", async () => { + const f = await fixture(), controller = new AbortController(); + const transport = vi.fn(async (_url, init) => { + controller.abort(); + init!.signal!.throwIfAborted(); + throw new Error("unreachable"); + }); + const receipt = await forwardEvalRequest({ ...f, signal: controller.signal, transport }); + expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled", final: false, costUsd: null }); + expect(transport).toHaveBeenCalledOnce(); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 }); +}); +it("does not release a reservation for an otherwise final response missing cache-write usage", async () => { + const f = await fixture(), body = structuredClone(usage); + delete (body.usage.input_tokens_details as any).cache_write_tokens; + const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) }); + expect(receipt).toMatchObject({ outcome: "succeeded", usageComplete: false, final: true, costUsd: null }); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1 }); +}); +it.each([{ version: "prices with spaces" }, { model: "model with spaces" }])("rejects invalid ledger price identities before reserving or dispatching", async invalid => { + const f = await fixture(), transport = vi.fn(); + const price = { ...route.price, ...invalid }; + await expect(forwardEvalRequest({ ...f, route: { ...route, price }, payload: { model: price.model }, transport })).rejects.toThrow("price identity"); + expect(transport).not.toHaveBeenCalled(); + expect(f.ledger.snapshot().requests).toHaveLength(0); +}); +it("retains invalid reasoning counters privately and finishes an unknown charge", async () => { + const f = await fixture(), body = structuredClone(usage); + body.usage.output_tokens_details.reasoning_tokens = 6; + const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json(body, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }) }); + expect(receipt).toMatchObject({ outcome: "succeeded", errorCode: "invalid_usage", usageComplete: false, final: true, costUsd: null, + usage: { tokens: { output: 5, reasoning: 6 } } }); + expect(f.receipts.at(-1)).toEqual(receipt); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 }); + const journal = await readFile(f.file, "utf8"); + expect(journal).not.toContain('"reasoning":6'); + expect(journal).toContain('"type":"finish"'); +}); +it("retains malformed provider identifiers without losing the final receipt or ledger finish", async () => { + const f = await fixture(); + const receipt = await forwardEvalRequest({ ...f, transport: async () => Response.json({ ...usage, model: "unexpected model name" }, + { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY", "x-request-id": "request with spaces" } }) }); + expect(receipt).toMatchObject({ providerRequestId: "request with spaces", usage: { model: "unexpected model name" }, final: true, costUsd: null }); + expect(f.receipts.at(-1)).toEqual(receipt); + expect(f.ledger.snapshot()).toMatchObject({ heldUsd: "0.003000000", unknownRequests: 1, inflight: 0 }); +}); +it("admits later HTTP requests after temporary concurrency exhaustion", async () => { + const f = await fixture("1", 1); + let admitted!: () => void, release!: () => void; + const started = new Promise(resolve => { admitted = resolve; }); + const blocked = new Promise(resolve => { release = resolve; }); + const transport = vi.fn(async () => { admitted(); await blocked; return response(); }); + const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close()); + const request = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) }); + const first = request(); await started; + try { + const denied = await request(); + expect(denied.status).toBe(429); expect(transport).toHaveBeenCalledOnce(); + } finally { release(); } + expect(await (await first).json()).toEqual(usage); + const later = await request(); + expect(later.status).toBe(200); expect(await later.json()).toEqual(usage); + expect(transport).toHaveBeenCalledTimes(2); + expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 0 }); +}); +it("rejects malformed, interrupted and unqualified uploads without disabling later valid requests", async () => { + const f = await fixture(), transport = vi.fn(async () => response()); + const gateway = await startEvalGateway({ ...f, transport }); closes.push(() => gateway.close()); + const headers = { authorization: `Bearer ${gateway.token}` }; + for (const body of ["{", JSON.stringify({ model: "other" }), JSON.stringify({ model: "fixture", tools: [{ type: "web_search" }] })]) { + const rejected = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body }); + expect(rejected.status).toBe(400); await rejected.text(); + } + await new Promise((resolve, reject) => { + const socket = connect(Number(new URL(gateway.baseUrl).port), "127.0.0.1"); + socket.on("error", reject); socket.on("close", () => resolve()); socket.resume(); + socket.on("connect", () => socket.end(`POST /responses HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer ${gateway.token}\r\nContent-Length: 100\r\n\r\n{`)); + }); + expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(0); + const valid = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers, body: JSON.stringify(f.payload) }); + expect(valid.status).toBe(200); expect(await valid.json()).toEqual(usage); + expect(transport).toHaveBeenCalledOnce(); expect(f.ledger.snapshot().requests).toHaveLength(1); +}); +it("still disables the HTTP gateway after a receipt persistence failure", async () => { + const f = await fixture(), transport = vi.fn(async () => response()); + const gateway = await startEvalGateway({ ...f, transport, persist: async () => { throw new Error("storage failure"); } }); closes.push(() => gateway.close()); + for (let i = 0; i < 2; i++) { + const denied = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) }); + expect(denied.status).toBe(503); await denied.text(); + } + expect(transport).not.toHaveBeenCalled(); expect(f.ledger.snapshot().requests).toHaveLength(1); +}); +it("applies the request deadline while waiting for downstream backpressure", async () => { + const f = await fixture(); + const receipt = await forwardEvalRequest({ ...f, route: { ...f.route, timeoutMs: 50 }, transport: async () => response(), + onChunk: async (_chunk, signal) => { + signal.throwIfAborted(); + await new Promise((_resolve, reject) => signal.addEventListener("abort", () => reject(signal.reason), { once: true })); + }, + }); + expect(receipt).toMatchObject({ outcome: "cancelled", errorCode: "request_cancelled" }); + expect(f.ledger.snapshot()).toMatchObject({ inflight: 0, unknownRequests: 1, heldUsd: "0.003000000" }); +}); +it("forwards Claude's Messages beta query without admitting unrelated endpoints or queries", async () => { + const f = await fixture(); + const anthropicRoute: EvalGatewayRoute = { ...route, provider: "anthropic", mode: "direct", price: { provider: "anthropic", model: "fixture", + version: "fixture", sourceUrl: "https://provider.invalid/prices", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", + cacheWrite5mUsdPerMillion: "2.5", cacheWrite1hUsdPerMillion: "4" } }; + const body = { type: "message", model: "fixture", usage: { input_tokens: 2, output_tokens: 3, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 } }; + const transport = vi.fn(async () => Response.json(body)); + const gateway = await startEvalGateway({ ...f, route: anthropicRoute, transport }); closes.push(() => gateway.close()); + const request = (pathname: string) => fetch(gateway.baseUrl + pathname, { method: "POST", headers: { "x-api-key": gateway.token, "anthropic-beta": "fixture-beta" }, body: JSON.stringify({ model: "fixture", messages: [] }) }); + expect((await request("/v1/messages?beta=true&extra=true")).status).toBe(404); + const accepted = await request("/v1/messages?beta=true"); + expect(accepted.status).toBe(200); expect(await accepted.json()).toEqual(body); + expect(transport).toHaveBeenCalledOnce(); + expect(transport.mock.calls[0]![0]).toBe("https://api.anthropic.com/v1/messages?beta=true"); + expect(transport.mock.calls[0]![1]!.headers).toMatchObject({ "anthropic-beta": "fixture-beta" }); + expect(f.ledger.snapshot()).toMatchObject({ unknownRequests: 0, inflight: 0 }); +}); + +it("attributes setup at admission, preserves it during execution, and blocks missing phase evidence", async () => { + const f = await fixture(); + const phasePath = path.join(path.dirname(f.file), "phase.json"); + await createEvalAttemptPhase(phasePath, identity); + const env = { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: identity.attemptId }; + let started!: () => void, release!: () => void, calls = 0; + const firstStarted = new Promise(resolve => { started = resolve; }); + const continuation = new Promise(resolve => { release = resolve; }); + const gateway = await startEvalGateway({ ...f, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity), + transport: async () => { + calls++; + if (calls === 1) { started(); await continuation; } + return Response.json({ ...usage, id: `provider-${calls}` }, { headers: { "x-bt-used-endpoint": "OPENAI_API_KEY" } }); + }, + }); + closes.push(() => gateway.close()); + const send = () => fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) }); + const first = send(); + await firstStarted; + await beginEvalAgentPhase(identity.executionId, env); + release(); + expect((await first).status).toBe(200); + expect((await send()).status).toBe(200); + expect(f.ledger.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]); + await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, attemptId: "foreign-attempt" })); + expect((await send()).status).toBe(503); + expect(calls).toBe(2); + expect((await readEvalCostLedger(f.file)).state.snapshot().requests.map(request => request.phase)).toEqual(["setup", "agent"]); +}); + +it("refuses missing or unreviewed attribution checkpoints before request dispatch", async () => { + const f = await fixture(), transport = vi.fn(); + const phasePath = path.join(path.dirname(f.file), "missing-phase.json"); + const gateway = await startEvalGateway({ ...f, transport, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity) }); + closes.push(() => gateway.close()); + const response = await fetch(`${gateway.baseUrl}/responses`, { method: "POST", headers: { authorization: `Bearer ${gateway.token}` }, body: JSON.stringify(f.payload) }); + expect(response.status).toBe(503); + expect(transport).not.toHaveBeenCalled(); + expect(f.ledger.snapshot().requests).toEqual([]); + await expect(beginEvalAgentPhase(identity.executionId, { PAPERCLIP_E2E_GATEWAY_BASE_URL: gateway.baseUrl })).rejects.toThrow("Missing eval phase"); + await createEvalAttemptPhase(phasePath, identity); + await expect(createEvalAttemptPhase(phasePath, identity)).rejects.toThrow(); + await writeFile(phasePath, JSON.stringify({ version: 1, ...identity, providerResponse: "private" })); + await expect(readEvalAttemptPhase(phasePath, identity)).rejects.toThrow("Invalid eval phase"); +}); diff --git a/tests/runner-e2e/eval-gateway.ts b/tests/runner-e2e/eval-gateway.ts new file mode 100644 index 00000000000..72528908662 --- /dev/null +++ b/tests/runner-e2e/eval-gateway.ts @@ -0,0 +1,256 @@ +import { randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import { once } from "node:events"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { estimateProviderCost, ProviderUsageAccumulator, type EvalModelPrice, type ProviderUsageReceipt } from "./provider-usage.js"; +import { EvalAdmissionError, validEvalLedgerId, type EvalChargeIdentity, type openEvalCostLedger } from "./cost-ledger.js"; + +type Ledger = Awaited>; +export interface EvalGatewayRoute { + provider: "openai" | "anthropic"; + mode: "direct" | "braintrust"; + apiKey: string; + projectId?: string; + endpointName?: string; + price: EvalModelPrice; + maximumInputTokens: number; + maximumOutputTokens: number; + timeoutMs: number; +} +export interface EvalGatewayReceipt { + requestId: string; status?: number; route: "direct" | "braintrust"; + gatewayRequestId?: string; gatewaySpanId?: string; providerRequestId?: string; + endpoint?: string; cache?: string; failoverFrom?: string; failoverTo?: string; errorOrigin?: string; + outcome: "succeeded" | "failed" | "cancelled"; errorCode?: string; + usageComplete: boolean; final: boolean; costUsd: string | null; + usage?: ProviderUsageReceipt; +} +const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`; +const safeHeader = (value: string | null) => value && /^[A-Za-z0-9_.:/ -]{1,512}$/.test(value) ? value : undefined; +const row = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +class EvalRequestError extends Error {} + +export function evalRequestReservation(route: EvalGatewayRoute): string { + if (![route.maximumInputTokens, route.maximumOutputTokens, route.timeoutMs].every(n => Number.isSafeInteger(n) && n > 0) + || route.timeoutMs > 300_000 || route.provider !== route.price.provider || !validEvalLedgerId(route.price.version) || !validEvalLedgerId(route.price.model)) throw new Error("Invalid eval gateway limits or price identity"); + const parse = (rate: string) => { + if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid eval gateway price"); + return usdToUnits(rate); + }; + if (route.price.longContext && (!Number.isSafeInteger(route.price.longContext.aboveInputTokens) || route.price.longContext.aboveInputTokens < 1)) throw new Error("Invalid long-context threshold"); + const tiers = [route.price, ...(route.price.longContext ? [route.price.longContext.rates] : [])]; + let maximumInputRate = 0n, maximumOutputRate = 0n; + for (const tier of tiers) { + const writes = route.provider === "openai" ? [tier.cacheWrite30mUsdPerMillion, tier.cacheWrite12hUsdPerMillion] : [tier.cacheWrite5mUsdPerMillion, tier.cacheWrite1hUsdPerMillion]; + if (writes.some(rate => rate === undefined)) throw new Error("Reservation requires both provider cache-write prices"); + for (const rate of [tier.inputUsdPerMillion, tier.cachedReadUsdPerMillion, ...writes]) { + const units = parse(rate!); if (units > maximumInputRate) maximumInputRate = units; + } + const output = parse(tier.outputUsdPerMillion); if (output > maximumOutputRate) maximumOutputRate = output; + } + const numerator = BigInt(route.maximumInputTokens) * maximumInputRate + BigInt(route.maximumOutputTokens) * maximumOutputRate; + return money((numerator + 999_999n) / 1_000_000n); +} + +function boundedPayload(value: unknown, route: EvalGatewayRoute) { + const body = row(value); + if (body.model !== route.price.model) throw new EvalRequestError("unqualified_model"); + if (body.background === true || body.mcp_servers !== undefined || body.service_tier !== undefined && !["auto", "default", "standard_only"].includes(String(body.service_tier))) throw new EvalRequestError("unqualified_billing_mode"); + if (body.speed !== undefined && body.speed !== "standard" || body.inference_geo !== undefined) throw new EvalRequestError("unqualified_billing_mode"); + if (body.tools !== undefined && (!Array.isArray(body.tools) || body.tools.some(tool => { + const type = row(tool).type; + return route.provider === "openai" ? !["function", "custom"].includes(String(type)) : type !== undefined && type !== "custom"; + }))) throw new EvalRequestError("unqualified_hosted_tool"); + const inspect = (item: unknown): void => { + if (Array.isArray(item)) { item.forEach(inspect); return; } + if (!item || typeof item !== "object") return; + const r = row(item); + if (["input_image", "input_audio", "input_video", "image", "audio", "document", "video", "input_file"].includes(String(r.type))) throw new EvalRequestError("unqualified_multimodal_input"); + Object.values(r).forEach(inspect); + }; + inspect(body); + const field = route.provider === "openai" ? "max_output_tokens" : "max_tokens"; + if (body[field] !== undefined && (!Number.isSafeInteger(body[field]) || Number(body[field]) < 1)) throw new EvalRequestError("invalid_output_limit"); + return { ...body, [field]: Math.min(Number(body[field] ?? route.maximumOutputTokens), route.maximumOutputTokens), + service_tier: route.provider === "openai" ? "default" : "standard_only" }; +} + +export class EvalUsageStream { + private readonly decoder = new TextDecoder(); + private pending = ""; + constructor(private readonly observe: (value: unknown) => void) {} + push(bytes: Uint8Array, final = false) { + this.pending += this.decoder.decode(bytes, { stream: !final }); + if (this.pending.length > 4 * 1024 * 1024) throw new Error("usage_frame_too_large"); + for (;;) { + const match = /\r?\n\r?\n/.exec(this.pending); + if (!match) break; + const frame = this.pending.slice(0, match.index); this.pending = this.pending.slice(match.index + match[0].length); + const data = frame.split(/\r?\n/).filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n"); + if (data && data !== "[DONE]") this.observe(JSON.parse(data)); + } + if (final && this.pending.trim()) throw new Error("incomplete_usage_stream"); + } +} + +export async function forwardEvalRequest(input: { + ledger: Ledger; identity: EvalChargeIdentity; route: EvalGatewayRoute; payload: unknown; + deadlineAt: number; signal?: AbortSignal; anthropicBeta?: string; anthropicBetaQuery?: boolean; + onHeaders: (status: number, headers: Record) => void; + onChunk: (chunk: Uint8Array, signal: AbortSignal) => Promise; + persist: (receipt: EvalGatewayReceipt) => Promise; + transport?: typeof fetch; +}): Promise { + const { route, ledger } = input; + let body: ReturnType; + try { body = boundedPayload(input.payload, route); } + catch (error) { throw error instanceof EvalRequestError ? error : new EvalRequestError("invalid_request_payload"); } + const reservation = evalRequestReservation(route); + if (!route.apiKey || /[\r\n]/.test(route.apiKey)) throw new Error("missing_gateway_credential"); + if (route.mode === "braintrust" && (!/^[0-9a-f-]{36}$/i.test(route.projectId ?? "") || !/^[A-Za-z0-9_-]+$/.test(route.endpointName ?? ""))) throw new Error("invalid_project_provider"); + if (input.signal?.aborted) throw new EvalRequestError("request_cancelled"); + if (Date.now() >= input.deadlineAt) throw new EvalAdmissionError("request_deadline_reached"); + const requestId = randomUUID(); + const receipt: EvalGatewayReceipt = { requestId, route: route.mode, outcome: "failed", usageComplete: false, final: false, costUsd: null }; + const usage = new ProviderUsageAccumulator(route.provider, body); + await ledger.reserve(requestId, input.identity, reservation); + const abort = new AbortController(); + const signal = AbortSignal.any([abort.signal, AbortSignal.timeout(Math.max(1, Math.min(route.timeoutMs, input.deadlineAt - Date.now()))), ...(input.signal ? [input.signal] : [])]); + let response: Response | undefined; + try { + await input.persist(receipt); + const headers: Record = { "content-type": "application/json", "accept-encoding": "identity" }; + if (route.mode === "braintrust") Object.assign(headers, { + authorization: `Bearer ${route.apiKey}`, "x-bt-project-id": route.projectId!, "x-bt-endpoint-name": route.endpointName!, + "x-bt-parent": `project_id:${route.projectId}`, "x-bt-use-cache": "never", "cache-control": "no-cache, no-store", + }); + else if (route.provider === "openai") headers.authorization = `Bearer ${route.apiKey}`; + else headers["x-api-key"] = route.apiKey; + if (route.provider === "anthropic") { + headers["anthropic-version"] = "2023-06-01"; + if (input.anthropicBeta && /^[A-Za-z0-9, _-]{1,2048}$/.test(input.anthropicBeta)) headers["anthropic-beta"] = input.anthropicBeta; + } + const base = route.mode === "braintrust" ? "https://gateway.braintrust.dev" : route.provider === "openai" ? "https://api.openai.com/v1" : "https://api.anthropic.com/v1"; + const endpoint = route.provider === "openai" ? "/responses" : input.anthropicBetaQuery ? "/messages?beta=true" : "/messages"; + response = await (input.transport ?? fetch)(base + endpoint, { method: "POST", headers, body: JSON.stringify(body), redirect: "error", signal }); + receipt.status = response.status; + receipt.gatewayRequestId = safeHeader(response.headers.get("x-bt-request-id")); + receipt.gatewaySpanId = safeHeader(response.headers.get("x-bt-span-id")); + receipt.providerRequestId = safeHeader(response.headers.get("x-request-id") ?? response.headers.get("request-id")); + receipt.endpoint = safeHeader(response.headers.get("x-bt-used-endpoint")); + receipt.cache = safeHeader(response.headers.get("x-bt-cached")); + receipt.failoverFrom = safeHeader(response.headers.get("x-bt-failover-from")); + receipt.failoverTo = safeHeader(response.headers.get("x-bt-failover-to")); + receipt.errorOrigin = safeHeader(response.headers.get("x-bt-error-origin")); + await input.persist(receipt); + if (route.mode === "braintrust" && (receipt.cache === "HIT" || receipt.failoverFrom || receipt.failoverTo + || response.ok && receipt.endpoint !== route.endpointName)) throw new Error("gateway_route_not_qualified"); + const contentType = response.headers.get("content-type") ?? "application/json"; + input.onHeaders(response.status, { "content-type": contentType, "cache-control": "no-store", "x-paperclip-eval-request-id": requestId }); + const reader = response.body?.getReader(); + const stream = contentType.startsWith("text/event-stream") ? new EvalUsageStream(value => usage.observe(value)) : undefined; + const chunks: Uint8Array[] = []; let size = 0; + try { + while (reader) { + const next = await reader.read(); if (next.done) break; + size += next.value.length; if (size > 32 * 1024 * 1024) throw new Error("response_too_large"); + if (stream) stream.push(next.value); else chunks.push(next.value); + await input.onChunk(next.value, signal); + } + if (stream) stream.push(new Uint8Array(), true); + else if (size) { + try { usage.observe(JSON.parse(Buffer.concat(chunks).toString("utf8"))); } + catch (error) { if (response.ok) throw error; } + } + } finally { await reader?.cancel().catch(() => {}); } + receipt.outcome = response.ok ? "succeeded" : "failed"; + if (!response.ok) receipt.errorCode = "upstream_rejected"; + } catch (error) { + receipt.outcome = signal.aborted ? "cancelled" : "failed"; + const code = error instanceof Error ? error.message : ""; + receipt.errorCode = ["gateway_route_not_qualified", "response_too_large", "usage_frame_too_large", "incomplete_usage_stream"].includes(code) ? code : signal.aborted ? "request_cancelled" : "request_failed"; + } finally { + abort.abort(); + await response?.body?.cancel().catch(() => {}); + const observed = usage.receipt(); + receipt.usage = observed; + receipt.usageComplete = observed.complete; + receipt.final = observed.final; + receipt.costUsd = estimateProviderCost(observed, route.price); + const tokens = { ...observed.tokens }; + if (tokens.reasoning !== undefined && tokens.output !== undefined && tokens.reasoning > tokens.output) { + delete tokens.reasoning; + receipt.errorCode ??= "invalid_usage"; + } + await input.persist(receipt); + await ledger.observe({ observationId: `${requestId}:usage`, requestId, source: route.mode === "braintrust" ? "gateway" : "provider", + provider: route.provider, + ...(validEvalLedgerId(observed.model) ? { model: observed.model } : {}), + ...(validEvalLedgerId(receipt.providerRequestId) ? { providerRequestId: receipt.providerRequestId } : {}), + currency: "USD", route: route.mode, classification: receipt.costUsd === null ? "unknown" : "estimated", + ...(receipt.costUsd !== null ? { costUsd: receipt.costUsd, pricingVersion: route.price.version } : {}), tokens, final: observed.final }); + await ledger.finish(requestId, receipt.outcome); + } + return receipt; +} + +async function requestJson(req: IncomingMessage) { + const chunks: Buffer[] = []; let size = 0; + for await (const chunk of req) { + size += chunk.length; if (size > 4 * 1024 * 1024) throw new Error("request_too_large"); + chunks.push(chunk); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")); +} + +export async function startEvalGateway(input: Omit[0], "payload" | "onHeaders" | "onChunk" | "signal" | "anthropicBeta" | "anthropicBetaQuery"> & { + resolveIdentity?: () => Promise; +}) { + const token = randomBytes(32).toString("hex"), pending = new Set>(), controllers = new Set(); + let closed = false, poisoned = false; + const respond = (res: ServerResponse, status: number, code: string) => { + if (res.destroyed) return; + if (!res.headersSent) { res.writeHead(status, { "content-type": "application/json" }); res.end(JSON.stringify({ error: { type: "eval_gateway", message: code } })); } + else res.destroy(); + }; + const server = createServer((req, res) => { + const operation = (async () => { + const auth = String(req.headers.authorization ?? "").replace(/^Bearer /, "") || String(req.headers["x-api-key"] ?? ""); + if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) { respond(res, 401, "unauthorized"); return; } + const endpoint = input.route.provider === "openai" ? "responses" : "messages"; + if (closed || poisoned) { respond(res, 503, "gateway_unavailable"); return; } + const paths = [`/${endpoint}`, `/v1/${endpoint}`]; + if (input.route.provider === "anthropic") paths.push("/messages?beta=true", "/v1/messages?beta=true"); + if (req.method !== "POST" || !paths.includes(req.url ?? "")) { respond(res, 404, "unqualified_endpoint"); return; } + const abort = new AbortController(); controllers.add(abort); + const onClose = () => { if (!res.writableEnded) abort.abort(); }; res.on("close", onClose); + try { + let payload: unknown; + try { payload = await requestJson(req); } + catch { respond(res, 400, "invalid_request_body"); return; } + const identity = input.resolveIdentity ? await input.resolveIdentity() : input.identity; + if (identity.executionId !== input.identity.executionId || identity.attemptId !== input.identity.attemptId) throw new Error("Invalid eval request attribution"); + const receipt = await forwardEvalRequest({ ...input, identity, payload, signal: abort.signal, + anthropicBeta: typeof req.headers["anthropic-beta"] === "string" ? req.headers["anthropic-beta"] : undefined, + anthropicBetaQuery: input.route.provider === "anthropic" && req.url?.endsWith("?beta=true"), + onHeaders: (status, headers) => { res.writeHead(status, headers); }, + onChunk: async (chunk, signal) => { if (!res.write(chunk)) await once(res, "drain", { signal }); }, + }); + if (receipt.outcome === "succeeded" || receipt.errorCode === "upstream_rejected") res.end(); + else respond(res, 502, receipt.errorCode ?? "request_failed"); + } catch (error) { + if (error instanceof EvalRequestError) { respond(res, 400, "invalid_request_payload"); return; } + if (error instanceof EvalAdmissionError) { respond(res, 429, "eval_admission_denied"); return; } + poisoned = true; respond(res, 503, "eval_admission_or_persistence_failed"); + } finally { controllers.delete(abort); res.off("close", onClose); } + })(); + pending.add(operation); void operation.then(() => pending.delete(operation), () => { poisoned = true; respond(res, 503, "gateway_unavailable"); pending.delete(operation); }); + }); + server.requestTimeout = 15_000; server.headersTimeout = 10_000; + server.listen(0, "127.0.0.1"); await once(server, "listening"); + const address = server.address(); if (!address || typeof address === "string") throw new Error("Eval gateway did not bind loopback"); + return { baseUrl: `http://127.0.0.1:${address.port}`, token, + async close() { closed = true; controllers.forEach(controller => controller.abort()); server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); await Promise.allSettled([...pending]); }, + }; +} diff --git a/tests/runner-e2e/eval-otlp-payload.ts b/tests/runner-e2e/eval-otlp-payload.ts new file mode 100644 index 00000000000..345734f73ec --- /dev/null +++ b/tests/runner-e2e/eval-otlp-payload.ts @@ -0,0 +1,127 @@ +import { evalOtlpSchema as root, protobuf } from "./eval-otlp-schema.js"; + +export type EvalOtlpSignal = "traces" | "logs"; +export const isEvalProjectId = (value: string) => /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value); + +export function validateEvalOtlpParent(value: string, projectId: string): string { + if (!isEvalProjectId(projectId)) throw new Error("invalid_otlp_project"); + if (value === `project_id:${projectId}`) return value; + if (value.length > 2048 || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) throw new Error("invalid_otlp_parent"); + const bytes = Buffer.from(value, "base64"); + if (bytes.toString("base64") !== value || !bytes.subarray(0, 3).equals(Buffer.from([4, 2, 0]))) throw new Error("invalid_otlp_parent"); + const data = JSON.parse(bytes.subarray(3).toString("utf8")); + if (!data || Array.isArray(data) || Object.keys(data).sort().join(",") !== "object_id,root_span_id,row_id,span_id" + || data.object_id !== projectId || data.row_id !== data.span_id + || !/^[0-9a-f]{16}$/.test(data.span_id) || /^0+$/.test(data.span_id) + || !/^[0-9a-f]{32}$/.test(data.root_span_id) || /^0+$/.test(data.root_span_id)) throw new Error("invalid_otlp_parent"); + return value; +} + +function validateWire(body: Buffer, type: import("../../server/node_modules/protobufjs/index.js").Type) { + const reader = protobuf.Reader.create(body); + let fields = 0; + const inspect = (message: typeof type, end: number, depth: number): void => { + if (depth > 64) throw new Error("otlp_structure_limit"); + const seen = new Set(); + while (reader.pos < end) { + if (++fields > 200_000) throw new Error("otlp_structure_limit"); + const tag = reader.uint32(), id = tag >>> 3, wire = tag & 7, field = message.fieldsById[id]; + if (!id || ![0, 1, 2, 5].includes(wire)) throw new Error("invalid_otlp_wire"); + if (field) { + if (!field.repeated && seen.has(id) || field.partOf && seen.has(field.partOf.name)) throw new Error("ambiguous_otlp_wire"); + seen.add(id); if (field.partOf) seen.add(field.partOf.name); + const nested = field.resolvedType instanceof protobuf.Type ? field.resolvedType : undefined; + const expected = nested || ["string", "bytes"].includes(field.type) ? 2 + : ["double", "fixed64", "sfixed64"].includes(field.type) ? 1 + : ["float", "fixed32", "sfixed32"].includes(field.type) ? 5 : 0; + if (wire !== expected) throw new Error("invalid_otlp_wire"); + if (nested) { + const length = reader.uint32(), next = reader.pos + length; + if (next > end) throw new Error("invalid_otlp_wire"); + inspect(nested, next, depth + 1); + } else reader.skipType(wire); + } else throw new Error("unqualified_otlp_field"); + if (reader.pos > end) throw new Error("invalid_otlp_wire"); + } + }; + inspect(type, body.length, 0); +} + +export function validateEvalOtlpPayload(body: Buffer, contentType: string, signal: EvalOtlpSignal, projectId: string) { + const type = root.lookupType(signal === "traces" ? "TracesData" : "LogsData"); + if (contentType !== "application/json") validateWire(body, type); + const decoded: unknown = contentType === "application/json" ? JSON.parse(body.toString("utf8")) : type.decode(body).toJSON(); + if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_payload"); + let visited = 0; + const inspect = (value: unknown, depth: number): void => { + if (++visited > 200_000 || depth > 64) throw new Error("otlp_structure_limit"); + if (Array.isArray(value)) { value.forEach(item => inspect(item, depth + 1)); return; } + if (!value || typeof value !== "object") return; + const row = value as Record; + if (row.key === "braintrust.parent") { + const parent = row.value as { stringValue?: unknown; string_value?: unknown } | undefined; + if (!parent || Object.keys(parent).length !== 1) throw new Error("invalid_otlp_parent"); + const text = parent.stringValue ?? parent.string_value; + if (typeof text !== "string") throw new Error("invalid_otlp_parent"); + validateEvalOtlpParent(text, projectId); + } + if (Object.hasOwn(row, "braintrust.parent")) { + if (typeof row["braintrust.parent"] !== "string") throw new Error("invalid_otlp_parent"); + validateEvalOtlpParent(row["braintrust.parent"], projectId); + } + Object.values(row).forEach(item => inspect(item, depth + 1)); + }; + inspect(decoded, 0); + const canonical = Buffer.from(JSON.stringify(decoded)); + const sizing = expandedEvalOtlpSize(decoded as Record, signal); + return { body: contentType === "application/json" ? canonical : body, ...sizing }; +} + +function expandedEvalOtlpSize(decoded: Record, signal: EvalOtlpSignal) { + const bytes = (value: unknown) => Buffer.byteLength(JSON.stringify(value)); + const object = (value: unknown): Record => { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_otlp_record"); + return value as Record; + }; + const take = (row: Record, camel: string, snake = camel) => { + if (camel !== snake && Object.hasOwn(row, camel) && Object.hasOwn(row, snake)) throw new Error("ambiguous_otlp_collection"); + const value = Object.hasOwn(row, camel) ? row[camel] : Object.hasOwn(row, snake) ? row[snake] : []; + if (!Array.isArray(value)) throw new Error("invalid_otlp_collection"); + const rest = Object.fromEntries(Object.entries(row).filter(([key]) => key !== camel && key !== snake)); + return { rows: value, rest }; + }; + const traces = signal === "traces"; + const resources = take(decoded, traces ? "resourceSpans" : "resourceLogs", traces ? "resource_spans" : "resource_logs"); + let recordCount = 0, expandedBytes = bytes(decoded); + for (const resource of resources.rows) { + const scopes = take(object(resource), traces ? "scopeSpans" : "scopeLogs", traces ? "scope_spans" : "scope_logs"); + for (const scope of scopes.rows) { + const records = traces ? take(object(scope), "spans") : take(object(scope), "logRecords", "log_records"); + const envelopeBytes = bytes({ source: resources.rest, resource: scopes.rest, scope: records.rest, record: {} }) - 2; + for (const record of records.rows) { + recordCount++; + expandedBytes += envelopeBytes + bytes(object(record)); + } + } + } + return { recordCount, expandedBytes }; +} + +export function evalOtlpRejected(body: Buffer, contentType: string, signal: EvalOtlpSignal): number { + if (!body.length) return 0; + const decoded = contentType === "application/json" ? JSON.parse(body.toString("utf8")) + : root.lookupType("Response").decode(body).toJSON(); + if (!decoded || typeof decoded !== "object" || Array.isArray(decoded)) throw new Error("invalid_otlp_response"); + const partial = decoded.partialSuccess; + if (partial !== undefined && (!partial || typeof partial !== "object" || Array.isArray(partial))) throw new Error("invalid_otlp_response"); + const rejected = partial?.[signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"] ?? partial?.rejected ?? 0; + if (!/^[0-9]+$/.test(String(rejected)) || !Number.isSafeInteger(Number(rejected))) throw new Error("invalid_otlp_response"); + return Number(rejected); +} + +export function evalOtlpResponse(rejected: number, contentType: string, signal: EvalOtlpSignal): Buffer { + if (contentType === "application/json") return Buffer.from(JSON.stringify(rejected + ? { partialSuccess: { [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: String(rejected) } } : {})); + const type = root.lookupType("Response"); + return Buffer.from(type.encode(type.fromObject(rejected ? { partialSuccess: { rejected: String(rejected) } } : {})).finish()); +} diff --git a/tests/runner-e2e/eval-otlp-relay.test.ts b/tests/runner-e2e/eval-otlp-relay.test.ts new file mode 100644 index 00000000000..d241934242a --- /dev/null +++ b/tests/runner-e2e/eval-otlp-relay.test.ts @@ -0,0 +1,258 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, open, readFile, rm, stat } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { createServer } from "node:http"; +import { once } from "node:events"; +import { gzipSync } from "node:zlib"; +import { setTimeout as delay } from "node:timers/promises"; +import { EVAL_OTLP_LIMITS, startEvalOtlpRelay } from "./eval-otlp-relay.js"; +import { EvalAttemptTrace, exportBraintrustAttemptParent } from "./braintrust.js"; +import { evalOtlpRejected, evalOtlpResponse } from "./eval-otlp-payload.js"; +import { evalOtlpSchema } from "./eval-otlp-schema.js"; + +const projectId = "11111111-1111-4111-8111-111111111111", apiKey = "upstream-private-fixture"; +const protobuf = createRequire(new URL("../../server/package.json", import.meta.url))("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js"); +const message = (field: number, ...parts: Uint8Array[]) => Buffer.from(protobuf.Writer.create().uint32(field * 8 + 2).bytes(Buffer.concat(parts)).finish()); +const roots: string[] = [], closers: Array<() => Promise> = []; +afterEach(async () => { await Promise.allSettled(closers.splice(0).map(close => close())); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); vi.restoreAllMocks(); }); +async function fixture(overrides: Partial[0]> = {}) { + const root = await mkdtemp(path.join(tmpdir(), "eval-otlp-")); roots.push(root); + const policy = { projectId, apiKey, campaignId: "campaign", attemptId: "attempt", receiptPath: path.join(root, "receipts.jsonl"), + deadlineAt: Date.now() + 60_000, ...overrides }; + const relay = await startEvalOtlpRelay(policy); closers.push(relay.close); + const send = (body = "{}", headers: Record = {}, endpoint = "/v1/traces") => fetch(relay.baseUrl + endpoint, { + method: "POST", headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json", ...headers }, body, + }); + const records = async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + return { relay, policy, send, records }; +} + +it("forwards real attempt protobuf over HTTP using only the controller credential and retains body-free receipts", async () => { + const received: Array<{ authorization?: string; parent?: string; body: Buffer }> = []; + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; for await (const chunk of req) chunks.push(Buffer.from(chunk)); + received.push({ authorization: req.headers.authorization, parent: req.headers["x-bt-parent"] as string, body: Buffer.concat(chunks) }); + res.writeHead(200, { "content-type": "application/x-protobuf" }); res.end(); + }); + server.listen(0, "127.0.0.1"); await once(server, "listening"); + closers.push(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(() => resolve())); }); + const address = server.address(); if (!address || typeof address === "string") throw new Error("fixture_bind_failed"); + const upstream = vi.fn((url, init) => { + expect(String(url)).toBe("https://api.braintrust.dev/otel/v1/traces"); + expect(init?.redirect).toBe("error"); + return fetch(`http://127.0.0.1:${address.port}`, init); + }); + const f = await fixture({ transport: upstream }); + const trace = new EvalAttemptTrace({ projectId, apiKey: f.relay.token, endpointBaseUrl: f.relay.baseUrl }, + { campaignId: "campaign", attemptId: "attempt", executionId: "fixture.local", provider: "fixture", model: "fixture", environment: "local" }); + trace.phase("setup"); + expect(await trace.export({ failed: false, runIds: [], cleanup: "passed", complaints: 0, suggestions: 0 })).toMatchObject({ status: "accepted" }); + expect(received).toHaveLength(1); + expect(received[0]).toMatchObject({ authorization: `Bearer ${apiKey}`, parent: `project_id:${projectId}` }); + expect(received[0]!.body.toString()).toContain("paperclip.eval.attempt"); + expect(received[0]!.body.toString()).not.toContain(f.relay.token); + await f.relay.close(); + const records = await f.records(); + expect(records.map(row => row.type)).toEqual(["policy", "admitted", "outcome", "closed"]); + expect(records[2]).toMatchObject({ outcome: "accepted", dispatched: true, httpStatus: 200, rejectedRecords: 0 }); + expect(records[3]).toMatchObject({ admittedRequests: 1, admittedBytes: received[0]!.body.length, + attemptedForwardBytes: received[0]!.body.length, acceptedRequests: 1, incompleteRequests: 0 }); + expect(JSON.stringify(records)).not.toMatch(/upstream-private-fixture|paperclip.eval.attempt/); + expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600); + await expect(startEvalOtlpRelay(f.policy)).rejects.toMatchObject({ code: "EEXIST" }); +}); + +it("refuses token, endpoint, header and payload routing bypasses before contacting Braintrust", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream }); + expect((await f.send("{}", { authorization: `Bearer ${apiKey}` })).status).toBe(401); + expect((await f.send("{}", {}, "/../api/project")).status).toBe(404); + expect((await f.send("{}", {}, "/v1/traces?target=other")).status).toBe(404); + expect((await f.send("{}", { "x-bt-parent": "project_name:another" })).status).toBe(400); + for (const attributes of [ + { resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent", value: { stringValue: "project_id:22222222-2222-4222-8222-222222222222" } }] } }] }, + { resourceSpans: [{ scopeSpans: [{ spans: [{ attributes: [{ key: "braintrust.parent", value: { stringValue: "experiment_id:private" } }] }] }] }] }, + ]) expect((await f.send(JSON.stringify(attributes))).status).toBe(400); + expect(upstream).not.toHaveBeenCalled(); + const parent = exportBraintrustAttemptParent({ projectId, spanId: "1234567890abcdef", traceId: "abcdef0123456789abcdef0123456789" }); + expect((await f.send("{}", { "x-bt-parent": parent })).status).toBe(200); + expect(upstream.mock.calls[0]?.[1]?.headers).toMatchObject({ "x-bt-parent": parent }); +}); + +it("counts decompressed payload bytes and refuses a compressed body above its allowance", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRequestBytes: 1024 } }); + const body = JSON.stringify({ resourceLogs: [{ resource: { attributes: [{ key: "test", value: { stringValue: "private-content-canary" } }] } }] }); + const request = (bytes: Buffer) => fetch(f.relay.baseUrl + "/v1/logs", { method: "POST", body: Buffer.from(bytes), + headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/json", "content-encoding": "gzip" } }); + expect((await request(gzipSync(body))).status).toBe(200); + expect((await request(gzipSync(JSON.stringify({ payload: "x".repeat(2048) })))).status).toBe(400); + expect(upstream).toHaveBeenCalledTimes(1); + expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).toBe(body); + const records = await f.records(); + expect(records[1]).toMatchObject({ signal: "logs", bodyBytes: Buffer.byteLength(body), wireBytes: gzipSync(body).length }); + expect(JSON.stringify(records)).not.toContain("private-content-canary"); +}); + +it("rejects protobuf routing metadata and ambiguous singular fields before forwarding", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream }); + const attribute = message(1, Buffer.from("braintrust.parent")); + const foreign = message(2, message(1, Buffer.from("project_id:22222222-2222-4222-8222-222222222222"))); + const local = message(2, message(1, Buffer.from(`project_id:${projectId}`))); + const resource = message(1, message(1, attribute, foreign)); + const bodies = [ + message(1, resource), + message(1, resource, message(1)), + message(1, message(1, message(1, attribute, foreign, local))), + message(1, message(2, message(2, message(9, attribute, foreign)))), + message(1, message(2, message(1, message(3, attribute, foreign)))), + ]; + for (const body of bodies) { + const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body), + headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } }); + expect(response.status).toBe(400); + } + expect(upstream).not.toHaveBeenCalled(); +}); + +it("canonicalizes duplicate JSON keys and refuses conflicting parent value aliases", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream }); + const foreign = "project_id:22222222-2222-4222-8222-222222222222"; + const body = `{"resourceSpans":[{"resource":{"attributes":[{"key":"braintrust.parent","value":{"stringValue":"${foreign}","stringValue":"project_id:${projectId}"}}]}}]}`; + expect((await f.send(body)).status).toBe(200); + expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array).toString()).not.toContain(foreign); + const ambiguous = JSON.stringify({ resourceSpans: [{ resource: { attributes: [{ key: "braintrust.parent", + value: { stringValue: `project_id:${projectId}`, string_value: foreign } }] } }] }); + expect((await f.send(ambiguous)).status).toBe(400); + expect(upstream).toHaveBeenCalledTimes(1); +}); + +it("does not dispatch when durable admission fails", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream }); + const handle = await open(path.join(path.dirname(f.policy.receiptPath), "fixture"), "w"); + const sync = vi.spyOn(Object.getPrototypeOf(handle), "sync").mockRejectedValueOnce(new Error("fixture_disk_failure")); + try { + expect((await f.send()).status).toBe(503); + expect((await f.send()).status).toBe(503); + expect(upstream).not.toHaveBeenCalled(); + await expect(f.relay.close()).rejects.toThrow("fixture_disk_failure"); + } finally { sync.mockRestore(); await handle.close(); } +}); + +it("bounds simultaneous exports without retrying the refused request", async () => { + let entered!: () => void, release!: () => void; + const active = new Promise(resolve => { entered = resolve; }); + const wait = new Promise(resolve => { release = resolve; }); + const upstream = vi.fn(async () => { entered(); await wait; return new Response(null, { status: 200 }); }); + const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxConcurrent: 1 } }); + const first = f.send(); await active; + try { expect((await f.send()).status).toBe(429); } finally { release(); } + expect((await first).status).toBe(200); + expect(upstream).toHaveBeenCalledTimes(1); +}); + +it.each(["count", "bytes"])("retains failed attempts against the %s allowance without retrying or resetting", async kind => { + const upstream = vi.fn(async () => new Response("private-vendor-error", { status: 503 })); + const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, + ...(kind === "count" ? { maxRequests: 1 } : { maxTotalBytes: 3 }) } }); + const first = await f.send(); expect(first.status).toBe(502); expect(await first.text()).not.toContain("private-vendor-error"); + expect((await f.send()).status).toBe(429); + expect((await f.send()).status).toBe(429); + expect(upstream).toHaveBeenCalledTimes(1); + await f.relay.close(); + const records = await f.records(); + expect(records.filter(row => row.type === "quota_exhausted")).toHaveLength(1); + expect(records.find(row => row.type === "outcome")).toMatchObject({ outcome: "rejected", httpStatus: 503 }); + expect(records.at(-1)).toMatchObject({ admittedRequests: 1, admittedBytes: 2, incompleteRequests: 1 }); +}); + +it.each(["traces", "logs"] as const)("preserves partial %s rejection without returning upstream diagnostic text", async signal => { + const upstream = vi.fn(async () => Response.json({ partialSuccess: { + [signal === "traces" ? "rejectedSpans" : "rejectedLogRecords"]: "2", errorMessage: "private-upstream-diagnostic" } })); + const f = await fixture({ transport: upstream }); + const response = await f.send("{}", {}, `/v1/${signal}`); + expect(response.status).toBe(200); + const text = await response.text(); expect(text).not.toContain("private-upstream-diagnostic"); + expect(evalOtlpRejected(Buffer.from(text), "application/json", signal)).toBe(2); + const protobuf = evalOtlpResponse(2, "application/x-protobuf", signal); + expect(evalOtlpRejected(protobuf, "application/x-protobuf", signal)).toBe(2); + expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "partial", rejectedRecords: 2 }); +}); + +it("aborts an in-flight export on close and retains its uncertain outcome", async () => { + let started!: () => void, aborted = false; + const active = new Promise(resolve => { started = resolve; }); + const upstream = vi.fn(async (_url, init) => { + started(); await new Promise((_resolve, reject) => init!.signal!.addEventListener("abort", () => { aborted = true; reject(new Error("fixture_abort")); }, { once: true })); + return new Response(null, { status: 200 }); + }); + const f = await fixture({ transport: upstream }); + const request = f.send().catch(() => undefined); await active; + await f.relay.close(); await request; + expect(aborted).toBe(true); + expect((await f.records()).find(row => row.type === "outcome")).toMatchObject({ outcome: "uncertain", dispatched: true }); + await f.relay.close(); + expect(upstream).toHaveBeenCalledTimes(1); +}); + +it("retires the listening endpoint at its deadline without a supervisor close", async () => { + const upstream = vi.fn(async () => new Response(null, { status: 200 })); + const f = await fixture({ transport: upstream, deadlineAt: Date.now() + 120 }); + await delay(180); + await expect(f.send()).rejects.toThrow(); + await f.relay.close(); + expect(upstream).not.toHaveBeenCalled(); + expect((await f.records()).at(-1)).toMatchObject({ type: "closed", admittedRequests: 0 }); +}); + +it.each(["json", "protobuf"])("meters repeated resource/scope data and full record fields in %s before dispatch", async encoding => { + const upstream = vi.fn(async () => Response.json({})); + const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxExpandedBytes: 40_000 } }); + const payload = { resourceSpans: [{ resource: { attributes: [{ key: "shared", value: { stringValue: "r".repeat(3000) } }] }, + scopeSpans: [{ scope: { name: "s".repeat(1000), version: "fixture" }, spans: Array.from({ length: 4 }, () => ({ + name: "n".repeat(1000), traceId: "AQEBAQEBAQEBAQEBAQEBAQ==", spanId: "AgICAgICAgI=", flags: 1, + startTimeUnixNano: "1234567890123456789", endTimeUnixNano: "1234567890123456790", + events: [{ name: "e".repeat(1000), timeUnixNano: "1234567890123456789" }], + links: [{ traceId: "AwMDAwMDAwMDAwMDAwMDAw==", spanId: "BAQEBAQEBAQ=", flags: 1 }], + })) }] }] }; + const type = evalOtlpSchema.lookupType("TracesData"); + const body = encoding === "json" ? Buffer.from(JSON.stringify(payload)) : Buffer.from(type.encode(type.fromObject(payload)).finish()); + const send = () => fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body: Buffer.from(body), + headers: { authorization: `Bearer ${f.relay.token}`, "content-type": encoding === "json" ? "application/json" : "application/x-protobuf" } }); + expect(body.length).toBeLessThan(20_000); + expect((await send()).status).toBe(200); + expect((await send()).status).toBe(429); + expect(upstream).toHaveBeenCalledTimes(1); + expect(Buffer.from(upstream.mock.calls[0]![1]!.body as Uint8Array)).toEqual(body); + await f.relay.close(); + const records = await f.records(), admitted = records.find(row => row.type === "admitted"); + expect(admitted.recordCount).toBe(4); + expect(admitted.expandedBytes).toBeGreaterThan(35_000); + expect(records.at(-1)).toMatchObject({ admittedRecords: 4, admittedExpandedBytes: admitted.expandedBytes }); +}); + +it("accepts thousands of small native records while bounding their total count", async () => { + const upstream = vi.fn(async () => Response.json({})); + const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRecords: 3000 } }); + const body = JSON.stringify({ resource_logs: [{ scope_logs: [{ log_records: Array.from({ length: 2533 }, () => ({ body: { stringValue: "event" } })) }] }] }); + expect((await f.send(body, {}, "/v1/logs")).status).toBe(200); + expect((await f.send(body, {}, "/v1/logs")).status).toBe(429); + expect(upstream).toHaveBeenCalledTimes(1); + expect((await f.records()).find(row => row.type === "admitted").recordCount).toBe(2533); +}); + +it("rejects unknown protobuf fields instead of excluding forwarded data from sizing", async () => { + const upstream = vi.fn(async () => Response.json({})); + const f = await fixture({ transport: upstream }); + const body = message(1, message(2, message(2, message(100, Buffer.from("unknown-content"))))); + const response = await fetch(f.relay.baseUrl + "/v1/traces", { method: "POST", body, + headers: { authorization: `Bearer ${f.relay.token}`, "content-type": "application/x-protobuf" } }); + expect(response.status).toBe(400); + expect(upstream).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/eval-otlp-relay.ts b/tests/runner-e2e/eval-otlp-relay.ts new file mode 100644 index 00000000000..36e61d9dde4 --- /dev/null +++ b/tests/runner-e2e/eval-otlp-relay.ts @@ -0,0 +1,139 @@ +import { createHash, randomBytes, randomUUID, timingSafeEqual } from "node:crypto"; +import { once } from "node:events"; +import { mkdir, open } from "node:fs/promises"; +import { createServer, type ServerResponse } from "node:http"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { evalOtlpRejected, evalOtlpResponse, isEvalProjectId, validateEvalOtlpParent, validateEvalOtlpPayload, type EvalOtlpSignal } from "./eval-otlp-payload.js"; + +export const EVAL_OTLP_LIMITS = { maxRequests: 2048, maxRequestBytes: 8 * 1024 * 1024, + maxTotalBytes: 64 * 1024 * 1024, maxConcurrent: 4, + maxExpandedBytes: 256 * 1024 * 1024, maxRecords: 16_384 } as const; + +export async function startEvalOtlpRelay(input: { + projectId: string; apiKey: string; campaignId: string; attemptId: string; receiptPath: string; deadlineAt: number; + limits?: { [K in keyof typeof EVAL_OTLP_LIMITS]: number }; transport?: typeof fetch; +}) { + const limits = input.limits ?? EVAL_OTLP_LIMITS; + if (!isEvalProjectId(input.projectId) || !input.apiKey.trim() || /[\r\n]/.test(input.apiKey) + || ![input.campaignId, input.attemptId].every(value => /^[A-Za-z0-9_.:-]{1,512}$/.test(value)) + || !path.isAbsolute(input.receiptPath) || !Number.isFinite(input.deadlineAt) + || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 4 * 3_600_000 + || Object.keys(limits).sort().join(",") !== Object.keys(EVAL_OTLP_LIMITS).sort().join(",") + || Object.entries(EVAL_OTLP_LIMITS).some(([key, maximum]) => !Number.isSafeInteger(limits[key as keyof typeof limits]) + || limits[key as keyof typeof limits] < 1 || limits[key as keyof typeof limits] > maximum)) throw new Error("Invalid eval OTLP relay policy"); + await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 }); + const file = await open(input.receiptPath, "ax", 0o600); + let writes = Promise.resolve(), poisoned = false, closed = false, quotaRecorded = false; + const record = (value: object) => { + writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); }) + .catch(error => { poisoned = true; throw error; }); + return writes; + }; + try { await record({ type: "policy", version: 2, sizing: "otlp-v1.9-expanded-source-json", projectId: input.projectId, campaignId: input.campaignId, + attemptId: input.attemptId, limits, deadlineAt: new Date(input.deadlineAt).toISOString(), openedAt: new Date().toISOString() }); } + catch (error) { await file.close(); throw error; } + const token = randomBytes(32).toString("hex"); + const pending = new Set>(), controllers = new Set(); + const usage = { admittedRequests: 0, admittedWireBytes: 0, admittedBytes: 0, admittedExpandedBytes: 0, + admittedRecords: 0, attemptedForwardBytes: 0, acceptedRequests: 0, incompleteRequests: 0 }; + const deny = (response: ServerResponse, status: number, code: string) => { + if (response.destroyed) return; + response.writeHead(status, { "content-type": "application/json", "cache-control": "no-store" }); + response.end(JSON.stringify({ error: { type: "eval_otlp_relay", message: code } })); + }; + const server = createServer((request, response) => { + const operation = (async () => { + const auth = typeof request.headers.authorization === "string" ? request.headers.authorization.replace(/^Bearer /, "") : ""; + if (Buffer.byteLength(auth) !== Buffer.byteLength(token) || !timingSafeEqual(Buffer.from(auth), Buffer.from(token))) return deny(response, 401, "unauthorized"); + if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable"); + const signal: EvalOtlpSignal | undefined = request.url === "/otel/v1/traces" ? "traces" : request.url === "/otel/v1/logs" ? "logs" : undefined; + if (request.method !== "POST" || !signal) return deny(response, 404, "unqualified_endpoint"); + if (pending.size >= limits.maxConcurrent) return deny(response, 429, "concurrency_limit"); + const contentType = request.headers["content-type"]?.split(";")[0].trim(); + const encoding = request.headers["content-encoding"]; + if (!["application/json", "application/x-protobuf"].includes(contentType ?? "") || encoding && encoding !== "gzip" && encoding !== "identity") return deny(response, 415, "unqualified_encoding"); + let body: Buffer, parent: string, wireBytes = 0, expandedBytes = 0, recordCount = 0; + try { + parent = validateEvalOtlpParent(String(request.headers["x-bt-parent"] ?? `project_id:${input.projectId}`), input.projectId); + const chunks: Buffer[] = []; + for await (const chunk of request) { + wireBytes += chunk.length; + if (wireBytes > limits.maxRequestBytes) throw new Error("request_limit"); + chunks.push(Buffer.from(chunk)); + } + body = Buffer.concat(chunks); + if (encoding === "gzip") body = gunzipSync(body, { maxOutputLength: limits.maxRequestBytes }); + const validated = validateEvalOtlpPayload(body, contentType!, signal, input.projectId); + ({ body, expandedBytes, recordCount } = validated); + if (body.length > limits.maxRequestBytes) throw new Error("request_limit"); + } catch { return deny(response, 400, "invalid_or_oversized_payload"); } + if (closed || poisoned || Date.now() >= input.deadlineAt) return deny(response, 503, "relay_unavailable"); + if (usage.admittedRequests >= limits.maxRequests || usage.admittedBytes + body.length > limits.maxTotalBytes + || usage.admittedExpandedBytes + expandedBytes > limits.maxExpandedBytes + || usage.admittedRecords + recordCount > limits.maxRecords) { + if (!quotaRecorded) { quotaRecorded = true; await record({ type: "quota_exhausted", at: new Date().toISOString(), ...usage }); } + return deny(response, 429, "ingestion_allowance_exhausted"); + } + const id = randomUUID(); + usage.admittedRequests++; usage.admittedWireBytes += wireBytes; usage.admittedBytes += body.length; + usage.admittedExpandedBytes += expandedBytes; usage.admittedRecords += recordCount; + await record({ type: "admitted", id, signal, at: new Date().toISOString(), wireBytes, bodyBytes: body.length, + expandedBytes, recordCount, + payloadSha256: createHash("sha256").update(body).digest("hex") }); + const abort = new AbortController(); controllers.add(abort); + const disconnected = () => { if (!response.writableEnded) abort.abort(); }; + response.on("close", disconnected); + let status: number | undefined, outcome = "uncertain", rejected: number | undefined, dispatched = false; + try { + if (closed || poisoned || response.destroyed || Date.now() >= input.deadlineAt) throw new Error("relay_unavailable"); + dispatched = true; usage.attemptedForwardBytes += body.length; + const upstream = await (input.transport ?? fetch)(`https://api.braintrust.dev/otel/v1/${signal}`, { + method: "POST", redirect: "error", signal: AbortSignal.any([abort.signal, + AbortSignal.timeout(Math.max(1, Math.min(10_000, input.deadlineAt - Date.now())))]), + headers: { authorization: `Bearer ${input.apiKey}`, "x-bt-parent": parent, "content-type": contentType!, + "accept-encoding": "identity", accept: "application/x-protobuf, application/json" }, body: Buffer.from(body), + }); + status = upstream.status; + if (!upstream.ok) { await upstream.body?.cancel(); outcome = "rejected"; throw new Error("receiver_rejected"); } + const type = upstream.headers.get("content-type")?.split(";")[0].trim() ?? contentType!; + if (!["application/json", "application/x-protobuf"].includes(type)) { await upstream.body?.cancel(); throw new Error("invalid_response"); } + const reader = upstream.body?.getReader(), chunks: Uint8Array[] = []; let bytes = 0; + try { + while (reader) { + const next = await reader.read(); if (next.done) break; + bytes += next.value.length; if (bytes > 65_536) throw new Error("response_limit"); + chunks.push(next.value); + } + } finally { await reader?.cancel().catch(() => {}); } + rejected = evalOtlpRejected(Buffer.concat(chunks), type, signal); + outcome = rejected ? "partial" : "accepted"; + } catch { if (!dispatched) outcome = "not_dispatched"; } + finally { controllers.delete(abort); response.off("close", disconnected); } + if (outcome === "accepted") usage.acceptedRequests++; else usage.incompleteRequests++; + await record({ type: "outcome", id, at: new Date().toISOString(), outcome, dispatched, + ...(status === undefined ? {} : { httpStatus: status }), ...(rejected === undefined ? {} : { rejectedRecords: rejected }) }); + if (response.destroyed) return; + if (outcome !== "accepted" && outcome !== "partial") return deny(response, 502, "export_unconfirmed"); + response.writeHead(200, { "content-type": contentType!, "cache-control": "no-store" }); + response.end(evalOtlpResponse(rejected!, contentType!, signal)); + })(); + pending.add(operation); + void operation.then(() => pending.delete(operation), () => { poisoned = true; pending.delete(operation); deny(response, 503, "receipt_persistence_failed"); }); + }); + server.requestTimeout = 15_000; server.headersTimeout = 10_000; server.maxConnections = limits.maxConcurrent + 8; + try { server.listen(0, "127.0.0.1"); await once(server, "listening"); } + catch (error) { await file.close(); throw error; } + const address = server.address(); + if (!address || typeof address === "string") { await file.close(); throw new Error("Eval OTLP relay did not bind loopback"); } + let closing: Promise | undefined; + const close = () => closing ??= (async () => { + closed = true; clearTimeout(timer); controllers.forEach(controller => controller.abort()); server.closeAllConnections(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + await Promise.allSettled([...pending]); + try { await record({ type: "closed", at: new Date().toISOString(), ...usage, poisoned }); } + finally { await file.close(); } + })(); + const timer = setTimeout(() => { void close().catch(() => {}); }, Math.max(1, input.deadlineAt - Date.now())); timer.unref(); + return { baseUrl: `http://127.0.0.1:${address.port}/otel`, token, receiptPath: input.receiptPath, close }; +} diff --git a/tests/runner-e2e/eval-otlp-schema.ts b/tests/runner-e2e/eval-otlp-schema.ts new file mode 100644 index 00000000000..a277cbd7f11 --- /dev/null +++ b/tests/runner-e2e/eval-otlp-schema.ts @@ -0,0 +1,170 @@ +import { createRequire } from "node:module"; +const require = createRequire(new URL("../../server/package.json", import.meta.url)); +export const protobuf = require("protobufjs") as typeof import("../../server/node_modules/protobufjs/index.js"); +export const evalOtlpSchema = protobuf.parse(`syntax = "proto3"; +message AnyValue { +oneof value { +string string_value = 1; +bool bool_value = 2; +int64 int_value = 3; +double double_value = 4; +ArrayValue array_value = 5; +KeyValueList kvlist_value = 6; +bytes bytes_value = 7; +} +} +message ArrayValue { +repeated AnyValue values = 1; +} +message KeyValueList { +repeated KeyValue values = 1; +} +message KeyValue { +string key = 1; +AnyValue value = 2; +} +message InstrumentationScope { +string name = 1; +string version = 2; +repeated KeyValue attributes = 3; +uint32 dropped_attributes_count = 4; +} +message EntityRef { +string schema_url = 1; +string type = 2; +repeated string id_keys = 3; +repeated string description_keys = 4; +} +message Resource { +repeated KeyValue attributes = 1; +uint32 dropped_attributes_count = 2; +repeated EntityRef entity_refs = 3; +} +message TracesData { +repeated ResourceSpans resource_spans = 1; +} +message ResourceSpans { +Resource resource = 1; +repeated ScopeSpans scope_spans = 2; +string schema_url = 3; +} +message ScopeSpans { +InstrumentationScope scope = 1; +repeated Span spans = 2; +string schema_url = 3; +} +message Span { +bytes trace_id = 1; +bytes span_id = 2; +string trace_state = 3; +bytes parent_span_id = 4; +fixed32 flags = 16; +string name = 5; +enum SpanKind { +SPAN_KIND_UNSPECIFIED = 0; +SPAN_KIND_INTERNAL = 1; +SPAN_KIND_SERVER = 2; +SPAN_KIND_CLIENT = 3; +SPAN_KIND_PRODUCER = 4; +SPAN_KIND_CONSUMER = 5; +} +SpanKind kind = 6; +fixed64 start_time_unix_nano = 7; +fixed64 end_time_unix_nano = 8; +repeated KeyValue attributes = 9; +uint32 dropped_attributes_count = 10; +message Event { +fixed64 time_unix_nano = 1; +string name = 2; +repeated KeyValue attributes = 3; +uint32 dropped_attributes_count = 4; +} +repeated Event events = 11; +uint32 dropped_events_count = 12; +message Link { +bytes trace_id = 1; +bytes span_id = 2; +string trace_state = 3; +repeated KeyValue attributes = 4; +uint32 dropped_attributes_count = 5; +fixed32 flags = 6; +} +repeated Link links = 13; +uint32 dropped_links_count = 14; +Status status = 15; +} +message Status { +string message = 2; +enum StatusCode { +STATUS_CODE_UNSET = 0; +STATUS_CODE_OK = 1; +STATUS_CODE_ERROR = 2; +}; +StatusCode code = 3; +} +enum SpanFlags { +SPAN_FLAGS_DO_NOT_USE = 0; +SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; +SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100; +SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200; +} +message LogsData { +repeated ResourceLogs resource_logs = 1; +} +message ResourceLogs { +Resource resource = 1; +repeated ScopeLogs scope_logs = 2; +string schema_url = 3; +} +message ScopeLogs { +InstrumentationScope scope = 1; +repeated LogRecord log_records = 2; +string schema_url = 3; +} +enum SeverityNumber { +SEVERITY_NUMBER_UNSPECIFIED = 0; +SEVERITY_NUMBER_TRACE = 1; +SEVERITY_NUMBER_TRACE2 = 2; +SEVERITY_NUMBER_TRACE3 = 3; +SEVERITY_NUMBER_TRACE4 = 4; +SEVERITY_NUMBER_DEBUG = 5; +SEVERITY_NUMBER_DEBUG2 = 6; +SEVERITY_NUMBER_DEBUG3 = 7; +SEVERITY_NUMBER_DEBUG4 = 8; +SEVERITY_NUMBER_INFO = 9; +SEVERITY_NUMBER_INFO2 = 10; +SEVERITY_NUMBER_INFO3 = 11; +SEVERITY_NUMBER_INFO4 = 12; +SEVERITY_NUMBER_WARN = 13; +SEVERITY_NUMBER_WARN2 = 14; +SEVERITY_NUMBER_WARN3 = 15; +SEVERITY_NUMBER_WARN4 = 16; +SEVERITY_NUMBER_ERROR = 17; +SEVERITY_NUMBER_ERROR2 = 18; +SEVERITY_NUMBER_ERROR3 = 19; +SEVERITY_NUMBER_ERROR4 = 20; +SEVERITY_NUMBER_FATAL = 21; +SEVERITY_NUMBER_FATAL2 = 22; +SEVERITY_NUMBER_FATAL3 = 23; +SEVERITY_NUMBER_FATAL4 = 24; +} +enum LogRecordFlags { +LOG_RECORD_FLAGS_DO_NOT_USE = 0; +LOG_RECORD_FLAGS_TRACE_FLAGS_MASK = 0x000000FF; +} +message LogRecord { +fixed64 time_unix_nano = 1; +fixed64 observed_time_unix_nano = 11; +SeverityNumber severity_number = 2; +string severity_text = 3; +AnyValue body = 5; +repeated KeyValue attributes = 6; +uint32 dropped_attributes_count = 7; +fixed32 flags = 8; +bytes trace_id = 9; +bytes span_id = 10; +string event_name = 12; +} +message PartialSuccess { int64 rejected = 1; } +message Response { PartialSuccess partial_success = 1; } +`).root.resolveAll(); diff --git a/tests/runner-e2e/experiment-ingestion.test.ts b/tests/runner-e2e/experiment-ingestion.test.ts new file mode 100644 index 00000000000..984bcb91a78 --- /dev/null +++ b/tests/runner-e2e/experiment-ingestion.test.ts @@ -0,0 +1,108 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { EVAL_EXPERIMENT_LIMITS, openExperimentIngestion } from "./experiment-ingestion.js"; + +const projectId = "11111111-1111-4111-8111-111111111111", experimentId = "22222222-2222-4222-8222-222222222222"; +const roots: string[] = [], meters: Array>> = []; +afterEach(async () => { await Promise.allSettled(meters.splice(0).map(meter => meter.close())); + await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture(transport: typeof fetch = async () => Response.json({ ok: true })) { + const root = await mkdtemp(path.join(tmpdir(), "experiment-ingestion-")); roots.push(root); + const policy = { projectId, campaignId: "campaign", deadlineAt: Date.now() + 60_000, receiptPath: path.join(root, "receipts.jsonl"), transport }; + const meter = await openExperimentIngestion(policy); meters.push(meter); + return { ...meter, policy, records: async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)), + send: (payload: unknown, endpoint = "experiment") => meter.transport(`https://api.braintrust.dev/v1/${endpoint}`, { + method: "POST", body: JSON.stringify(payload), headers: { authorization: "Bearer private-secret" }, + }) }; +} + +it("durably meters payloads and scores before dispatch while retaining body-free private evidence", async () => { + const upstream = vi.fn(async (_url, init) => { + expect((await f.records()).at(-1)).toMatchObject({ type: "admitted", bodyBytes: init?.body === undefined ? 0 : Buffer.byteLength(String(init.body)) }); + expect(init!.redirect).toBe("error"); + return Response.json({ ok: true }); + }); + const f = await fixture(upstream); + const create = { project_id: projectId, name: "private-payload-canary" }; + const insert = { events: [{ scores: { product_pass: 0, cleanup_pass: 0 }, input: "private-input-canary" }] }; + await f.send(create); await f.send(insert, `experiment/${experimentId}/insert`); + await f.transport(`https://api.braintrust.dev/v1/experiment/${experimentId}/summarize`, { method: "GET" }); + await f.close(); + const records = await f.records(); + expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 3, admittedScores: 2, + admittedBytes: Buffer.byteLength(JSON.stringify(create)) + Buffer.byteLength(JSON.stringify(insert)), unresolvedRequests: 0 }); + const raw = await readFile(f.policy.receiptPath, "utf8"); + expect(raw).not.toMatch(/private-secret|private-payload-canary|private-input-canary/); + expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600); + await expect(openExperimentIngestion(f.policy)).rejects.toMatchObject({ code: "EEXIST" }); + expect(await readFile(f.policy.receiptPath, "utf8")).toBe(raw); +}); + +it("keeps failed forwarding charged to the finite allowance and records an unresolved outcome", async () => { + const upstream = vi.fn(async () => { throw new Error("private-upstream-error"); }); + const f = await fixture(upstream); + for (let i = 0; i < EVAL_EXPERIMENT_LIMITS.maxRequests; i++) { + await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed"); + } + await expect(f.send({ project_id: projectId })).rejects.toThrow("allowance exhausted"); + expect(upstream).toHaveBeenCalledTimes(3); + await f.close(); + expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, responses: 0, unresolvedRequests: 3 }); +}); + +it("retains an unresolved allowance when response headers arrive but the body disconnects", async () => { + const f = await fixture(async () => new Response(new ReadableStream({ pull(controller) { + controller.error(new Error("private-disconnect-error")); + } }), { status: 200 })); + await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed"); + await f.close(); + const rows = await f.records(); + expect(rows[2]).toMatchObject({ type: "outcome", outcome: "unconfirmed" }); + expect(rows.at(-1)).toMatchObject({ admittedRequests: 1, responses: 0, unresolvedRequests: 1 }); + expect(JSON.stringify(rows)).not.toContain("private-disconnect-error"); +}); + +it("keeps a pending response body unresolved until the whole response is read", async () => { + let finish!: () => void; + const f = await fixture(async () => new Response(new ReadableStream({ start(controller) { + controller.enqueue(new TextEncoder().encode('{"ok":')); + finish = () => { controller.enqueue(new TextEncoder().encode('true}')); controller.close(); }; + } }))); + const pending = f.send({ project_id: projectId }); + await vi.waitFor(() => expect(finish).toBeTypeOf("function")); + expect((await f.records()).at(-1)).toMatchObject({ type: "admitted" }); + await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); + finish(); + expect(await (await pending).json()).toEqual({ ok: true }); + await f.close(); + expect((await f.records()).at(-1)).toMatchObject({ responses: 1, unresolvedRequests: 0 }); +}); + +it("refuses excess bytes, scores, foreign project and unqualified endpoints before dispatch", async () => { + const upstream = vi.fn(async () => Response.json({})); + const f = await fixture(upstream); + await expect(f.send({ project_id: experimentId })).rejects.toThrow("project mismatch"); + await expect(f.send({ project_id: projectId, data: "x".repeat(EVAL_EXPERIMENT_LIMITS.maxTotalBytes) })).rejects.toThrow("byte limit"); + await expect(f.send({ events: Array.from({ length: 9 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`)) + .rejects.toThrow("allowance exhausted"); + await expect(f.send({}, "project")).rejects.toThrow("Unqualified"); + expect(upstream).not.toHaveBeenCalled(); + await f.close(); + expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 0, admittedBytes: 0, admittedScores: 0 }); +}); + +it("refuses concurrent requests and expired windows without resetting reservations", async () => { + let release!: () => void; + const upstream = vi.fn(async () => { await new Promise(resolve => { release = resolve; }); return Response.json({}); }); + const f = await fixture(upstream); + const pending = f.send({ project_id: projectId }); + await vi.waitFor(() => expect(upstream).toHaveBeenCalledOnce()); + await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); + release(); await pending; + const now = vi.spyOn(Date, "now").mockReturnValue(f.policy.deadlineAt); + try { await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); } + finally { now.mockRestore(); } + expect(upstream).toHaveBeenCalledOnce(); +}); diff --git a/tests/runner-e2e/experiment-ingestion.ts b/tests/runner-e2e/experiment-ingestion.ts new file mode 100644 index 00000000000..7cb1c102bbe --- /dev/null +++ b/tests/runner-e2e/experiment-ingestion.ts @@ -0,0 +1,91 @@ +import { createHash, randomUUID } from "node:crypto"; +import { mkdir, open } from "node:fs/promises"; +import path from "node:path"; +import { isEvalProjectId } from "./eval-otlp-payload.js"; + +export const EVAL_EXPERIMENT_LIMITS = { maxRequests: 3, maxTotalBytes: 4 * 1024 * 1024, maxScores: 8 } as const; + +export async function openExperimentIngestion(input: { + receiptPath: string; campaignId: string; projectId: string; deadlineAt: number; transport?: typeof fetch; +}) { + if (!path.isAbsolute(input.receiptPath) || !/^[A-Za-z0-9_.:-]{1,512}$/.test(input.campaignId) + || !isEvalProjectId(input.projectId) || !Number.isFinite(input.deadlineAt) + || input.deadlineAt <= Date.now() || input.deadlineAt > Date.now() + 60_000) throw new Error("Invalid experiment ingestion policy"); + await mkdir(path.dirname(input.receiptPath), { recursive: true, mode: 0o700 }); + const file = await open(input.receiptPath, "ax", 0o600); + let writes = Promise.resolve(), closed = false, poisoned = false, active = false; + let admittedRequests = 0, admittedBytes = 0, admittedScores = 0, responses = 0; + const record = (value: object) => { + writes = writes.then(async () => { await file.writeFile(JSON.stringify(value) + "\n"); await file.sync(); }) + .catch(error => { poisoned = true; throw error; }); + return writes; + }; + try { await record({ type: "policy", version: 1, kind: "experiment", projectId: input.projectId, campaignId: input.campaignId, + limits: EVAL_EXPERIMENT_LIMITS, openedAt: new Date().toISOString(), deadlineAt: new Date(input.deadlineAt).toISOString() }); } + catch (error) { await file.close(); throw error; } + const transport: typeof fetch = async (target, init) => { + if (closed || poisoned || active || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable"); + const url = String(target), method = init?.method; + const create = url === "https://api.braintrust.dev/v1/experiment" && method === "POST"; + const insert = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/insert$/.test(url) && method === "POST"; + const summarize = /^https:\/\/api\.braintrust\.dev\/v1\/experiment\/[0-9a-f-]{36}\/summarize$/.test(url) && method === "GET"; + if (!create && !insert && !summarize || summarize && init?.body !== undefined + || !summarize && typeof init?.body !== "string") throw new Error("Unqualified experiment ingestion request"); + const body = summarize ? "" : init!.body as string; + const bytes = Buffer.byteLength(body); + if (bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes) throw new Error("Experiment ingestion byte limit"); + let scores = 0; + if (body) { + const payload = JSON.parse(body); + if (create && payload.project_id !== input.projectId) throw new Error("Experiment ingestion project mismatch"); + if (insert) { + if (!Array.isArray(payload.events)) throw new Error("Invalid experiment ingestion events"); + scores = payload.events.reduce((sum: number, event: { scores?: object }) => sum + Object.keys(event.scores ?? {}).length, 0); + } + } + if (admittedRequests >= EVAL_EXPERIMENT_LIMITS.maxRequests || admittedBytes + bytes > EVAL_EXPERIMENT_LIMITS.maxTotalBytes + || admittedScores + scores > EVAL_EXPERIMENT_LIMITS.maxScores) throw new Error("Experiment ingestion allowance exhausted"); + active = true; + const id = randomUUID(); + admittedRequests++; admittedBytes += bytes; admittedScores += scores; + try { + await record({ type: "admitted", id, at: new Date().toISOString(), operation: create ? "create" : insert ? "insert" : "summarize", + bodyBytes: bytes, scores, payloadSha256: createHash("sha256").update(body).digest("hex") }); + if (closed || poisoned || Date.now() >= input.deadlineAt) throw new Error("Experiment ingestion unavailable"); + let response: Response; + try { + const upstream = await (input.transport ?? fetch)(target, { ...init, redirect: "error", signal: AbortSignal.any([ + AbortSignal.timeout(Math.max(1, Math.min(15_000, input.deadlineAt - Date.now()))), ...(init?.signal ? [init.signal] : []), + ]) }); + const reader = upstream.body?.getReader(), chunks: Uint8Array[] = []; + let responseBytes = 0; + try { + while (reader) { + const next = await reader.read(); + if (next.done) break; + responseBytes += next.value.length; + if (responseBytes > 1_048_576) throw new Error("Oversized experiment response"); + chunks.push(next.value); + } + response = new Response(upstream.body === null ? null : Buffer.concat(chunks), { status: upstream.status, headers: upstream.headers }); + } finally { await reader?.cancel().catch(() => {}); } + } catch { + await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "unconfirmed" }); + throw new Error("Experiment ingestion response unconfirmed"); + } + try { await record({ type: "outcome", id, at: new Date().toISOString(), outcome: "response", httpStatus: response.status }); } + catch (error) { await response.body?.cancel().catch(() => {}); throw error; } + responses++; + return response; + } finally { active = false; } + }; + let closing: Promise | undefined; + return { transport, close: () => closing ??= (async () => { + closed = true; + try { + if (active) throw new Error("Experiment ingestion is still in flight"); + await record({ type: "closed", at: new Date().toISOString(), admittedRequests, admittedBytes, admittedScores, + responses, unresolvedRequests: admittedRequests - responses }); + } finally { await file.close(); } + })() }; +} diff --git a/tests/runner-e2e/feedback-dispositions.test.ts b/tests/runner-e2e/feedback-dispositions.test.ts new file mode 100644 index 00000000000..743ac95fb88 --- /dev/null +++ b/tests/runner-e2e/feedback-dispositions.test.ts @@ -0,0 +1,129 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { emptyFeedbackSummary } from "./feedback.js"; +import { buildRunnerCampaign } from "./history.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; +import { campaignFeedbackEvidence, campaignFeedbackReviewText, feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js"; +import type { RunnerE2EResult } from "./types.js"; + +function fixture() { + const cell = runnerMatrix[0]!; + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, suiteId: cell.suite.id, + suiteDefinitionHash: cell.suiteDefinitionHash, attempt: 1, status: "failed", profileId: cell.profile.id, + environmentId: cell.environment.id, caseId: cell.task.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode, + runIds: [], startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed", + feedback: { ...emptyFeedbackSummary(), collection: "partial", invocationCollection: "complete", complaints: 2, suggestions: 1, bodiesCollected: 3, + review: ["a", "b", "c"].map(value => ({ fingerprint: value.repeat(64), kind: value === "c" ? "suggestion" : "complaint", occurrences: 1, disposition: "needs_investigation" })) } }; + return buildRunnerCampaign({ campaignId: "feedback-campaign", generatedAt: result.finishedAt, expected: [cell.id], results: [result] }); +} +function reviewed(campaign = fixture()): CampaignFeedbackReview { + return { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "agent", + decisions: [ + { fingerprint: "a".repeat(64), disposition: "actionable", issueNumber: 123 }, + { fingerprint: "b".repeat(64), disposition: "duplicate", duplicateOf: "a".repeat(64) }, + { fingerprint: "c".repeat(64), disposition: "unsupported" }, + ] }; +} + +it("keeps an unreviewed template invalid until a reviewer records the decision metadata", () => { + const campaign = fixture(); + expect(() => validateCampaignFeedbackReview(campaign, feedbackReviewTemplate(campaign) as never)).toThrow("Invalid or stale"); + expect(campaignFeedbackReviewText(campaign)).toContain("3 distinct feedback items await review"); + validateCampaignFeedbackReview(campaign, reviewed(campaign)); +}); + +it("preserves dispositions on regeneration without changing failed grades or incomplete collection", async () => { + const campaign = fixture(); campaign.feedbackReview = reviewed(campaign); + const root = await mkdtemp(path.join(tmpdir(), "feedback-review-")); + try { + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify({ ...campaign, feedbackReview: undefined })); + const retainedHeader = "# Existing campaign\n\nProvider-reported LLM cost: $0.123456\n\n"; + await writeFile(path.join(root, "summary.md"), `${retainedHeader}${campaignFeedbackReviewText({ ...campaign, feedbackReview: undefined })}\n`); + const feedbackReviewFile = path.join(root, "review.json"); + await writeFile(feedbackReviewFile, JSON.stringify(campaign.feedbackReview)); + await regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile }); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + const retained = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")); + expect(retained.feedbackReview).toEqual(campaign.feedbackReview); + expect(retained.results[0].status).toBe("failed"); + expect(retained.accounting.attempts[0].feedback.collection).toBe("partial"); + const html = await readFile(path.join(root, "dashboard.html"), "utf8"); + expect(html).toContain("1 actionable; 1 duplicate; 0 already fixed; 1 unsupported; 0 needs investigation"); + expect(html).toContain("1 attempts have incomplete collection"); + expect(html).toContain("https://github.com/paperclipai/paperclip/issues/123"); + const summary = await readFile(path.join(root, "summary.md"), "utf8"); + expect(summary).toBe(`${retainedHeader}${campaignFeedbackReviewText(campaign)}\n`); + expect(summary).not.toContain("await review"); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("rejects stale reviews when attempts or collection evidence change, but permits a render refresh", () => { + const campaign = fixture(), review = reviewed(campaign); + campaign.generatedAt = "2026-10-21T00:00:00Z"; + validateCampaignFeedbackReview(campaign, review); + const changed = structuredClone(campaign); + changed.accounting!.attempts[0]!.feedback!.collection = "complete"; + expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale"); + changed.accounting = structuredClone(campaign.accounting); + changed.accounting!.missingAttempts++; + expect(() => validateCampaignFeedbackReview(changed, review)).toThrow("Invalid or stale"); + expect(campaignFeedbackEvidence({ ...campaign, expected: [...campaign.expected].reverse() })).toBe(review.evidenceHash); +}); + +it("requires exactly one closed decision per group and refuses raw notes and duplicate cycles", () => { + const campaign = fixture(), review = reviewed(campaign); + for (const changed of [ + { ...review, rationale: "private notes" }, + { ...review, decisions: review.decisions.slice(1) }, + { ...review, decisions: [...review.decisions, review.decisions[0]!] }, + { ...review, decisions: review.decisions.map(item => ({ ...item, rawBody: "private body" })) }, + { ...review, decisions: review.decisions.map(item => item.fingerprint === "a".repeat(64) ? { ...item, disposition: "duplicate", duplicateOf: "b".repeat(64) } : item) }, + ]) expect(() => validateCampaignFeedbackReview(campaign, changed as never)).toThrow("Invalid or stale"); +}); + +it.each(["2026-10-20T01:00:00Z", "2026-10-20T01:00:00.936739+00:00", "2026-10-19T20:00:00-05:00"])("accepts explicit ISO review timestamps: %s", reviewedAt => { + const campaign = fixture(); + validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt }); +}); + +it("rejects private timestamp notes before a review can alter the public report", async () => { + const campaign = fixture(); + const reviewedAt = "Tue, 20 Oct 2026 01:00:00 GMT (Private rationale: customer account was suspended)"; + expect(Number.isFinite(Date.parse(reviewedAt))).toBe(true); + const root = await mkdtemp(path.join(tmpdir(), "feedback-private-timestamp-")); + try { + const original = JSON.stringify(campaign); + await writeFile(path.join(root, "normalized-results.json"), original); + const feedbackReviewFile = path.join(root, "review.json"); + await writeFile(feedbackReviewFile, JSON.stringify({ ...reviewed(campaign), reviewedAt })); + await expect(regenerateRunnerDashboard({ bundle: root, historyFile: null, feedbackReviewFile })).rejects.toThrow("Invalid or stale"); + expect(await readFile(path.join(root, "normalized-results.json"), "utf8")).toBe(original); + for (const invalid of ["2026-10-20T01:00:00Z\n", "2026-10-20", "2026-10-20T24:00:00Z"]) { + expect(() => validateCampaignFeedbackReview(campaign, { ...reviewed(campaign), reviewedAt: invalid })).toThrow("Invalid or stale"); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("records an explicit empty review without implying that uncaptured feedback was absent", () => { + const campaign = fixture(); campaign.accounting!.attempts[0]!.feedback = emptyFeedbackSummary(); + campaign.feedbackReview = { ...feedbackReviewTemplate(campaign), version: 1, reviewedAt: "2026-10-20T01:00:00Z", reviewedBy: "human", decisions: [] }; + validateCampaignFeedbackReview(campaign, campaign.feedbackReview); + expect(campaignFeedbackReviewText(campaign)).toContain("1 attempts have incomplete collection"); +}); + +it("refreshes an older summary that kept the awaiting-review count inline", async () => { + const campaign = fixture(); campaign.feedbackReview = reviewed(campaign); + const root = await mkdtemp(path.join(tmpdir(), "feedback-legacy-summary-")); + try { + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await writeFile(path.join(root, "summary.md"), "# Existing campaign\n\nFeedback across retained attempts: 2 complaints; 1 suggestions; 3 distinct items awaiting review; 1 attempts with incomplete collection.\n"); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + const summary = await readFile(path.join(root, "summary.md"), "utf8"); + expect(summary).toContain("2 complaints; 1 suggestions; 1 attempts with incomplete collection."); + expect(summary).toContain(campaignFeedbackReviewText(campaign)); + expect(summary).not.toContain("awaiting review"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/feedback-dispositions.ts b/tests/runner-e2e/feedback-dispositions.ts new file mode 100644 index 00000000000..243a79a70d5 --- /dev/null +++ b/tests/runner-e2e/feedback-dispositions.ts @@ -0,0 +1,95 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { summarizeCampaignFeedback } from "./feedback.js"; +import { html, isPublicReportTimestamp } from "./report-format.js"; +import type { RunnerE2ECampaign } from "./types.js"; + +const dispositions = ["actionable", "duplicate", "already_fixed", "unsupported", "needs_investigation"] as const; +export interface CampaignFeedbackReview { + version: 1; + campaignId: string; + evidenceHash: string; + reviewedAt: string; + reviewedBy: "human" | "agent"; + decisions: Array<{ + fingerprint: string; + disposition: typeof dispositions[number]; + duplicateOf?: string; + issueNumber?: number; + }>; +} + +export function campaignFeedbackEvidence(campaign: RunnerE2ECampaign) { + if (!campaign.accounting) throw new Error("Feedback review requires retained attempt accounting"); + const attempts = campaign.accounting.attempts.map(attempt => ({ + id: attempt.id, executionId: attempt.executionId, attempt: attempt.attempt, + feedback: attempt.feedback ? { + ...attempt.feedback, + review: [...attempt.feedback.review].sort((a, b) => a.fingerprint.localeCompare(b.fingerprint)), + } : null, + })).sort((a, b) => a.id.localeCompare(b.id)); + const evidence = { campaignId: campaign.campaignId, expected: [...campaign.expected].sort(), + scope: campaign.accounting.scope, missingAttempts: campaign.accounting.missingAttempts, attempts }; + const canonical = (_key: string, value: unknown) => value && typeof value === "object" && !Array.isArray(value) + ? Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))) : value; + return createHash("sha256").update(JSON.stringify(evidence, canonical)).digest("hex"); +} + +export function feedbackReviewTemplate(campaign: RunnerE2ECampaign) { + const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); + return { version: 1, campaignId: campaign.campaignId, evidenceHash: campaignFeedbackEvidence(campaign), + reviewedAt: null, reviewedBy: null, + decisions: feedback.review.map(({ fingerprint }) => ({ fingerprint, disposition: "needs_investigation" })) }; +} + +export function validateCampaignFeedbackReview(campaign: RunnerE2ECampaign, review: CampaignFeedbackReview): void { + const fail = (): never => { throw new Error("Invalid or stale campaign feedback review"); }; + const fields = (value: unknown, allowed: string[]) => { + if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail(); + }; + fields(review, ["version", "campaignId", "evidenceHash", "reviewedAt", "reviewedBy", "decisions"]); + if (review.version !== 1 || review.campaignId !== campaign.campaignId || review.evidenceHash !== campaignFeedbackEvidence(campaign) + || !isPublicReportTimestamp(review.reviewedAt) + || !["human", "agent"].includes(review.reviewedBy) || !Array.isArray(review.decisions)) fail(); + const groups = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results).review; + const expected = new Set(groups.map(group => group.fingerprint)), seen = new Set(); + for (const decision of review.decisions) { + fields(decision, ["fingerprint", "disposition", "duplicateOf", "issueNumber"]); + if (!expected.has(decision.fingerprint) || seen.has(decision.fingerprint) || !dispositions.includes(decision.disposition)) fail(); + seen.add(decision.fingerprint); + if (decision.issueNumber !== undefined && (!Number.isSafeInteger(decision.issueNumber) || decision.issueNumber < 1)) fail(); + if (decision.disposition === "duplicate") { + if (!decision.duplicateOf || decision.duplicateOf === decision.fingerprint || !expected.has(decision.duplicateOf) + || review.decisions.find(item => item.fingerprint === decision.duplicateOf)?.disposition === "duplicate") fail(); + } else if (decision.duplicateOf !== undefined) fail(); + } + if (seen.size !== expected.size) fail(); +} + +export function campaignFeedbackReviewText(campaign: RunnerE2ECampaign): string { + const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); + const review = campaign.feedbackReview; + if (!review) return `${feedback.review.length} distinct feedback items await review. Collection gaps and failed submissions remain separate findings.`; + validateCampaignFeedbackReview(campaign, review); + const count = (disposition: typeof dispositions[number]) => review.decisions.filter(item => item.disposition === disposition).length; + return `Feedback reviewed by ${review.reviewedBy} at ${review.reviewedAt}: ${dispositions.map(value => `${count(value)} ${value.replaceAll("_", " ")}`).join("; ")}. ${feedback.incomplete} attempts have incomplete collection; review does not close these gaps or change product grades.`; +} + +export function renderCampaignFeedbackReview(campaign: RunnerE2ECampaign): string { + const summary = campaignFeedbackReviewText(campaign); + const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); + const rows = feedback.review.map(group => { + const decision = campaign.feedbackReview?.decisions.find(item => item.fingerprint === group.fingerprint); + return `${html(group.kind)}${html(group.fingerprint)}${group.occurrences}${html(decision?.disposition ?? "unreviewed")}${decision?.duplicateOf ? html(decision.duplicateOf) : ""}${decision?.issueNumber ? `#${decision.issueNumber}` : ""}`; + }).join(""); + return `

Campaign feedback review

${html(summary)}

Full bodies and review rationale stay in private evidence. Exact duplicate bodies are grouped once; related causes can be linked by a reviewed duplicate disposition.

${rows ? `
${rows}
KindFingerprintOccurrencesDispositionFollow-up
` : ""}
`; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [campaignFile, outputFile] = process.argv.slice(2); + if (!campaignFile || !outputFile) throw new Error("Usage: feedback-dispositions.ts "); + const campaign = JSON.parse(await readFile(campaignFile, "utf8")) as RunnerE2ECampaign; + await writeFile(outputFile, `${JSON.stringify(feedbackReviewTemplate(campaign), null, 2)}\n`, { mode: 0o600, flag: "wx" }); + console.log("Created an unreviewed template. Inspect private feedback bodies, fill dispositions and review metadata, then import with PAPERCLIP_E2E_FEEDBACK_REVIEW."); +} diff --git a/tests/runner-e2e/feedback-persistence.test.ts b/tests/runner-e2e/feedback-persistence.test.ts new file mode 100644 index 00000000000..ca178d36a78 --- /dev/null +++ b/tests/runner-e2e/feedback-persistence.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { expectedFixtureFeedback, feedbackPersistenceDigest, gradeFeedbackPersistence } from "./feedback-persistence.js"; +import { runnerMatrix, runnerTasks, suiteDefinitionHash } from "./catalog.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; + +function fixture() { + const nonce = "fixture-nonce", bodies = expectedFixtureFeedback(nonce); + return { nonce, companyId: "company", agentId: "agent", taskId: "task", runIds: ["run"], + evidence: { version: 1, companyId: "company", summary: { collection: "complete", invocationCollection: "complete" }, + entries: (["complaint", "suggestion"] as const).map(kind => ({ id: kind, kind, body: bodies[kind], + agent: { id: "agent" }, task: { id: "task" }, runId: "run" })), + invocations: (["complaint", "suggestion"] as const).map(kind => ({ callId: `${kind}-call`, kind, runId: "run", outcome: "completed" })), + }, + }; +} + +describe("feedback persistence oracle", () => { + it("changes the suite hash when the inherited prompt or matchers change", () => { + const base = runnerTasks.find(task => task.id === "message-marker")!; + const suite = runnerMatrix.find(execution => execution.suite.id === "feedback-persistence")!.suite; + const baseline = suiteDefinitionHash(suite); + expect(suite.definitionMetadata?.sourceDigest).toBe(feedbackPersistenceDigest(base)); + for (const changed of [ + { ...base, buildPrompt: () => "A different completion contract" }, + { ...base, buildMatchers: () => [] }, + ]) { + expect(suiteDefinitionHash({ ...suite, definitionMetadata: { + ...suite.definitionMetadata, sourceDigest: feedbackPersistenceDigest(changed), + } })).not.toBe(baseline); + } + }); + it("accepts independently retained bodies with same-run real tool receipts", () => { + expect(gradeFeedbackPersistence(fixture()).every(check => check.passed)).toBe(true); + }); + it.each([ + ["missing saved evidence", (f: ReturnType) => { f.evidence.entries = []; }], + ["prose without tool receipts", (f: ReturnType) => { f.evidence.invocations = []; }], + ["wrong saved body", (f: ReturnType) => { f.evidence.entries[0]!.body = "submitted successfully"; }], + ["foreign company", (f: ReturnType) => { f.evidence.companyId = "other"; }], + ["foreign task", (f: ReturnType) => { f.evidence.entries[0]!.task.id = "other"; }], + ["foreign agent", (f: ReturnType) => { f.evidence.entries[0]!.agent.id = "other"; }], + ["foreign run", (f: ReturnType) => { f.evidence.entries[0]!.runId = "other"; }], + ["unrelated successful receipt", (f: ReturnType) => { f.evidence.invocations[0]!.runId = "other"; }], + ["failed tool", (f: ReturnType) => { f.evidence.invocations[0]!.outcome = "failed"; }], + ["partial collection", (f: ReturnType) => { f.evidence.summary.collection = "partial"; }], + ["partial invocation scan", (f: ReturnType) => { f.evidence.summary.invocationCollection = "partial"; }], + ["duplicate submission", (f: ReturnType) => { f.evidence.invocations.push(f.evidence.invocations[0]!); }], + ["duplicate saved row", (f: ReturnType) => { f.evidence.entries.push(f.evidence.entries[0]!); }], + ] as const)("rejects %s", (_name, mutate) => { + const f = fixture(); mutate(f); + expect(gradeFeedbackPersistence(f).some(check => !check.passed)).toBe(true); + }); + it("fails closed on unavailable or malformed retained evidence", () => { + for (const evidence of [undefined, null, {}, { entries: [null], invocations: [null] }]) { + expect(gradeFeedbackPersistence({ ...fixture(), evidence }).some(check => !check.passed)).toBe(true); + } + }); + it("registers only two explicit local native cases with no automatic retry", () => { + const selected = selectRunnerExecutions(parseRunnerSelectors(["--suite", "feedback-persistence"])); + expect(selected.map(execution => execution.id)).toEqual([ + "feedback-persistence.runner-codex.local.complaint-and-suggestion", + "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", + ]); + expect(selected.every(execution => execution.task.automaticRetryPolicy === "single_attempt")).toBe(true); + expect(runnerMatrix).toEqual(expect.arrayContaining(selected)); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "feedback-persistence")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/feedback-persistence.ts b/tests/runner-e2e/feedback-persistence.ts new file mode 100644 index 00000000000..fd4faf1ee76 --- /dev/null +++ b/tests/runner-e2e/feedback-persistence.ts @@ -0,0 +1,78 @@ +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import type { RunnerTaskFixture } from "./types.js"; + +export const FEEDBACK_PERSISTENCE_SUITE = "feedback-persistence"; +const feedbackPersistenceSourceDigest = createHash("sha256") + .update(readFileSync(new URL(import.meta.url))) + .update(readFileSync(new URL("./catalog.ts", import.meta.url))) + .update(readFileSync(new URL("./feedback.ts", import.meta.url))) + .update(readFileSync(new URL("./runner.spec.ts", import.meta.url))) + .digest("hex"); + +export function feedbackPersistenceDigest(base: RunnerTaskFixture) { + return createHash("sha256").update(feedbackPersistenceSourceDigest) + .update(JSON.stringify(base, (_key, value) => typeof value === "function" ? value.toString() : value)) + .digest("hex"); +} + +export function expectedFixtureFeedback(nonce: string) { + return { + complaint: `Synthetic eval complaint ${nonce}: the fixture export omitted its receipt. This is test data, not a real product incident.`, + suggestion: `Synthetic eval suggestion ${nonce}: add a keyboard shortcut to the fixture preview. This is a separate test idea, not a real feature request.`, + }; +} + +export function feedbackPersistenceTask(base: RunnerTaskFixture): RunnerTaskFixture { + return { + ...base, id: "complaint-and-suggestion", label: "Retain submitted feedback", automaticRetryPolicy: "single_attempt", + attemptTimeoutMs: { local: 5 * 60_000, daytona: 5 * 60_000 }, + buildPrompt(nonce) { + const bodies = expectedFixtureFeedback(nonce); + return [ + "This disposable acceptance task verifies two different feedback tools. Before completing the task, discover the feedback tools and submit exactly one complaint and one suggestion through their real tool-call channel.", + `Call submit_complaint with body ${JSON.stringify(bodies.complaint)} and idempotencyKey ${JSON.stringify(`complaint-${nonce}`)}.`, + `Call submit_suggestion with body ${JSON.stringify(bodies.suggestion)} and idempotencyKey ${JSON.stringify(`suggestion-${nonce}`)}.`, + "Wait for both saved acknowledgments. Do not substitute prose, a task comment, direct API calls, or files for these two tool submissions. These are distinct synthetic items; do not submit additional feedback.", + base.buildPrompt(nonce), + ].join("\n"); + }, + }; +} + +const row = (value: unknown): Record => + value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +export function gradeFeedbackPersistence(input: { + evidence: unknown; nonce: string; companyId: string | undefined; agentId: string | undefined; + taskId: string | undefined; runIds: string[]; +}) { + const evidence = row(input.evidence), summary = row(evidence.summary); + const entries = Array.isArray(evidence.entries) ? evidence.entries.map(row) : []; + const invocations = Array.isArray(evidence.invocations) ? evidence.invocations.map(row) : []; + const expected = expectedFixtureFeedback(input.nonce); + const checks: Array<{ id: string; passed: boolean; detail: string }> = []; + const check = (id: string, passed: boolean, detail: string) => checks.push({ id, passed, detail }); + check("complete-retained-evidence", evidence.version === 1 && Boolean(input.companyId) + && evidence.companyId === input.companyId && summary.collection === "complete" && summary.invocationCollection === "complete", + "Both collection passes completed and the saved evidence belongs to this company."); + check("one-of-each", entries.length === 2 && new Set(entries.map(entry => entry.id)).size === 2 + && entries.every(entry => typeof entry.id === "string" && entry.id.length > 0), + "Exactly two distinct saved feedback records remain after teardown."); + for (const kind of ["complaint", "suggestion"] as const) { + const saved = entries.filter(entry => entry.kind === kind); + const entry = saved[0] ?? {}; + const calls = invocations.filter(call => call.kind === kind); + check(`${kind}-body-and-attribution`, saved.length === 1 && entry.body === expected[kind] + && Boolean(input.agentId) && row(entry.agent).id === input.agentId && Boolean(input.taskId) + && row(entry.task).id === input.taskId && typeof entry.runId === "string" && input.runIds.includes(entry.runId), + "The independently read saved body has the exact nonce, agent, task, and observed run."); + check(`${kind}-completed-tool`, calls.length === 1 && calls[0]!.outcome === "completed" + && typeof calls[0]!.callId === "string" && Boolean(calls[0]!.callId) + && typeof entry.runId === "string" && calls[0]!.runId === entry.runId, + "A real completed tool receipt from the same run corroborates the saved feedback."); + } + check("only-expected-submissions", invocations.length === 2 && new Set(invocations.map(call => call.callId)).size === 2, + "There are no duplicate or extra feedback submissions."); + return checks; +} diff --git a/tests/runner-e2e/feedback-recovery.test.ts b/tests/runner-e2e/feedback-recovery.test.ts new file mode 100644 index 00000000000..21e23ea2000 --- /dev/null +++ b/tests/runner-e2e/feedback-recovery.test.ts @@ -0,0 +1,111 @@ +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { afterEach, expect, it, vi } from "vitest"; +import { recoverInterruptedFeedback, readRecoveredFeedback } from "./feedback-recovery.js"; +import { mustPreserveRecoveryState } from "./cleanup-verification.js"; +import { isHistoricalBundlePathAllowed } from "./history-publish.js"; +import type { RunnerApi } from "./api.js"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, writeFile: vi.fn(actual.writeFile) }; +}); + +const roots: string[] = []; +afterEach(async () => { vi.mocked(writeFile).mockReset(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" }; +const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac"; +const api = (get: (url: string) => unknown): Pick => ({ async get(url: string) { return get(url) as T; } }); + +async function fixture() { + const root = await mkdtemp(path.join(os.tmpdir(), "eval-feedback-recovery-")); roots.push(root); + await mkdir(path.join(root, "snapshots")); + await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId })); + return root; +} + +it("retains saved feedback after worker loss while active-run coverage stays partial", async () => { + const root = await fixture(); + const entry = { id: "saved", kind: "complaint", runId: "run", body: "Synthetic lost-worker complaint. private-canary-value" }; + await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, + secrets: ["private-canary-value"], api: api(url => { + if (url.includes("agent-commentary?")) return { items: [entry], total: 1 }; + if (url.endsWith("agent-commentary/saved")) return entry; + if (url.endsWith("heartbeat-runs?limit=1000")) return [{ id: "run" }]; + if (url.endsWith("heartbeat-runs/run")) return { id: "run", runtimeMode: "native", status: "running" }; + return []; + }), + }); + const summary = await readRecoveredFeedback(root, identity); + expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", complaints: 1, bodiesCollected: 1 }); + expect(summary?.review).toHaveLength(1); + const saved = await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8"); + expect(saved).toContain("Synthetic lost-worker complaint."); + expect(saved).not.toContain("private-canary-value"); + expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback-recovered.json")).toBe(false); + expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/feedback-recovery.json")).toBe(false); + await writeFile(path.join(root, "snapshots/agent-feedback.json"), saved); + await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [], + api: api(() => { throw new Error("controller stopped after the worker checkpoint"); }), + }); + expect(await readRecoveredFeedback(root, identity)).toMatchObject({ complaints: 1, bodiesCollected: 1, collection: "partial" }); + expect(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8")).toContain("Synthetic lost-worker complaint."); + expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined(); + const receiptPath = path.join(root, "snapshots/feedback-recovery.json"); + const receipt = JSON.parse(await readFile(receiptPath, "utf8")); + receipt.summary.collection = "complete"; + await writeFile(receiptPath, JSON.stringify(receipt)); + expect(await readRecoveredFeedback(root, identity)).toBeUndefined(); +}); + +it("preserves completed attempt evidence without making another API request", async () => { + const root = await fixture(); + await writeFile(path.join(root, "result.json"), "retained original"); + let calls = 0; + await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [], + api: api(() => { calls++; throw new Error("must not read"); }), + }); + expect(calls).toBe(0); + expect(await readFile(path.join(root, "result.json"), "utf8")).toBe("retained original"); + expect(await readRecoveredFeedback(root, identity)).toBeUndefined(); +}); + +it("reports the earlier atomic checkpoint when a later write exhausts the recovery deadline", async () => { + const root = await fixture(); + const actual = await vi.importActual("node:fs/promises"); + let checkpointWrites = 0; + vi.mocked(writeFile).mockImplementation(async (file, data, options) => { + if (String(file).endsWith("agent-feedback-recovered.json.pending") && ++checkpointWrites === 3) { + const signal = typeof options === "object" && options ? options.signal : undefined; + if (!signal?.aborted) await new Promise(resolve => signal!.addEventListener("abort", resolve, { once: true })); + signal!.throwIfAborted(); + } + return actual.writeFile(file, data, options); + }); + const entry = { id: "saved", kind: "suggestion", runId: "run", body: "Keep this earlier durable checkpoint." }; + await expect(recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 1000, + secrets: [], api: api(url => { + if (url.includes("agent-commentary?")) return { items: [entry], total: 1 }; + if (url.endsWith("agent-commentary/saved")) return entry; + return []; + }), + })).rejects.toThrow(); + expect(checkpointWrites).toBe(3); + expect(await readRecoveredFeedback(root, identity)).toMatchObject({ suggestions: 1, bodiesCollected: 1, + collection: "partial", invocationCollection: "unavailable" }); + expect(await readRecoveredFeedback(root, { ...identity, attemptId: "different" })).toBeUndefined(); + await expect(readFile(path.join(root, "snapshots/feedback-recovery.json"))).rejects.toMatchObject({ code: "ENOENT" }); + const saved = JSON.parse(await readFile(path.join(root, "snapshots/agent-feedback-recovered.json"), "utf8")); + expect(saved.entries[0].body).toBe(entry.body); +}); + +it("reports unavailable recovery without fabricating an empty successful collection", async () => { + const root = await fixture(); + await recoverInterruptedFeedback({ identity, caseDirectory: root, deadlineAt: Date.now() + 5000, secrets: [], + api: api(() => { throw new Error("controller unavailable"); }), + }); + expect(await readRecoveredFeedback(root, identity)).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable" }); + expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: true }] })).toBe(true); + expect(mustPreserveRecoveryState({ processCleanupFailed: false, results: [{ cleanup: "passed", synthetic: false }] })).toBe(false); +}); diff --git a/tests/runner-e2e/feedback-recovery.ts b/tests/runner-e2e/feedback-recovery.ts new file mode 100644 index 00000000000..0d8c571fd90 --- /dev/null +++ b/tests/runner-e2e/feedback-recovery.ts @@ -0,0 +1,101 @@ +import { access, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { RunnerApi } from "./api.js"; +import { collectAttemptFeedback, emptyFeedbackSummary, type FeedbackEvidence, type FeedbackSummary } from "./feedback.js"; +import { assertSecretFree, sanitizeJson } from "./redaction.js"; + +export interface FeedbackRecoveryIdentity { + campaignId: string; + attemptId: string; + executionId: string; +} + +const exists = (file: string) => access(file).then(() => true, () => false); + +export async function recoverInterruptedFeedback(input: { + identity: FeedbackRecoveryIdentity; + caseDirectory: string; + api: Pick; + secrets: readonly string[]; + deadlineAt: number; +}) { + const deadline = AbortSignal.timeout(Math.max(1, input.deadlineAt - Date.now())); + if (await exists(path.join(input.caseDirectory, "result.json"))) return; + const snapshots = path.join(input.caseDirectory, "snapshots"); + let fixture: { executionId?: string; companyId?: string }; + try { fixture = JSON.parse(await readFile(path.join(snapshots, "fixtures.json"), { encoding: "utf8", signal: deadline })); } + catch { return; } + if (fixture.executionId !== input.identity.executionId || !/^[a-f0-9-]{36}$/i.test(fixture.companyId ?? "")) return; + const companyId = fixture.companyId!; + let summary = emptyFeedbackSummary(); + let checkpoint: FeedbackEvidence | undefined; + const recoveryReceipt = (summary: FeedbackSummary) => ({ version: 1, ...input.identity, companyId, + reason: "worker_exited_without_result", summary }); + const save = async (name: string, value: unknown, signal: AbortSignal = deadline) => { + const text = JSON.stringify(sanitizeJson(value, input.secrets), null, 2) + "\n"; + assertSecretFree(text, input.secrets, name); + await writeFile(path.join(snapshots, `${name}.pending`), text, { mode: 0o600, signal }); + signal.throwIfAborted(); + await rename(path.join(snapshots, `${name}.pending`), path.join(snapshots, name)); + }; + const saveCheckpoint = (evidence: FeedbackEvidence, signal: AbortSignal = deadline) => + save("agent-feedback-recovered.json", { ...evidence, recovery: recoveryReceipt({ ...evidence.summary, + collection: evidence.summary.bodiesCollected ? "partial" : "unavailable", + invocationCollection: evidence.invocations.length ? "partial" : "unavailable" }) }, signal); + try { + const original = JSON.parse(await readFile(path.join(snapshots, "agent-feedback.json"), { encoding: "utf8", signal: deadline })); + if (original.version === 1 && original.companyId === companyId && original.summary?.version === 1 + && Array.isArray(original.entries) && Array.isArray(original.invocations) + && original.summary.bodiesCollected === original.entries.length) { + await saveCheckpoint(original); + checkpoint = original; + summary = structuredClone(original.summary); + } + } catch {} + try { + const collected = await collectAttemptFeedback({ api: input.api, companyId, deadlineAt: input.deadlineAt, + persist: async (evidence, signal) => { + if (checkpoint && (!checkpoint.entries.every(entry => evidence.entries.some(next => next.id === entry.id)) + || !checkpoint.invocations.every(call => evidence.invocations.some(next => next.runId === call.runId && next.callId === call.callId)))) return; + await saveCheckpoint(evidence, signal); + checkpoint = structuredClone(evidence); + summary = structuredClone(evidence.summary); + }, + }); + summary.collection = summary.bodiesCollected ? "partial" : collected.collection; + summary.invocationCollection = summary.invocations.completed + summary.invocations.failed + summary.invocations.unknown + ? "partial" : collected.invocationCollection; + } catch { + summary.collection = summary.bodiesCollected ? "partial" : "unavailable"; + summary.invocationCollection = "unavailable"; + } + await save("feedback-recovery.json", recoveryReceipt(summary)); +} + +export async function readRecoveredFeedback(caseDirectory: string, identity: FeedbackRecoveryIdentity): Promise { + try { + const snapshots = path.join(caseDirectory, "snapshots"); + const evidence = JSON.parse(await readFile(path.join(snapshots, "agent-feedback-recovered.json"), "utf8")); + let receipt = evidence.recovery; + try { receipt = JSON.parse(await readFile(path.join(snapshots, "feedback-recovery.json"), "utf8")); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") return; } + if (receipt?.version !== 1 || evidence.version !== 1 || receipt.reason !== "worker_exited_without_result" + || !receipt.companyId || receipt.companyId !== evidence.companyId + || Object.entries(identity).some(([key, value]) => receipt[key] !== value)) return; + const summary = evidence.summary as FeedbackSummary; + const statuses = ["complete", "partial", "unavailable", "not_started"]; + if (summary?.version !== 1 || !Array.isArray(evidence.entries) || !Array.isArray(evidence.invocations) + || summary.bodiesCollected !== evidence.entries.length || !Array.isArray(summary.review) + || ![summary.complaints, summary.suggestions, summary.bodiesCollected, summary.runsInspected, + summary.invocations?.completed, summary.invocations?.failed, summary.invocations?.unknown] + .every(value => Number.isSafeInteger(value) && value >= 0) + || !["observed", "unknown"].includes(summary.toolAvailability) + || !["collection", "invocationCollection"].every(key => { + const original = summary[key as "collection" | "invocationCollection"]; + const recovered = receipt.summary?.[key]; + return statuses.includes(original) && statuses.includes(recovered) + && (recovered === original || recovered === "partial" || recovered === "unavailable"); + })) return; + return { ...summary, collection: receipt.summary.collection, invocationCollection: receipt.summary.invocationCollection }; + } catch { return; } +} diff --git a/tests/runner-e2e/feedback-shutdown-process.test.ts b/tests/runner-e2e/feedback-shutdown-process.test.ts new file mode 100644 index 00000000000..ffd6119416d --- /dev/null +++ b/tests/runner-e2e/feedback-shutdown-process.test.ts @@ -0,0 +1,113 @@ +import { spawn } from "node:child_process"; +import { createRequire } from "node:module"; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { expect, it } from "vitest"; +import { readRecoveredFeedback } from "./feedback-recovery.js"; +import { runnerE2EWebServerCommand, runnerE2EPlaywrightInvocation, runnerE2EWebServerGracefulShutdown, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js"; +import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; + +it.skipIf(process.platform === "win32")("retains feedback before shutdown through the real Playwright launch chain", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "eval feedback 'shutdown-")); + const repository = path.resolve(import.meta.dirname, "../.."); + const caseDirectory = path.join(root, "case"); + const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution" }; + const companyId = "88c1939b-2d9f-4bb3-8ebc-8a7a85efeeac"; + let child: ReturnType | undefined; + let timer: ReturnType | undefined; + let supervisorPid: number | undefined; + let owner: ReturnType | undefined; + try { + await mkdir(path.join(root, "tests/runner-e2e"), { recursive: true }); + await mkdir(path.join(caseDirectory, "snapshots"), { recursive: true }); + await symlink(path.join(repository, "cli"), path.join(root, "cli")); + await writeFile(path.join(root, "package.json"), JSON.stringify({ type: "module" })); + await writeFile(path.join(caseDirectory, "snapshots/fixtures.json"), JSON.stringify({ executionId: identity.executionId, companyId })); + await writeFile(path.join(root, "tests/runner-e2e/server.ts"), ` + import { createServer } from 'node:http'; + import { writeFile } from 'node:fs/promises'; + import { recoverInterruptedFeedback } from ${JSON.stringify(pathToFileURL(path.join(repository, "tests/runner-e2e/feedback-recovery.ts")).href)}; + const entries = ['complaint', 'suggestion'].map(kind => ({ id: kind, kind, runId: 'run', body: 'Saved ' + kind + ' private-canary-value' })); + const server = createServer((req, res) => { + setTimeout(() => { + res.setHeader('content-type', 'application/json'); + res.end(JSON.stringify(req.url.includes('agent-commentary?') ? { items: entries, total: 2 } + : req.url.includes('agent-commentary/') ? entries.find(entry => req.url.endsWith('/' + entry.id)) + : req.url.endsWith('/heartbeat-runs/run') ? { id: 'run', runtimeMode: 'native', status: 'running' } + : req.url.includes('heartbeat-runs?') ? [{ id: 'run' }] : [])); + }, 100); + }); + let closing; + process.on('SIGTERM', () => closing ??= (async () => { + await recoverInterruptedFeedback({ identity: ${JSON.stringify(identity)}, caseDirectory: ${JSON.stringify(caseDirectory)}, + deadlineAt: Date.now() + 5000, secrets: ['private-canary-value'], api: { + async get(endpoint, options) { + return (await fetch('http://127.0.0.1:' + server.address().port + endpoint, + { signal: AbortSignal.timeout(options?.timeout ?? 1000) })).json(); + } + } }); + server.close(); + server.closeAllConnections(); + })()); + server.listen(0, '127.0.0.1', async () => { + await writeFile(${JSON.stringify(path.join(root, "ready.json"))}, JSON.stringify({ pid: process.pid })); + console.log('SUPERVISOR_READY'); + }); + `); + const testModule = createRequire(import.meta.url).resolve("@playwright/test"); + const configPath = path.join(root, "playwright.config.cjs"); + await writeFile(configPath, `module.exports = { + testDir: ${JSON.stringify(root)}, testMatch: 'interrupted.spec.cjs', workers: 1, reporter: 'line', timeout: 30000, + webServer: { command: ${JSON.stringify(runnerE2EWebServerCommand(root))}, + wait: { stdout: /SUPERVISOR_READY/ }, timeout: 10000, + gracefulShutdown: ${JSON.stringify(runnerE2EWebServerGracefulShutdown)} } + };`); + await writeFile(path.join(root, "interrupted.spec.cjs"), ` + const { test } = require(${JSON.stringify(testModule)}); + const { writeFile } = require('node:fs/promises'); + test('interrupted worker', async () => { + await writeFile(${JSON.stringify(path.join(root, "worker-ready"))}, 'ready'); + await new Promise(() => {}); + }); + `); + const invocation = runnerE2EPlaywrightInvocation(repository, ["test", "--config", configPath]); + child = spawn(invocation.command, invocation.args, { cwd: repository, detached: true, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, CI: "1" } }); + owner = createProcessTreeOwner(child); + let output = ""; + child.stdout!.on("data", chunk => { output += chunk; }); + child.stderr!.on("data", chunk => { output += chunk; }); + const closed = new Promise((resolve, reject) => { + child!.once("error", reject); + child!.once("close", () => resolve()); + }); + const deadline = Date.now() + 10_000; + let workerReady = false; + while ((!supervisorPid || !workerReady) && Date.now() < deadline) { + const ready = await readFile(path.join(root, "ready.json"), "utf8").catch(() => null); + if (ready) supervisorPid = JSON.parse(ready).pid; + workerReady = await readFile(path.join(root, "worker-ready"), "utf8").then(() => true, () => false); + if (!supervisorPid || !workerReady) await new Promise(resolve => setTimeout(resolve, 25)); + } + expect(supervisorPid, output).toBeTypeOf("number"); + expect(workerReady, output).toBe(true); + await stopOwnedProcessTree(child, owner, 10_000, 2_000, runnerE2EPlaywrightShutdownSignal); + await Promise.race([closed, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`Shutdown did not finish: ${output}`)), 10_000); + })]); + expect(await readRecoveredFeedback(caseDirectory, identity), output).toMatchObject({ + collection: "partial", invocationCollection: "partial", complaints: 1, suggestions: 1, bodiesCollected: 2, + }); + const evidence = await readFile(path.join(caseDirectory, "snapshots/agent-feedback-recovered.json"), "utf8"); + expect(evidence).toContain("Saved complaint"); + expect(evidence).not.toContain("private-canary-value"); + expect(() => process.kill(supervisorPid!, 0)).toThrow(); + } finally { + owner?.stopObserving(); + if (timer) clearTimeout(timer); + if (owner) await owner.signal("SIGKILL").catch(() => {}); + if (child?.pid && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await rm(root, { recursive: true, force: true }); + } +}, 35_000); diff --git a/tests/runner-e2e/feedback.test.ts b/tests/runner-e2e/feedback.test.ts new file mode 100644 index 00000000000..6cb8ead6bb4 --- /dev/null +++ b/tests/runner-e2e/feedback.test.ts @@ -0,0 +1,319 @@ +import { expect, it, vi } from "vitest"; +import type { AgentCommentaryDetail } from "../../packages/shared/src/validators/agent-commentary.js"; +import type { RunnerApi } from "./api.js"; +import { collectAttemptFeedback, emptyFeedbackSummary, feedbackReview, observeFeedbackInvocations, summarizeCampaignFeedback, FEEDBACK_COLLECTION_LIMITS, type FeedbackEvidence } from "./feedback.js"; +import { runCleanupWithObservers } from "./cleanup-verification.js"; +import { isHistoricalBundlePathAllowed } from "./history-publish.js"; +import { canonicalProviderEventsFromOpenCodePart } from "../../packages/paperclip-runner/src/provider-events.js"; + +function entry(id: string, body = "Please preserve the original tool error."): AgentCommentaryDetail { + return { id, kind: "complaint", preview: body.slice(0, 280), body, + createdAt: "2026-10-20T00:00:00.000Z", agent: { id: "agent", name: "Fixture", icon: null }, runId: "run", task: null }; +} +const api = (read: (path: string) => unknown | Promise) => ({ get: async (path: string) => await read(path) as T }) as Pick; +const event = (seq: number, eventType: string, payload: unknown, runId = "run") => ({ seq, eventType, + payload: { prpEvent: { runId, sourceKind: "runner", eventType, payload } } }); + +it("collects full paginated company feedback, including feedback outside the selected task, and checkpoints it", async () => { + const entries = Array.from({ length: 101 }, (_, i) => ({ ...entry(String(i), "a".repeat(300)), runId: `child-${i}` })); + const checkpoints: FeedbackEvidence[] = []; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { checkpoints.push(structuredClone(value)); }, + api: api(path => { + if (path.includes("agent-commentary?")) { + const offset = Number(new URL(path, "http://fixture").searchParams.get("offset")); + return { items: entries.slice(offset, offset + 100).map(({ body: _body, ...rest }) => rest), total: 101 }; + } + if (path.includes("agent-commentary/")) return entries[Number(path.split("/").at(-1))]; + return []; + }), + }); + expect(summary).toMatchObject({ collection: "complete", complaints: 101, bodiesCollected: 101, toolAvailability: "unknown" }); + expect(summary.review).toEqual([{ fingerprint: expect.stringMatching(/^[a-f0-9]{64}$/), occurrences: 101, kind: "complaint", disposition: "needs_investigation" }]); + expect(checkpoints.at(-1)!.entries[100]!.body).toHaveLength(300); + expect(checkpoints.some(value => value.entries.length === 10)).toBe(true); + expect(JSON.stringify(summary)).not.toContain("aaa"); +}); + +it("keeps saved-feedback absence separate from unavailable reads and unknown tool availability", async () => { + const empty = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, + api: api(path => path.includes("agent-commentary") ? { items: [], total: 0 } : []), + }); + expect(empty).toMatchObject({ collection: "complete", complaints: 0, suggestions: 0, toolAvailability: "unknown" }); + const unavailable = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(() => { throw new Error("unavailable"); }) }); + expect(unavailable).toMatchObject({ collection: "unavailable", invocationCollection: "unavailable", complaints: 0 }); +}); + +it("retains partial bodies and failed submissions without inventing persistence from a completed tool", async () => { + let retained: FeedbackEvidence | undefined; + const events = [ + event(1, "tool.execution.started", { name: "submit_suggestion", executionId: "failed" }), + event(2, "tool.execution.completed", { name: "submit_suggestion", executionId: "failed", status: "failed" }), + event(3, "tool.execution.started", { name: "submit_complaint", executionId: "done" }), + event(4, "tool.execution.completed", { name: "submit_complaint", executionId: "done", status: "completed" }), + event(5, "tool.execution.started", { name: "submit_suggestion", executionId: "interrupted" }), + ]; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, + api: api(path => { + if (path.includes("agent-commentary?")) return { items: [entry("1"), entry("2")], total: 2 }; + if (path.endsWith("agent-commentary/1")) return entry("1"); + if (path.endsWith("agent-commentary/2")) throw new Error("body unavailable"); + if (path.includes("/events?")) return events; + if (path === "/api/heartbeat-runs/run") return { id: "run", runtimeMode: "native", status: "failed" }; + return [{ id: "run", status: "failed" }]; + }), + }); + expect(summary).toMatchObject({ collection: "partial", complaints: 2, bodiesCollected: 1, invocationCollection: "complete", + invocations: { completed: 1, failed: 1, unknown: 1 } }); + expect(retained!.diagnostics).toContain("feedback_detail_failed"); +}); + +it("binds native invocations to their run, ignores prose, and recognizes compatibility receipts", () => { + const events = [ + event(1, "item.started", { kind: "tool_call", item: { type: "tool_call", id: "call", name: "paperclip.submit_complaint" } }), + event(2, "item.completed", { kind: "tool_result", item: { type: "tool_result", id: "call", status: "completed", isError: true } }), + event(3, "tool.execution.started", { name: "submit_suggestion", executionId: "foreign" }, "foreign-run"), + event(4, "item.completed", { item: { type: "agentMessage", text: "I used submit_complaint" } }), + ]; + expect(observeFeedbackInvocations("run", events)).toEqual([{ runId: "run", callId: "call", kind: "complaint", outcome: "failed" }]); +}); + +it.each(["paperclip_", "paperclip__", "mcp__paperclip__"])("maps namespaced feedback receipts through native events: %s", prefix => { + const events = [ + event(1, "tool.execution.started", { executionId: "failed", name: `${prefix}submit_complaint` }), + event(2, "tool.execution.completed", { executionId: "failed", name: `${prefix}submit_complaint`, status: "failed" }), + event(3, "item.started", { kind: "tool_call", item: { id: "done", name: `${prefix}submit_suggestion` } }), + event(4, "item.completed", { kind: "tool_result", item: { id: "done", status: "completed" } }), + event(5, "tool.execution.started", { executionId: "interrupted", name: `${prefix}submit_suggestion` }), + event(6, "tool.execution.started", { executionId: "unrelated", name: "other_submit_complaint" }), + ]; + expect(observeFeedbackInvocations("run", events)).toEqual([ + { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" }, + { runId: "run", callId: "done", kind: "suggestion", outcome: "completed" }, + { runId: "run", callId: "interrupted", kind: "suggestion", outcome: "unknown" }, + ]); +}); + +it.each([{ runtimeMode: "legacy", status: "succeeded" }, { runtimeMode: "native", status: "running" }])("does not claim complete invocation coverage for legacy helpers or still-active work: %o", async run => { + const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) return []; + if (path === "/api/heartbeat-runs/run") return { id: "run", ...run }; + return [{ id: "run", status: run.status }]; + }) }); + expect(summary.invocationCollection).toBe("partial"); +}); + +it("captures feedback after cancellation failure but before destructive teardown, and still tears down if persistence fails", async () => { + const calls: string[] = []; + const result = await runCleanupWithObservers({ + retireRuns: async () => { throw new Error("cancel failed"); }, assertions: [], + teardown: async () => { + try { + await collectAttemptFeedback({ companyId: "company", api: api(() => []), persist: async () => { calls.push("capture"); throw new Error("disk full"); } }); + } finally { calls.push("teardown"); } + }, + }); + expect(calls).toEqual(["capture", "teardown"]); + expect(result.errors).toHaveLength(2); +}); + +it("bounds collection by a deadline and retains content-free diagnostics", async () => { + let requests = 0, last: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 0, + persist: async value => { last = structuredClone(value); }, api: api(() => { requests++; throw new Error("secret response"); }), + }); + expect(requests).toBe(0); + expect(summary.collection).toBe("unavailable"); + expect(last).toBeUndefined(); +}); + +it("preserves the read prefix when slow detail requests exhaust the read budget mid-page", async () => { + let now = 0, detailReads = 0; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + const checkpoints: FeedbackEvidence[] = []; + const slowApi = { get: async (url: string, options: { timeout: number }) => { + if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index))), total: 100 }; + if (!url.includes("agent-commentary/")) throw new Error("No read should start after the reserved cutoff"); + detailReads++; + now += Math.min(400, options.timeout); + if (options.timeout < 400) throw new Error("request timeout"); + return entry(url.split("/").at(-1)!); + } } as Pick; + try { + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, api: slowApi, + persist: async evidence => { checkpoints.push(structuredClone(evidence)); now += 20; }, + }); + expect(summary).toMatchObject({ collection: "partial", invocationCollection: "unavailable" }); + expect(summary.bodiesCollected).toBeGreaterThan(10); + expect(summary.bodiesCollected).toBeLessThan(100); + expect(checkpoints[1].entries).toHaveLength(10); + expect(checkpoints.at(-1)!.entries).toHaveLength(summary.bodiesCollected); + expect(checkpoints.at(-1)!.entries.every(value => value.body === entry("fixture").body)).toBe(true); + expect(checkpoints.at(-1)!.diagnostics).toContain("deadline_reached"); + expect(detailReads).toBe(summary.bodiesCollected + 1); + expect(now).toBeLessThan(30_000); + } finally { clock.mockRestore(); } +}); + +it("groups identical feedback for one review without conflating kinds", () => { + expect(feedbackReview([entry("1", " slow request "), entry("2", "slow request"), { ...entry("3", "slow request"), kind: "suggestion" }])) + .toMatchObject([{ kind: "complaint", occurrences: 2 }, { kind: "suggestion", occurrences: 1 }]); +}); + +it("groups feedback across retries without omitting incomplete collection and keeps bodies out of public history", () => { + const feedback = { ...emptyFeedbackSummary(), complaints: 1, review: feedbackReview([entry("1")]) }; + const aggregate = summarizeCampaignFeedback([{ feedback }, { feedback }, {}]); + expect(aggregate).toMatchObject({ complaints: 2, incomplete: 3, review: [{ occurrences: 2 }] }); + expect(feedback.review[0]!.occurrences).toBe(1); + expect(JSON.stringify(aggregate)).not.toContain(entry("1").body); + expect(isHistoricalBundlePathAllowed("evidence/case/attempt-1/snapshots/agent-feedback.json")).toBe(false); +}); + +it("uses run details for native coverage when the real list projection omits runtimeMode", async () => { + const reads: string[] = []; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async () => {}, api: api(path => { + reads.push(path); + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) return []; + if (path === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" }; + return [{ id: "run", status: "succeeded" }]; + }) }); + expect(reads).toContain("/api/heartbeat-runs/run"); + expect(summary).toMatchObject({ collection: "complete", invocationCollection: "complete", runsInspected: 1 }); +}); +it("retains partial evidence if the run-detail lookup fails", async () => { + let retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async value => { retained = structuredClone(value); }, api: api(path => { + if (path.includes("agent-commentary")) return { items: [], total: 0 }; + if (path.includes("/events?")) return [event(1, "tool.execution.started", { name: "submit_complaint", executionId: "call" })]; + if (path === "/api/heartbeat-runs/run") throw new Error("unavailable"); + return [{ id: "run", status: "succeeded" }]; + }) }); + expect(summary).toMatchObject({ collection: "partial", invocationCollection: "partial", invocations: { unknown: 1 }, runsInspected: 1 }); + expect(retained!.diagnostics).toContain("run_detail_failed"); +}); + +it("retains OpenCode feedback whose first observed update is running or terminal", () => { + const updates = [ + { id: "running", status: "running", tool: "paperclip_submit_suggestion" }, + { id: "failed", status: "error", tool: "paperclip_submit_complaint" }, + { id: "completed", status: "completed", tool: "paperclip_submit_suggestion" }, + ].flatMap(update => canonicalProviderEventsFromOpenCodePart({ + id: update.id, messageID: "message", type: "tool", callID: update.id, tool: update.tool, + state: { status: update.status, input: {}, output: "fixture result", error: "fixture failure", time: { start: 1, end: 2 } }, + })); + expect(updates.map(update => update.eventType)).toEqual(["tool.execution.progressed", "tool.execution.completed", "tool.execution.completed"]); + expect(observeFeedbackInvocations("run", updates.map((update, index) => event(index, update.eventType, update.payload)))).toEqual([ + { runId: "run", callId: "running", kind: "suggestion", outcome: "unknown" }, + { runId: "run", callId: "failed", kind: "complaint", outcome: "failed" }, + { runId: "run", callId: "completed", kind: "suggestion", outcome: "completed" }, + ]); +}); + +it("bounds retained bytes and checkpoint writes for maximum-size feedback bodies", async () => { + const body = "x".repeat(524_288); + let detailReads = 0, writtenBytes = 0, writes = 0, retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", + persist: async evidence => { writes++; writtenBytes += Buffer.byteLength(JSON.stringify(evidence, null, 2)); retained = structuredClone(evidence); }, + api: api(url => { + if (url.includes("agent-commentary?")) return { items: Array.from({ length: 100 }, (_, index) => entry(String(index), "preview")), total: 10_000 }; + if (url.includes("agent-commentary/")) { detailReads++; return entry(url.split("/").at(-1)!, body); } + return []; + }), + }); + expect(detailReads).toBeLessThanOrEqual(16); + expect(writes).toBeLessThanOrEqual(4); + expect(writtenBytes).toBeLessThan(3 * FEEDBACK_COLLECTION_LIMITS.retainedBytes); + expect(Buffer.byteLength(JSON.stringify(retained!.entries))).toBeLessThanOrEqual(FEEDBACK_COLLECTION_LIMITS.retainedBytes); + expect(summary.collection).toBe("partial"); + expect(retained!.diagnostics).toContain("evidence_limit_reached"); + expect(retained!.entries.every(item => item.body === body)).toBe(true); +}); + +it("caps entry count and stops requesting further feedback pages", async () => { + let pages = 0, retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); }, + api: api(url => { + if (url.includes("agent-commentary?")) { + const offset = Number(new URL(url, "http://fixture").searchParams.get("offset")); pages++; + return { items: Array.from({ length: 100 }, (_, index) => entry(String(offset + index))), total: 2000 }; + } + if (url.includes("agent-commentary/")) return entry(url.split("/").at(-1)!); + return []; + }), + }); + expect(pages).toBe(10); + expect(summary.bodiesCollected).toBe(FEEDBACK_COLLECTION_LIMITS.entries); + expect(summary.collection).toBe("partial"); + expect(retained!.diagnostics).toContain("evidence_limit_reached"); +}); + +it("aborts stalled evidence persistence at the collection deadline", async () => { + let signal: AbortSignal | undefined, reads = 0; + await expect(collectAttemptFeedback({ companyId: "company", deadlineAt: Date.now() + 50, + persist: async (_evidence, currentSignal) => { signal = currentSignal; await new Promise(() => {}); }, + api: api(() => { reads++; return []; }), + })).rejects.toThrow("Feedback persistence deadline reached"); + expect(signal?.aborted).toBe(true); + expect(reads).toBe(0); +}); + +it("acknowledges only successfully persisted feedback when a later checkpoint fails", async () => { + let retained: FeedbackEvidence | undefined, acknowledged = emptyFeedbackSummary(); + const entries = Array.from({ length: 11 }, (_, index) => entry(String(index))); + await expect(collectAttemptFeedback({ companyId: "company", + onCheckpoint: summary => { acknowledged = summary; }, + persist: async evidence => { + if (evidence.entries.length > 10) throw new Error("disk full"); + retained = structuredClone(evidence); + }, + api: api(url => url.includes("agent-commentary?") ? { items: entries, total: entries.length } + : entries[Number(url.split("/").at(-1))]), + })).rejects.toThrow("disk full"); + expect(retained!.entries).toHaveLength(10); + expect(acknowledged).toEqual(retained!.summary); + expect(acknowledged.bodiesCollected).toBe(10); + expect(acknowledged.review).toMatchObject([{ occurrences: 10 }]); +}); + +it("returns saved review counts when the deadline expires before a later checkpoint", async () => { + let now = 0, retained: FeedbackEvidence | undefined; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + const entries = Array.from({ length: 11 }, (_, index) => entry(String(index))); + try { + const summary = await collectAttemptFeedback({ companyId: "company", deadlineAt: 30_000, + persist: async evidence => { retained = structuredClone(evidence); }, + api: api(url => { + if (url.includes("agent-commentary?")) return { items: entries, total: entries.length }; + const index = Number(url.split("/").at(-1)); + if (index === 10) now = 30_001; + return entries[index]; + }), + }); + expect(retained!.entries).toHaveLength(10); + expect(summary).toMatchObject({ bodiesCollected: 10, collection: "partial", invocationCollection: "unavailable" }); + expect(summary.review).toEqual(retained!.summary.review); + } finally { clock.mockRestore(); } +}); + +it("bounds raw event inspection while retaining the observed feedback receipt prefix", async () => { + let pages = 0, retained: FeedbackEvidence | undefined; + const summary = await collectAttemptFeedback({ companyId: "company", persist: async evidence => { retained = structuredClone(evidence); }, + api: api(url => { + if (url.includes("agent-commentary?")) return { items: [], total: 0 }; + if (url.includes("/events?")) { + pages++; + return Array.from({ length: 100 }, (_, index) => event(index + 1, "tool.execution.started", { + executionId: `call-${index}`, name: "submit_complaint", input: "x".repeat(200_000), + })); + } + if (url === "/api/heartbeat-runs/run") return { id: "run", status: "succeeded", runtimeMode: "native" }; + return [{ id: "run", status: "succeeded" }]; + }), + }); + expect(pages).toBe(1); + expect(summary.invocationCollection).toBe("partial"); + expect(summary.invocations.unknown).toBeGreaterThan(0); + expect(summary.invocations.unknown).toBeLessThan(100); + expect(retained!.diagnostics).toContain("evidence_limit_reached"); +}); diff --git a/tests/runner-e2e/feedback.ts b/tests/runner-e2e/feedback.ts new file mode 100644 index 00000000000..7a24b21cb6d --- /dev/null +++ b/tests/runner-e2e/feedback.ts @@ -0,0 +1,268 @@ +import { createHash } from "node:crypto"; +import type { AgentCommentaryDetail, AgentCommentaryPage } from "../../packages/shared/src/validators/agent-commentary.js"; +import type { RunnerApi } from "./api.js"; + +type Row = Record; +const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +export type FeedbackCollectionStatus = "complete" | "partial" | "unavailable" | "not_started"; +export interface FeedbackSummary { + version: 1; + collection: FeedbackCollectionStatus; + invocationCollection: FeedbackCollectionStatus; + toolAvailability: "observed" | "unknown"; + complaints: number; + suggestions: number; + bodiesCollected: number; + runsInspected: number; + invocations: { completed: number; failed: number; unknown: number }; + review: Array<{ fingerprint: string; kind: "complaint" | "suggestion"; occurrences: number; disposition: "needs_investigation" }>; +} +interface FeedbackInvocation { + runId: string; + callId: string; + kind: "complaint" | "suggestion"; + outcome: "completed" | "failed" | "unknown"; +} +export interface FeedbackEvidence { + version: 1; + companyId: string; + entries: AgentCommentaryDetail[]; + invocations: FeedbackInvocation[]; + diagnostics: Array<"feedback_read_failed" | "feedback_detail_failed" | "feedback_pagination_incomplete" | "run_read_failed" | "run_detail_failed" | "run_limit_reached" | "events_read_failed" | "deadline_reached" | "evidence_limit_reached">; + summary: FeedbackSummary; +} + +export const FEEDBACK_COLLECTION_LIMITS = { + entries: 1000, retainedBytes: 8 * 1024 * 1024, eventBytes: 16 * 1024 * 1024, events: 10_000, +} as const; + +export function emptyFeedbackSummary(): FeedbackSummary { + return { version: 1, collection: "not_started", invocationCollection: "not_started", toolAvailability: "unknown", + complaints: 0, suggestions: 0, bodiesCollected: 0, runsInspected: 0, + invocations: { completed: 0, failed: 0, unknown: 0 }, review: [] }; +} + +function feedbackKind(name: unknown): FeedbackInvocation["kind"] | undefined { + if (typeof name !== "string") return undefined; + const canonicalName = name.replace(/^(?:mcp__paperclip__|paperclip__|paperclip_)/, ""); + if (/(?:^|[.\/])submit_complaint$/.test(canonicalName)) return "complaint"; + if (/(?:^|[.\/])submit_suggestion$/.test(canonicalName)) return "suggestion"; + return undefined; +} + +export function observeFeedbackInvocations(runId: string, events: readonly Row[]): FeedbackInvocation[] { + const calls = new Map(); + for (const event of events) { + const envelope = row(row(event.payload).prpEvent); + if (envelope.runId !== runId || envelope.sourceKind !== "runner" || envelope.eventType !== event.eventType) continue; + const payload = row(envelope.payload), item = row(payload.item); + const execution = typeof event.eventType === "string" && event.eventType.startsWith("tool.execution."); + const callId = execution ? payload.executionId : item.id; + if (typeof callId !== "string" || !callId) continue; + const kind = feedbackKind(execution ? payload.name : item.name); + const started = event.eventType === "tool.execution.started" || event.eventType === "item.started" && (payload.kind === "tool_call" || item.type === "tool_call"); + const completed = event.eventType === "tool.execution.completed" || event.eventType === "item.completed" && (payload.kind === "tool_result" || item.type === "tool_result"); + if ((execution || started || completed) && kind) calls.set(callId, calls.get(callId) ?? { runId, callId, kind, outcome: "unknown" }); + const call = calls.get(callId); + if (!completed || !call || kind && kind !== call.kind) continue; + const status = execution ? payload.status : item.status; + if (payload.isError === true || item.isError === true || ["failed", "error", "cancelled"].includes(String(status))) call.outcome = "failed"; + else if (status === "completed") call.outcome = "completed"; + } + return [...calls.values()]; +} + +export function feedbackReview(entries: readonly AgentCommentaryDetail[]): FeedbackSummary["review"] { + const groups = new Map(); + for (const entry of entries) { + const fingerprint = createHash("sha256").update(`${entry.kind}\n${entry.body.trim().replace(/\s+/g, " ")}`).digest("hex"); + const group = groups.get(fingerprint); + if (group) group.occurrences++; + else groups.set(fingerprint, { fingerprint, kind: entry.kind, occurrences: 1, disposition: "needs_investigation" }); + } + return [...groups.values()]; +} + +export function summarizeCampaignFeedback(attempts: readonly { feedback?: FeedbackSummary }[]) { + const review = new Map(); + let complaints = 0, suggestions = 0, incomplete = 0, failedSubmissions = 0, unknownSubmissions = 0; + for (const { feedback } of attempts) { + if (!feedback || feedback.collection !== "complete" || feedback.invocationCollection !== "complete") incomplete++; + if (!feedback) continue; + complaints += feedback.complaints; + suggestions += feedback.suggestions; + failedSubmissions += feedback.invocations.failed; + unknownSubmissions += feedback.invocations.unknown; + for (const group of feedback.review) { + const previous = review.get(group.fingerprint); + if (previous) previous.occurrences += group.occurrences; + else review.set(group.fingerprint, { ...group }); + } + } + return { complaints, suggestions, incomplete, failedSubmissions, unknownSubmissions, review: [...review.values()] }; +} + +export async function collectAttemptFeedback(input: { + api: Pick; + companyId: string; + persist: (evidence: FeedbackEvidence, signal: AbortSignal) => Promise; + onCheckpoint?: (summary: FeedbackSummary) => void; + deadlineAt?: number; +}): Promise { + const deadlineAt = input.deadlineAt ?? Date.now() + 30_000; + const readDeadlineAt = deadlineAt - Math.min(5_000, Math.max(0, Math.floor((deadlineAt - Date.now()) / 5))); + const summary = emptyFeedbackSummary(); + let checkpointSummary = emptyFeedbackSummary(); + const retainedSummary = () => ({ ...structuredClone(checkpointSummary), + ...(summary.collection !== "complete" ? { collection: summary.collection } : {}), + ...(summary.invocationCollection !== "complete" ? { invocationCollection: summary.invocationCollection } : {}), + }); + const evidence: FeedbackEvidence = { version: 1, companyId: input.companyId, entries: [], invocations: [], diagnostics: [], summary }; + let retainedBytes = 0, checkpointBytes = 0, eventBytes = 0, eventCount = 0; + const diagnose = (code: FeedbackEvidence["diagnostics"][number]) => { if (!evidence.diagnostics.includes(code)) evidence.diagnostics.push(code); }; + const get = (url: string) => { + const remaining = readDeadlineAt - Date.now(); + if (remaining <= 0) { diagnose("deadline_reached"); throw new Error("Feedback collection deadline reached"); } + return input.api.get(url, { timeout: Math.min(5_000, remaining) }); + }; + const persist = async (force = false) => { + summary.bodiesCollected = evidence.entries.length; + summary.invocations = { completed: 0, failed: 0, unknown: 0 }; + for (const invocation of evidence.invocations) summary.invocations[invocation.outcome]++; + summary.toolAvailability = evidence.invocations.length ? "observed" : "unknown"; + if (Date.now() >= deadlineAt) { diagnose("deadline_reached"); return false; } + if (!force && (!retainedBytes || checkpointBytes > 0 && retainedBytes < Math.max(1024 * 1024, checkpointBytes * 2))) return true; + summary.review = feedbackReview(evidence.entries); + const remaining = deadlineAt - Date.now(); + if (remaining <= 0) { diagnose("deadline_reached"); return false; } + const abort = new AbortController(); + let timer: ReturnType | undefined; + try { + await Promise.race([ + new Promise((_resolve, reject) => { timer = setTimeout(() => { + abort.abort(); reject(new Error("Feedback persistence deadline reached")); + }, remaining); }), + input.persist(evidence, abort.signal), + ]); + if (Date.now() >= deadlineAt) { abort.abort(); diagnose("deadline_reached"); return false; } + checkpointBytes = retainedBytes; + checkpointSummary = structuredClone(summary); + input.onCheckpoint?.(structuredClone(checkpointSummary)); + } finally { clearTimeout(timer); } + return true; + }; + if (!await persist(true)) { + summary.collection = "unavailable"; + summary.invocationCollection = "unavailable"; + return summary; + } + const seen = new Set(); + let expectedTotal: number | undefined; + let detailsFailed = false; + feedbackPages: for (let offset = 0; offset < FEEDBACK_COLLECTION_LIMITS.entries; offset += 100) { + let page: AgentCommentaryPage; + try { + page = await get(`/api/companies/${input.companyId}/agent-commentary?limit=100&offset=${offset}`); + if (!Array.isArray(page.items) || !Number.isSafeInteger(page.total) || page.total < 0 || page.items.length > 100) throw new Error("Invalid feedback page"); + } catch { + summary.collection = seen.size ? "partial" : "unavailable"; + diagnose("feedback_read_failed"); + break; + } + expectedTotal ??= page.total; + summary.collection = "partial"; + for (const entry of page.items) { + if (Date.now() >= readDeadlineAt) { diagnose("deadline_reached"); break feedbackPages; } + if (seen.has(entry.id)) continue; + seen.add(entry.id); + if (entry.kind === "complaint") summary.complaints++; + else if (entry.kind === "suggestion") summary.suggestions++; + else { detailsFailed = true; continue; } + try { + const detail = await get(`/api/companies/${input.companyId}/agent-commentary/${encodeURIComponent(entry.id)}`); + if (detail.id !== entry.id || detail.kind !== entry.kind || detail.runId !== entry.runId || typeof detail.body !== "string") throw new Error("Invalid feedback detail"); + const bytes = Buffer.byteLength(JSON.stringify(detail)) + 1; + if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break feedbackPages; } + evidence.entries.push(detail); + retainedBytes += bytes; + } catch { + detailsFailed = true; + diagnose("feedback_detail_failed"); + } + if (evidence.entries.length % 10 === 0 && !await persist()) break feedbackPages; + } + if (!await persist()) break; + if (offset + page.items.length >= page.total || page.items.length < 100) { + if (seen.size === page.total && expectedTotal === page.total && !detailsFailed) summary.collection = "complete"; + else diagnose("feedback_pagination_incomplete"); + break; + } + if (offset + 100 >= FEEDBACK_COLLECTION_LIMITS.entries) diagnose("evidence_limit_reached"); + } + if (!await persist(true)) summary.collection = evidence.entries.length ? "partial" : "unavailable"; + let runs: Array<{ id: string; status?: string }>; + try { + runs = await get(`/api/companies/${input.companyId}/heartbeat-runs?limit=1000`); + if (!Array.isArray(runs) || runs.length > 1000 || runs.some(run => typeof run.id !== "string")) throw new Error("Invalid runs"); + } catch { + summary.invocationCollection = "unavailable"; + diagnose("run_read_failed"); + await persist(true); + return retainedSummary(); + } + summary.invocationCollection = runs.length < 1000 ? "complete" : "partial"; + if (runs.length >= 1000) diagnose("run_limit_reached"); + for (const listed of runs) { + let run: { id: string; status?: string; runtimeMode?: string } = listed; + try { + const detail = await get(`/api/heartbeat-runs/${encodeURIComponent(listed.id)}`); + if (detail.id !== listed.id) throw new Error("Invalid run detail"); + run = detail; + } catch { + summary.invocationCollection = "partial"; + summary.collection = summary.collection === "complete" ? "partial" : summary.collection; + diagnose("run_detail_failed"); + } + if (!["succeeded", "interrupted", "failed", "cancelled", "timed_out"].includes(run.status ?? "")) { + summary.collection = summary.collection === "complete" ? "partial" : summary.collection; + summary.invocationCollection = "partial"; + } + if (run.runtimeMode !== "native") summary.invocationCollection = "partial"; + const events: Row[] = []; + let afterSeq = 0; + try { + for (let pageNumber = 0; pageNumber < 100; pageNumber++) { + const page = await get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=100`); + if (!Array.isArray(page) || page.length > 100) throw new Error("Invalid events"); + for (const event of page) { + if (!Number.isSafeInteger(event.seq) || Number(event.seq) <= afterSeq) throw new Error("Invalid event sequence"); + const bytes = Buffer.byteLength(JSON.stringify(event)); + if (eventCount >= FEEDBACK_COLLECTION_LIMITS.events || eventBytes + bytes > FEEDBACK_COLLECTION_LIMITS.eventBytes) { + diagnose("evidence_limit_reached"); throw new Error("Feedback event limit reached"); + } + eventCount++; eventBytes += bytes; + afterSeq = Number(event.seq); + events.push(event); + } + if (page.length < 100) break; + if (pageNumber === 99) { diagnose("evidence_limit_reached"); throw new Error("Event limit reached"); } + } + summary.runsInspected++; + } catch { + summary.invocationCollection = "partial"; + diagnose("events_read_failed"); + } + for (const invocation of observeFeedbackInvocations(run.id, events)) { + const bytes = Buffer.byteLength(JSON.stringify(invocation)) + 1; + if (retainedBytes + bytes + 4 > FEEDBACK_COLLECTION_LIMITS.retainedBytes) { diagnose("evidence_limit_reached"); break; } + retainedBytes += bytes; + evidence.invocations.push(invocation); + } + if (!await persist() || evidence.diagnostics.includes("evidence_limit_reached")) { + summary.invocationCollection = "partial"; + break; + } + } + if (!await persist(true)) summary.invocationCollection = "partial"; + return retainedSummary(); +} diff --git a/tests/runner-e2e/first-task-flow.ts b/tests/runner-e2e/first-task-flow.ts index 11c16a07921..f131ee9131c 100644 --- a/tests/runner-e2e/first-task-flow.ts +++ b/tests/runner-e2e/first-task-flow.ts @@ -276,6 +276,7 @@ export async function setupFirstTaskFixtures(input: { daytonaImage?: string; deadlineAt?: number; observe: (fixtures: LiveFixtureValues) => void; + onCompanyCreated?: (companyId: string) => Promise; }): Promise { const { page, api, execution, nonce } = input; await api.patch("/api/instance/settings/experimental", { @@ -296,6 +297,7 @@ export async function setupFirstTaskFixtures(input: { const companies = await api.get("/api/companies"); const company = companies.find((c) => c.name === `First task ${nonce}`); if (!company) throw new Error("Onboarding fixture company missing"); + await input.onCompanyCreated?.(company.id); const fixtures = await provisionFirstTaskFixtures({ api, execution, diff --git a/tests/runner-e2e/github-operating-usage.test.ts b/tests/runner-e2e/github-operating-usage.test.ts new file mode 100644 index 00000000000..dac23f7ba35 --- /dev/null +++ b/tests/runner-e2e/github-operating-usage.test.ts @@ -0,0 +1,64 @@ +import { expect, it } from "vitest"; +import { readGitHubOperatingUsage } from "./github-operating-usage.js"; + +const now = Date.parse("2026-10-21T12:00:00Z"), sha = "a".repeat(40); +const run = { id: 123, repository: { id: 456, full_name: "paperclipai/paperclip" }, + path: ".github/workflows/runner-full-stack-e2e.yml", run_attempt: 1, status: "completed", conclusion: "failure", head_sha: sha, + created_at: "2026-10-21T10:00:00Z", run_started_at: "2026-10-21T10:01:00Z", updated_at: "2026-10-21T10:10:00Z" }; +const job = (id = 1) => ({ id, run_id: 123, run_attempt: 1, status: "completed", conclusion: "success", runner_id: 1, + runner_name: "GitHub Actions 1", runner_group_name: "GitHub Actions", labels: ["ubuntu-latest"], + started_at: "2026-10-21T10:01:00Z", completed_at: "2026-10-21T10:02:00Z", name: "unretained arbitrary user text" }); +const artifact = { id: 9, workflow_run: { id: 123, head_sha: sha }, size_in_bytes: 1024, expired: false, + created_at: "2026-10-21T10:03:00Z", expires_at: "2026-11-20T10:03:00Z", name: "unretained arbitrary artifact text" }; +function api(overrides: Record = {}) { + const reads: string[] = []; + return { reads, read: async (endpoint: string) => { + reads.push(endpoint); + if (endpoint === "repos/paperclipai/paperclip") return { id: 456, full_name: "paperclipai/paperclip", private: false, ...overrides.repo }; + if (endpoint.endsWith("/runs/123")) return { ...run, ...overrides.run, ...(reads.filter(row => row.endsWith("/runs/123")).length === 2 ? overrides.finalRun : {}) }; + if (endpoint.includes("/jobs?")) { + const jobs = overrides.jobs ?? [job()]; + const page = Number(new URL(`https://api.github.com/${endpoint}`).searchParams.get("page")); + return { total_count: jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100), ...overrides.jobPage }; + } + if (endpoint.includes("/artifacts?")) return { total_count: 1, artifacts: [artifact], ...overrides.artifactPage }; + throw new Error("Unexpected read"); + } }; +} + +it("collects failed-run usage without equating job time, current rates or inventory with billed cost", async () => { + const source = api(); + const usage = await readGitHubOperatingUsage("123", source.read, now); + expect(usage).toMatchObject({ runAttempt: 1, sourceSha: sha, queueSeconds: 60, summedJobSeconds: 60, + allStartedJobsStandardHosted: true, artifacts: { retainedBytes: 1024, lifetimeCoverage: "incomplete" }, + billing: { ciComputeUsd: null, artifactStorageUsd: null, publicStandardComputeRateUsdPerMinute: "0" } }); + expect(JSON.stringify(usage)).not.toContain("unretained"); + expect(source.reads).toHaveLength(5); +}); + +it("paginates all jobs and preserves nonstandard compute as unpriced", async () => { + const jobs = Array.from({ length: 101 }, (_, index) => job(index + 1)); + jobs[100]!.runner_group_name = "custom-fleet"; + const source = api({ jobs }); + expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 6060, + allStartedJobsStandardHosted: false, billing: { publicStandardComputeRateUsdPerMinute: null } }); + expect(source.reads.some(endpoint => endpoint.includes("page=2"))).toBe(true); + expect((await readGitHubOperatingUsage("123", api({ repo: { private: true } }).read, now)).billing.publicStandardComputeRateUsdPerMinute).toBeNull(); +}); + +it("rejects changing, truncated, duplicated, misattributed, rerun and uncompleted evidence", async () => { + for (const override of [{ run: { run_attempt: 2 } }, { run: { status: "in_progress" } }, { run: { path: "other.yml" } }, + { finalRun: { head_sha: "b".repeat(40) } }, { jobPage: { total_count: 2 } }, { jobs: [job(), job()] }, + { jobs: [{ ...job(), run_id: 999 }] }, { jobs: [{ ...job(), completed_at: null }] }, { jobPage: { total_count: 1001 } }, + { artifactPage: { artifacts: [{ ...artifact, workflow_run: { id: 999, head_sha: sha } }] } }, + { artifactPage: { artifacts: [{ ...artifact, size_in_bytes: -1 }] } }]) { + await expect(readGitHubOperatingUsage("123", api(override).read, now)).rejects.toThrow(); + } +}); + +it("accounts skipped jobs without requiring a fictitious VM and keeps expired artifact identity", async () => { + const source = api({ jobs: [{ ...job(), conclusion: "skipped", runner_id: 0, started_at: null, completed_at: null }], + artifactPage: { artifacts: [{ ...artifact, expired: true }] } }); + expect(await readGitHubOperatingUsage("123", source.read, now)).toMatchObject({ summedJobSeconds: 0, + jobs: [{ runner: "not_started" }], artifacts: { retainedBytes: 0, inventory: [{ id: 9, expired: true }] } }); +}); diff --git a/tests/runner-e2e/github-operating-usage.ts b/tests/runner-e2e/github-operating-usage.ts new file mode 100644 index 00000000000..e55ead9a50e --- /dev/null +++ b/tests/runner-e2e/github-operating-usage.ts @@ -0,0 +1,100 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const repository = "paperclipai/paperclip"; +type Json = Record; +type ReadGitHub = (endpoint: string) => Promise; +const integer = (value: unknown) => Number.isSafeInteger(value) && Number(value) >= 0; +const timestamp = (value: unknown) => typeof value === "string" && Number.isFinite(Date.parse(value)); + +async function inventory(read: ReadGitHub, endpoint: string, field: string) { + let total: number | undefined; + const rows: Json[] = [], ids = new Set(); + for (let page = 1; page <= 10; page++) { + const response = await read(`${endpoint}${endpoint.includes("?") ? "&" : "?"}per_page=100&page=${page}`); + if (!integer(response.total_count) || response.total_count > 1_000 || total !== undefined && total !== response.total_count + || !Array.isArray(response[field]) || response[field].length > 100) throw new Error("Incomplete or changing GitHub inventory"); + total = response.total_count; + for (const row of response[field]) { + if (!row || !integer(row.id) || row.id < 1 || ids.has(row.id)) throw new Error("Invalid or duplicate GitHub inventory row"); + ids.add(row.id); rows.push(row); + } + if (rows.length === total) return rows; + if (rows.length > total! || response[field].length < 100) throw new Error("Incomplete GitHub inventory"); + } + throw new Error("GitHub inventory exceeds collection bound"); +} + +export async function readGitHubOperatingUsage(runId: string, read: ReadGitHub, now = Date.now()) { + if (!/^[1-9][0-9]{0,15}$/.test(runId) || !Number.isSafeInteger(Number(runId)) || !Number.isFinite(now)) throw new Error("Invalid workflow run identity"); + const prefix = `repos/${repository}`; + const repo = await read(prefix); + const run = await read(`${prefix}/actions/runs/${runId}`); + const validRun = (value: Json) => value.id === Number(runId) && value.repository?.id === repo.id + && value.repository?.full_name === repository && value.path === ".github/workflows/runner-full-stack-e2e.yml" + && value.run_attempt === 1 && value.status === "completed" && /^[a-f0-9]{40}$/.test(value.head_sha ?? "") + && timestamp(value.created_at) && timestamp(value.run_started_at) && timestamp(value.updated_at) + && Date.parse(value.created_at) <= Date.parse(value.run_started_at) + && Date.parse(value.run_started_at) <= Date.parse(value.updated_at) && Date.parse(value.updated_at) <= now; + if (!integer(repo.id) || repo.id < 1 || repo.full_name !== repository || typeof repo.private !== "boolean" || !validRun(run)) { + throw new Error("Operating usage requires an identified, completed first-attempt Product E2E workflow"); + } + const rawJobs = await inventory(read, `${prefix}/actions/runs/${runId}/attempts/1/jobs`, "jobs"); + if (!rawJobs.length) throw new Error("Missing workflow jobs"); + const jobs = rawJobs.map(job => { + if (job.run_id !== run.id || job.run_attempt !== 1 || job.status !== "completed" || !integer(job.runner_id) + || !Array.isArray(job.labels) || job.labels.some((label: unknown) => typeof label !== "string")) throw new Error("Invalid workflow job identity"); + const skipped = job.conclusion === "skipped" && job.runner_id === 0; + if (!skipped && (!timestamp(job.started_at) || !timestamp(job.completed_at) + || Date.parse(job.started_at) < Date.parse(run.created_at) || Date.parse(job.completed_at) > now + || Date.parse(job.completed_at) < Date.parse(job.started_at))) throw new Error("Incomplete workflow job timing"); + const standard = job.runner_group_name === "GitHub Actions" && /^GitHub Actions [0-9]+$/.test(job.runner_name ?? "") + && job.labels.length === 1 && job.labels[0] === "ubuntu-latest"; + return { id: job.id as number, runner: skipped ? "not_started" : standard ? "standard_github_hosted" : "other_or_unknown", + elapsedSeconds: skipped ? 0 : (Date.parse(job.completed_at) - Date.parse(job.started_at)) / 1_000 }; + }); + const rawArtifacts = await inventory(read, `${prefix}/actions/runs/${runId}/artifacts`, "artifacts"); + const artifacts = rawArtifacts.map(artifact => { + if (artifact.workflow_run?.id !== run.id || artifact.workflow_run?.head_sha !== run.head_sha || !integer(artifact.size_in_bytes) + || typeof artifact.expired !== "boolean" || !timestamp(artifact.created_at) || !timestamp(artifact.expires_at) + || Date.parse(artifact.created_at) < Date.parse(run.created_at) || Date.parse(artifact.created_at) > now + || Date.parse(artifact.expires_at) < Date.parse(artifact.created_at)) throw new Error("Invalid workflow artifact identity or retention"); + return { id: artifact.id as number, sizeBytes: artifact.size_in_bytes as number, + createdAt: artifact.created_at as string, expiresAt: artifact.expires_at as string, expired: artifact.expired as boolean }; + }); + const finalRun = await read(`${prefix}/actions/runs/${runId}`); + if (!validRun(finalRun) || ["id", "head_sha", "run_attempt", "status", "updated_at", "conclusion"].some(key => finalRun[key] !== run[key])) { + throw new Error("Workflow changed during operating usage collection"); + } + const retainedBytes = artifacts.filter(row => !row.expired).reduce((sum, row) => sum + row.sizeBytes, 0); + if (!Number.isSafeInteger(retainedBytes)) throw new Error("Artifact inventory byte count overflow"); + return { + version: 1, source: "github_actions", repository, repositoryId: repo.id as number, + observedAt: new Date(now).toISOString(), repositoryCurrentlyPublic: !repo.private, + runId: run.id as number, runAttempt: 1, sourceSha: run.head_sha as string, + workflowUrl: `https://github.com/${repository}/actions/runs/${runId}`, + startedAt: run.run_started_at as string, completedSnapshotAt: run.updated_at as string, + queueSeconds: (Date.parse(run.run_started_at) - Date.parse(run.created_at)) / 1_000, + summedJobSeconds: jobs.reduce((sum, row) => sum + row.elapsedSeconds, 0), + allStartedJobsStandardHosted: jobs.every(row => row.runner !== "other_or_unknown"), jobs, + artifacts: { inventory: artifacts, retainedBytes, lifetimeCoverage: "incomplete" }, + billing: { ciComputeUsd: null, artifactStorageUsd: null, classification: "unknown", + publicStandardComputeRateUsdPerMinute: !repo.private && jobs.every(row => row.runner !== "other_or_unknown") ? "0" : null, + rateSource: "https://docs.github.com/en/billing/concepts/product-billing/github-actions", + limitations: ["Current repository visibility does not prove visibility throughout the billing window.", + "Job elapsed time is not VM billed lifetime; queue and summed job time are separate.", + "Artifacts already deleted are absent; returned compressed bytes and expiry are an inventory, not billed byte-hours.", + "Caches, logs, external S3 history, model calls, observability and shared subscriptions require separate evidence."] }, + }; +} + +export async function collectGitHubOperatingUsage(runId: string) { + const run = promisify(execFile); + return readGitHubOperatingUsage(runId, async endpoint => { + try { + const result = await run("gh", ["api", "--hostname", "github.com", "--method", "GET", endpoint], + { timeout: 15_000, maxBuffer: 8 * 1024 * 1024, encoding: "utf8" }); + return JSON.parse(result.stdout) as Json; + } catch { throw new Error("GitHub operating usage read failed; no retry or paid dispatch was attempted"); } + }); +} diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index c52a430276e..2ea3cd91a40 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -11,6 +11,9 @@ const AMBIENT_PAPERCLIP_CREDENTIAL_KEYS = [ "PAPERCLIP_API_KEY", "PAPERCLIP_AGENT_API_KEY", "PAPERCLIP_TASK_BRIDGE_TOKEN", + "PAPERCLIP_E2E_GATEWAY_TOKEN", + "PAPERCLIP_E2E_GATEWAY_PHASE_PATH", + "PAPERCLIP_E2E_OTLP_RELAY_URL", "PAPERCLIP_SETUP_TOKEN", "PAPERCLIP_SECRETS_MASTER_KEY", "PAPERCLIP_SECRETS_MASTER_KEY_FILE", @@ -28,7 +31,25 @@ const AMBIENT_EXTERNAL_STATE_KEYS = [ "PAPERCLIP_STORAGE_S3_PREFIX", "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", ] as const; -const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/; +const PROVIDER_SECRET_KEY = /^(?:OPENAI|ANTHROPIC|OPENROUTER|BRAINTRUST|DAYTONA|XAI|GROK|CURSOR|COPILOT|GITHUB|GH)(?:_|$)/; + +export function applyEvalGatewayEnvironment(env: NodeJS.ProcessEnv, credential: string, gateway: { + token: string; baseUrl: string; phasePath: string; observability?: { token: string; baseUrl: string }; +}) { + for (const name of Object.keys(env)) { + if (CREDENTIAL_NAMES.includes(name as typeof CREDENTIAL_NAMES[number]) + || PROVIDER_SECRET_KEY.test(name) && /KEY|TOKEN|SECRET|PASSWORD/.test(name)) delete env[name]; + } + delete env.PAPERCLIP_E2E_OTLP_RELAY_URL; + env[credential] = gateway.token; + env.PAPERCLIP_E2E_GATEWAY_BASE_URL = gateway.baseUrl; + env.PAPERCLIP_E2E_GATEWAY_TOKEN = gateway.token; + env.PAPERCLIP_E2E_GATEWAY_PHASE_PATH = gateway.phasePath; + if (gateway.observability) { + env.BRAINTRUST_API_KEY = gateway.observability.token; + env.PAPERCLIP_E2E_OTLP_RELAY_URL = gateway.observability.baseUrl; + } +} export function runnerE2EServerControlPaths(temporaryRoot: string) { const controlDirectory = path.join(temporaryRoot, "control"); @@ -110,6 +131,8 @@ export function buildRunnerE2EProcessEnvironment( const installed = usesInstalledCli(source); if (installed) assertInstalledCliSelection(source, executions); const result = { ...source }; + delete result.PAPERCLIP_E2E_GATEWAY_PHASE_PATH; + delete result.PAPERCLIP_E2E_OTLP_RELAY_URL; // Announcements are unrelated to the scenarios and obscure screenshot evidence. result.PAPERCLIP_ANNOUNCEMENTS_ENABLED = "false"; delete result.OPENCODE_ALLOW_ALL_MODELS; diff --git a/tests/runner-e2e/history-budget.test.ts b/tests/runner-e2e/history-budget.test.ts new file mode 100644 index 00000000000..3b6b1aa1906 --- /dev/null +++ b/tests/runner-e2e/history-budget.test.ts @@ -0,0 +1,170 @@ +import { mkdir, mkdtemp, readFile, rm, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS, validateBoundedHistoryRetention } from "./history-budget.js"; +import { createBundleManifest, uploadImmutableBundle } from "./history-publish.js"; +import { buildRunnerCampaign, campaignHistoryRecord, emptyRunnerHistory, mergeRunnerHistory } from "./history.js"; + +const roots: string[] = []; +async function temporary() { + const root = await mkdtemp(path.join(os.tmpdir(), "history-budget-")); + roots.push(root); + return root; +} +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(roots.splice(0).map(root => rm(root, { force: true, recursive: true }))); +}); + +function lifecycle() { + return { Rules: [ + { Status: "Enabled", Filter: { Prefix: "" }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 7 } }, + { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { Days: 30 }, + NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } }, + { Status: "Enabled", Filter: { Prefix: "runner-e2e/bounded-v1/" }, Expiration: { ExpiredObjectDeleteMarker: true } }, + ] }; +} +function providerError(code: string) { return Object.assign(new Error("provider failure"), { stderr: `An error occurred (${code})` }); } + +describe("bounded public history", () => { + it("rejects excess bytes, count, invalid sizes and duplicate paths", () => { + expect(checkHistoryPayload([{ path: "a", bytes: 8 }], 8, 1, "report")).toEqual({ files: 1, bytes: 8 }); + for (const files of [[{ path: "a", bytes: 9 }], [{ path: "a", bytes: -1 }], + [{ path: "a", bytes: 0 }, { path: "b", bytes: 0 }], [{ path: "a", bytes: 0 }, { path: "a", bytes: 0 }]]) { + expect(() => checkHistoryPayload(files, 8, 1, "report")).toThrow(); + } + }); + + it("rejects oversized real bundle files before hashing them", async () => { + const root = await temporary(); + await writeFile(path.join(root, "index.html"), ""); + await truncate(path.join(root, "index.html"), HISTORY_LIMITS.campaignBytes + 1); + await expect(createBundleManifest(root, "oversized", false, new Set(), true)).rejects.toThrow("publication boundary"); + }); + + it("uploads an allowlisted uppercase PNG with the same raster content type", async () => { + const root = await temporary(); const bundle = path.join(root, "bundle"); + const screenshot = "evidence/execution/attempt-1/shot.PNG"; + await mkdir(path.dirname(path.join(bundle, screenshot)), { recursive: true }); + await writeFile(path.join(bundle, screenshot), Buffer.from("89504e470d0a1a0a0000000d494844520000000100000001", "hex")); + const manifest = await createBundleManifest(bundle, "uppercase", false, new Set([screenshot]), true); + const runner = vi.fn(async () => ({ stdout: "{}" })); + await uploadImmutableBundle("bucket", "campaigns/uppercase", bundle, manifest, + new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner)); + expect(runner).toHaveBeenCalledWith(expect.arrayContaining(["--content-type", "image/png", "--key", `campaigns/uppercase/${screenshot}`]), expect.any(Number)); + }); + + it("requires expiration for both current and noncurrent versions in the isolated prefix", () => { + expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e/bounded-v1")).not.toThrow(); + expect(() => validateBoundedHistoryRetention(lifecycle(), "runner-e2e")).toThrow("isolated"); + for (const mutate of [ + (config: ReturnType) => { config.Rules[1]!.Status = "Disabled"; }, + (config: ReturnType) => { config.Rules[1]!.NoncurrentVersionExpiration = undefined; }, + (config: ReturnType) => { Object.assign(config.Rules[1]!.NoncurrentVersionExpiration!, { NewerNoncurrentVersions: 10 }); }, + (config: ReturnType) => { Object.assign(config.Rules[1]!.Filter, { Tag: { Key: "optional", Value: "true" } }); }, + (config: ReturnType) => { config.Rules.pop(); }, + ]) { + const config = lifecycle(); mutate(config); + expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("expiry"); + } + }); + + it("rejects overlapping transitions or earlier expiration without affecting unrelated prefixes", () => { + const config = lifecycle(); + Object.assign(config.Rules[0]!, { Transitions: [{ Days: 1, StorageClass: "GLACIER" }] }); + expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping"); + config.Rules[0]!.Filter.Prefix = "storybook/"; + expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).not.toThrow(); + Object.assign(config.Rules[2]!.Expiration!, { Days: 1 }); + expect(() => validateBoundedHistoryRetention(config, "runner-e2e/bounded-v1")).toThrow("overlapping"); + }); + + it("rebuilds pointers after removing old, future and excess history records", () => { + const now = Date.parse("2026-10-21T12:00:00Z"); + const record = (id: string, age: number) => campaignHistoryRecord(buildRunnerCampaign({ campaignId: id, + generatedAt: new Date(now - age * 86_400_000).toISOString(), expected: [], results: [] }), "https://example.test/bounded-v1"); + const old = { ...record("expired-green", 29), complete: true, failed: 0 }; + const current = { ...emptyRunnerHistory(), campaigns: [old, record("future", -2), record("yesterday", 1), record("earlier", 2)] }; + const result = mergeRunnerHistory(current, record("today", 0), { now, maxAgeMs: 28 * 86_400_000, maxCampaigns: 2 }); + expect(result.campaigns.map(row => row.campaignId)).toEqual(["today", "yesterday"]); + expect(result.latestCampaignId).toBe("today"); + expect(result.latestGreenCampaignId).not.toBe("expired-green"); + expect(current.campaigns).toHaveLength(4); + }); + + it("distinguishes absent lock/replication from denied inspection", async () => { + const root = await temporary(); + const runner = vi.fn(async (args: string[]) => { + if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' }; + if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) }; + throw providerError(args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError"); + }); + await new BoundedHistoryPublisher(path.join(root, "allowed.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1"); + runner.mockImplementation(async () => { throw providerError("AccessDenied"); }); + await expect(new BoundedHistoryPublisher(path.join(root, "denied.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow(); + }); + + it.each(["get-object-lock-configuration", "get-bucket-replication"])("rejects configured %s", async blocked => { + const root = await temporary(); + const runner = async (args: string[]) => { + if (args[1] === "get-bucket-versioning") return { stdout: '{"Status":"Enabled"}' }; + if (args[1] === "get-bucket-lifecycle-configuration") return { stdout: JSON.stringify(lifecycle()) }; + if (args[1] === blocked) return { stdout: "{}" }; + throw providerError("ObjectLockConfigurationNotFoundError"); + }; + await expect(new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner).verifyRetention("bucket", "runner-e2e/bounded-v1")).rejects.toThrow("cannot prove"); + }); + + it("bounds history reads and preserves permission errors", async () => { + const root = await temporary(); const destination = path.join(root, "download.json"); + const runner = vi.fn(async (args: string[]) => { + expect(args).toContain(`bytes=0-${HISTORY_LIMITS.jsonBytes}`); + await writeFile(destination, ""); await truncate(destination, HISTORY_LIMITS.jsonBytes + 1); + return { stdout: "{}" }; + }); + const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner); + await expect(publisher.downloadJson("bucket", "history.json", destination)).rejects.toThrow("read boundary"); + runner.mockImplementation(async () => { throw providerError("NoSuchKey"); }); + await expect(publisher.downloadJson("bucket", "missing", destination)).resolves.toBeNull(); + runner.mockImplementation(async () => { throw providerError("AccessDenied"); }); + await expect(publisher.downloadJson("bucket", "denied", destination)).rejects.toThrow(); + }); + + it("retains an attempted upload before dispatch and preserves unknown failures", async () => { + const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json"); + await writeFile(source, "12345"); + const runner = vi.fn(async (args: string[], timeout: number) => { + expect(args).toContain("--if-none-match"); expect(args).toContain("STANDARD"); expect(timeout).toBeLessThanOrEqual(30_000); + expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ attemptedSourceBytes: 5, requests: [{ status: "unknown" }] }); + throw new Error("lost response"); + }); + const publisher = new BoundedHistoryPublisher(receipt, runner); + await expect(publisher.upload("bucket", "campaign/a", source, "application/json", "no-cache", true)).rejects.toThrow("lost response"); + expect(runner).toHaveBeenCalledTimes(1); + expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, attemptedSourceBytes: 5, + requests: [{ operation: "put-object", status: "failed_or_unknown", sourceBytes: 5 }] }); + }); + + it("stops before another command when the request or elapsed-time boundary is reached", async () => { + const root = await temporary(); const runner = vi.fn(async () => ({ stdout: "{}" })); + const publisher = new BoundedHistoryPublisher(path.join(root, "receipt.json"), runner); + for (let i = 0; i < HISTORY_LIMITS.requests; i++) await publisher.run(["s3api", "head-object"]); + await expect(publisher.run(["s3api", "head-object"])).rejects.toThrow("allowance"); + expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests); + const timed = new BoundedHistoryPublisher(path.join(root, "timed.json"), runner); + vi.spyOn(Date, "now").mockReturnValue(Date.now() + HISTORY_LIMITS.deadlineMs + 1); + await expect(timed.run(["s3api", "head-object"])).rejects.toThrow("allowance"); + expect(runner).toHaveBeenCalledTimes(HISTORY_LIMITS.requests); + }); + + it("retains the object identity and provider version separately from billed cost", async () => { + const root = await temporary(); const receipt = path.join(root, "receipt.json"); const source = path.join(root, "source.json"); + await writeFile(source, "{}"); + const publisher = new BoundedHistoryPublisher(receipt, async () => ({ stdout: '{"VersionId":"version-1","ETag":"etag-1"}' })); + await publisher.upload("bucket", "campaign/result.json", source, "application/json", "no-cache", true); + expect(JSON.parse(await readFile(receipt, "utf8"))).toMatchObject({ billedUsd: null, requests: [{ + operation: "put-object", status: "accepted", bucket: "bucket", key: "campaign/result.json", versionId: "version-1", etag: "etag-1", sourceBytes: 2, + }] }); + }); +}); diff --git a/tests/runner-e2e/history-budget.ts b/tests/runner-e2e/history-budget.ts new file mode 100644 index 00000000000..b7ffdf89d15 --- /dev/null +++ b/tests/runner-e2e/history-budget.ts @@ -0,0 +1,177 @@ +import { execFile } from "node:child_process"; +import { open, readFile, rename, stat } from "node:fs/promises"; +import { promisify } from "node:util"; + +export const HISTORY_LIMITS = Object.freeze({ + campaignBytes: 64 * 1024 * 1024, + campaignFiles: 256, + mutableBytes: 16 * 1024 * 1024, + pagesBytes: 64 * 1024 * 1024, + jsonBytes: 4 * 1024 * 1024, + indexDays: 28, + indexCampaigns: 64, + requests: 280, + deadlineMs: 10 * 60_000, +}); + +export function checkHistoryPayload( + files: readonly { path: string; bytes: number }[], + maxBytes: number, + maxFiles: number, + label: string, +) { + let bytes = 0; + const paths = new Set(); + for (const file of files) { + if (paths.has(file.path) || !Number.isSafeInteger(file.bytes) || file.bytes < 0) { + throw new Error(`${label} contains invalid or duplicate file metadata`); + } + paths.add(file.path); + bytes += file.bytes; + } + if (!Number.isSafeInteger(bytes) || bytes > maxBytes || files.length > maxFiles) { + throw new Error(`${label} exceeds the publication boundary (${files.length} files, ${bytes} bytes)`); + } + return { files: files.length, bytes }; +} + +interface LifecycleRule { + Status?: string; + Prefix?: string; + Filter?: { Prefix?: string; And?: { Prefix?: string } }; + Expiration?: { Days?: number; Date?: string; ExpiredObjectDeleteMarker?: boolean }; + NoncurrentVersionExpiration?: { NoncurrentDays?: number; NewerNoncurrentVersions?: number }; + AbortIncompleteMultipartUpload?: { DaysAfterInitiation?: number }; + Transitions?: unknown[]; + NoncurrentVersionTransitions?: unknown[]; +} + +export function validateBoundedHistoryRetention(configuration: unknown, prefix: string) { + const rules = (configuration as { Rules?: LifecycleRule[] })?.Rules; + if (!Array.isArray(rules) || !prefix.endsWith("/bounded-v1")) { + throw new Error("Bounded history requires an isolated bounded-v1 prefix and lifecycle rules"); + } + const scoped = rules.filter(rule => rule.Status === "Enabled" && + rule.Filter?.Prefix === `${prefix}/` && Object.keys(rule.Filter).length === 1); + const retention = scoped.some(rule => rule.Expiration?.Days === 30 && + rule.NoncurrentVersionExpiration?.NoncurrentDays === 1 && + rule.NoncurrentVersionExpiration?.NewerNoncurrentVersions === undefined && + rule.AbortIncompleteMultipartUpload?.DaysAfterInitiation === 1); + const markers = scoped.some(rule => rule.Expiration?.ExpiredObjectDeleteMarker === true); + if (!retention || !markers) { + throw new Error("Bounded history needs 30-day current, 1-day noncurrent, delete-marker and multipart expiry"); + } + for (const rule of rules.filter(rule => rule.Status === "Enabled")) { + const filterPrefix = rule.Filter?.Prefix ?? rule.Filter?.And?.Prefix ?? rule.Prefix ?? ""; + if (!`${prefix}/`.startsWith(filterPrefix) && !filterPrefix.startsWith(`${prefix}/`)) continue; + if (rule.Transitions?.length || rule.NoncurrentVersionTransitions?.length || + (rule.Expiration && !scoped.includes(rule))) { + throw new Error("An overlapping lifecycle rule changes bounded history retention or storage class"); + } + if (rule.Expiration?.Days !== undefined && rule.Expiration.Days !== 30 || rule.Expiration?.Date) { + throw new Error("An overlapping lifecycle rule changes bounded history expiration"); + } + } +} + +type AwsRunner = (args: string[], timeout: number) => Promise<{ stdout: string }>; +const execFileAsync = promisify(execFile); +const defaultRunner: AwsRunner = async (args, timeout) => execFileAsync("aws", args, { + timeout, maxBuffer: 1024 * 1024, + env: { ...process.env, AWS_MAX_ATTEMPTS: "1", AWS_RETRY_MODE: "standard", AWS_PAGER: "" }, +}); + +function errorDetail(error: unknown) { + return String((error as { stderr?: string })?.stderr ?? (error instanceof Error ? error.message : error)); +} + +export class BoundedHistoryPublisher { + private readonly startedAt = Date.now(); + private readonly observations: Array<{ operation: string; status: string; bucket?: string; key?: string; sourceBytes?: number; versionId?: string; etag?: string }> = []; + + constructor(private readonly receipt: string, private readonly runner: AwsRunner = defaultRunner) {} + + async run(args: string[], sourceBytes?: number) { + const remaining = HISTORY_LIMITS.deadlineMs - (Date.now() - this.startedAt); + if (remaining <= 0 || this.observations.length >= HISTORY_LIMITS.requests) { + throw new Error("Bounded history publisher exhausted its time or request allowance"); + } + const argument = (name: string) => args.includes(name) ? args[args.indexOf(name) + 1] : undefined; + const observation: typeof this.observations[number] = { operation: args[1]!, status: "unknown", + bucket: argument("--bucket"), key: argument("--key"), ...(sourceBytes === undefined ? {} : { sourceBytes }) }; + this.observations.push(observation); + await this.save(); + try { + const result = await this.runner([...args, "--cli-connect-timeout", "10", "--cli-read-timeout", "30"], Math.min(30_000, remaining)); + observation.status = "accepted"; + if (observation.operation === "put-object") { + const response = JSON.parse(result.stdout) as { VersionId?: unknown; ETag?: unknown }; + if (typeof response.VersionId === "string") observation.versionId = response.VersionId.slice(0, 1024); + if (typeof response.ETag === "string") observation.etag = response.ETag.slice(0, 1024); + } + return result; + } catch (error) { + observation.status = /\((?:NoSuchKey|404|NotFound|ObjectLockConfigurationNotFoundError|ReplicationConfigurationNotFoundError)\)/.test(errorDetail(error)) + ? "absent" : "failed_or_unknown"; + throw error; + } finally { + await this.save(); + } + } + + async save() { + const file = await open(`${this.receipt}.tmp`, "w", 0o600); + try { + await file.writeFile(`${JSON.stringify({ + schema: "paperclip.runner-e2e.history-publication/v1", + startedAt: new Date(this.startedAt).toISOString(), observedAt: new Date().toISOString(), + workflowRunId: process.env.GITHUB_RUN_ID ?? null, workflowAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, + limits: HISTORY_LIMITS, billedUsd: null, requests: this.observations, + attemptedSourceBytes: this.observations.reduce((sum, entry) => sum + (entry.sourceBytes ?? 0), 0), + }, null, 2)}\n`); + await file.sync(); + } finally { + await file.close(); + } + await rename(`${this.receipt}.tmp`, this.receipt); + } + + async verifyRetention(bucket: string, prefix: string) { + const config = async (operation: string) => JSON.parse((await this.run(["s3api", operation, "--bucket", bucket])).stdout); + if ((await config("get-bucket-versioning")).Status !== "Enabled") { + throw new Error("Bounded history requires versioning-enabled storage"); + } + validateBoundedHistoryRetention(await config("get-bucket-lifecycle-configuration"), prefix); + for (const [operation, absence] of [ + ["get-object-lock-configuration", "ObjectLockConfigurationNotFoundError"], + ["get-bucket-replication", "ReplicationConfigurationNotFoundError"], + ]) { + try { + await config(operation!); + } catch (error) { + if (errorDetail(error).includes(`(${absence})`)) continue; + throw error; + } + throw new Error(`Bounded history cannot prove expiration with ${operation} configured`); + } + } + + async downloadJson(bucket: string, key: string, destination: string): Promise { + try { + await this.run(["s3api", "get-object", "--bucket", bucket, "--key", key, + "--range", `bytes=0-${HISTORY_LIMITS.jsonBytes}`, destination]); + } catch (error) { + if (/\((?:NoSuchKey|404|NotFound)\)/.test(errorDetail(error))) return null; + throw error; + } + if ((await stat(destination)).size > HISTORY_LIMITS.jsonBytes) throw new Error("History JSON exceeds the read boundary"); + return JSON.parse(await readFile(destination, "utf8")) as T; + } + + async upload(bucket: string, key: string, file: string, contentType: string, cacheControl: string, immutable = false) { + const bytes = (await stat(file)).size; + return this.run(["s3api", "put-object", "--bucket", bucket, "--key", key, "--body", file, + "--content-length", String(bytes), "--content-type", contentType, "--cache-control", cacheControl, + "--storage-class", "STANDARD", ...(immutable ? ["--if-none-match", "*"] : [])], bytes); + } +} diff --git a/tests/runner-e2e/history-index.ts b/tests/runner-e2e/history-index.ts index 5c72cb9dde3..2845cc834fa 100644 --- a/tests/runner-e2e/history-index.ts +++ b/tests/runner-e2e/history-index.ts @@ -25,7 +25,8 @@ function usd(value: number | null) { return `$${value.toFixed(value < 0.01 ? 6 : 2)}`; } -function duration(durationMs: number) { +function duration(durationMs: number | null) { + if (durationMs === null) return "Unknown"; const seconds = Math.round(durationMs / 1_000); if (seconds < 60) return `${seconds}s`; const minutes = Math.floor(seconds / 60); diff --git a/tests/runner-e2e/history-publish.ts b/tests/runner-e2e/history-publish.ts index 80159a5f0a3..a39dee6ab7c 100644 --- a/tests/runner-e2e/history-publish.ts +++ b/tests/runner-e2e/history-publish.ts @@ -1,4 +1,6 @@ -import { validateRetainedRunnerResult } from "./result-validation.js"; +import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js"; +import { validateEvalOperationsReport } from "./operations-report.js"; +import { validateCampaignFeedbackReview } from "./feedback-dispositions.js"; import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; import path from "node:path"; @@ -22,6 +24,7 @@ import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; import { renderRunnerHistoryIndex } from "./history-index.js"; import { validateHistoryDestination } from "./history-destination.js"; import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js"; +import { BoundedHistoryPublisher, checkHistoryPayload, HISTORY_LIMITS } from "./history-budget.js"; import { campaignHistoryRecord, emptyRunnerHistory, @@ -277,10 +280,12 @@ async function relativeFiles( current = root, allowPublicSummary = false, publicScreenshots: ReadonlySet = EMPTY_PUBLIC_SCREENSHOTS, + boundary?: { entries: number }, ): Promise { const entries = await readdir(current, { withFileTypes: true }); const files: string[] = []; for (const entry of entries) { + if (boundary && ++boundary.entries > 1024) throw new Error("Public bundle exceeds the directory-entry boundary"); const absolute = path.join(current, entry.name); if (entry.isSymbolicLink()) { throw new Error(`Refusing to publish symbolic link ${entry.name}`); @@ -292,6 +297,7 @@ async function relativeFiles( absolute, allowPublicSummary, publicScreenshots, + boundary, )), ); } else if (entry.isFile()) { @@ -319,11 +325,17 @@ export async function createBundleManifest( campaignId: string, allowPublicSummary = false, publicScreenshots: ReadonlySet = EMPTY_PUBLIC_SCREENSHOTS, + bounded = false, ): Promise { if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(campaignId)) { throw new Error("Campaign ID is unsafe for immutable object storage"); } - const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots); + const retainedPaths = await relativeFiles(root, root, allowPublicSummary, publicScreenshots, bounded ? { entries: 0 } : undefined); + if (bounded) { + checkHistoryPayload(await Promise.all(retainedPaths.map(async file => ({ + path: file, bytes: (await stat(path.join(root, file))).size, + }))), HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign"); + } const retained = new Set(retainedPaths); const missingScreenshots = [...publicScreenshots].filter((relative) => !retained.has(relative)); if (missingScreenshots.length > 0) { @@ -431,7 +443,9 @@ async function downloadJson( bucket: string, key: string, destination: string, + bounded?: BoundedHistoryPublisher, ) { + if (bounded) return bounded.downloadJson(bucket, key, destination); if (!(await objectExists(bucket, key))) return null; await execFileAsync("aws", [ "s3", @@ -472,11 +486,24 @@ async function uploadFile( ]); } -async function uploadImmutableBundle( +export async function uploadImmutableBundle( bucket: string, key: string, directory: string, + manifest?: BundleManifest, + bounded?: BoundedHistoryPublisher, ) { + if (bounded && manifest) { + for (const file of manifest.files) { + const extension = path.extname(file.path).toLowerCase(); + const contentType = ({ ".png": "image/png", ".json": "application/json", ".html": "text/html; charset=utf-8", + ".woff2": "font/woff2", ".xml": "application/xml", ".md": "text/plain; charset=utf-8" } as Record)[extension]; + if (!contentType) throw new Error(`Missing public content type for ${file.path}`); + await bounded.upload(bucket, `${key}/${file.path}`, path.join(directory, file.path), contentType, + "public,max-age=2592000,immutable", true); + } + return; + } const common = [ "--recursive", "--only-show-errors", @@ -524,10 +551,19 @@ async function main() { prefix: process.env.RUNNER_E2E_HISTORY_PREFIX ?? "runner-e2e", publicBaseUrl: process.env.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL ?? "", }); + const bounded = process.env.PAPERCLIP_E2E_BOUNDED_HISTORY === "true"; + if (bounded && process.env.GITHUB_ACTIONS === "true" && process.env.GITHUB_RUN_ATTEMPT !== "1") { + throw new Error("Bounded history publication requires the original workflow attempt; retain failed publication evidence"); + } + const publisher = bounded ? new BoundedHistoryPublisher(path.join(path.dirname(reportRoot), "history-publication-receipt.json")) : undefined; + if (publisher) await publisher.verifyRetention(bucket, destination.prefix); const campaign = JSON.parse( await readFile(path.join(reportRoot, "normalized-results.json"), "utf8"), ) as RunnerE2ECampaign; for (const result of campaign.results ?? []) validateRetainedRunnerResult(result); + if (campaign.accounting) validateCampaignAccounting(campaign.accounting); + if (campaign.operations) validateEvalOperationsReport(campaign.operations); + if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview); assertPublicCampaign(campaign); if (campaign.schema !== "paperclip.runner-e2e.campaign/v2") { throw new Error("Historical publishing requires a v2 normalized campaign"); @@ -542,6 +578,7 @@ async function main() { bucket, historyKey, path.join(temporary, "current-history.json"), + publisher, )) ?? emptyRunnerHistory(); const history = mergeRunnerHistory( current, @@ -549,7 +586,11 @@ async function main() { campaign, `${destination.publicBaseUrl}/${destination.prefix}`, ), + bounded ? { now: Date.now(), maxAgeMs: HISTORY_LIMITS.indexDays * 86_400_000, maxCampaigns: HISTORY_LIMITS.indexCampaigns } : undefined, ); + if (bounded && !history.campaigns.some(entry => entry.campaignId === campaign.campaignId)) { + throw new Error("Bounded campaign is outside the retained history window"); + } const publicScreenshots = publicScreenshotPaths(campaign); // Campaign bundles are immutable and must not capture a mutable history @@ -581,6 +622,7 @@ async function main() { campaign.campaignId, true, publicScreenshots, + bounded, ); const campaignPrefix = `${destination.prefix}/campaigns/${campaign.campaignId}`; const manifestKey = `${campaignPrefix}/bundle-manifest.json`; @@ -588,6 +630,7 @@ async function main() { bucket, manifestKey, path.join(temporary, "existing-manifest.json"), + publisher, ); if ( existingManifest && @@ -597,17 +640,8 @@ async function main() { `Immutable campaign ${campaign.campaignId} already exists with a different digest`, ); } - if (!existingManifest) { - await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot); - const manifestFile = path.join(temporary, "bundle-manifest.json"); - await writeFile(manifestFile, json(manifest), "utf8"); - await uploadJson( - bucket, - manifestKey, - manifestFile, - "public,max-age=31536000,immutable", - ); - } + const manifestFile = path.join(temporary, "bundle-manifest.json"); + await writeFile(manifestFile, json(manifest), "utf8"); // Pages uses the same declared-screenshot boundary in a sibling stage. The // downloaded access-controlled report stays intact for this whole job. @@ -630,11 +664,12 @@ async function main() { writeFile(latestGreenFile, json(pointers.latestGreen), "utf8"), ]); await regenerateRunnerDashboard({ bundle: pagesRoot, historyFile }); - await createBundleManifest( + const pagesManifest = await createBundleManifest( pagesRoot, campaign.campaignId, false, publicScreenshots, + bounded, ); const landingDirectory = path.join(temporary, "landing"); await regenerateRunnerDashboard({ @@ -648,6 +683,37 @@ async function main() { renderRunnerHistoryIndex(history), "utf8", ); + if (publisher) { + const mutableFiles = [ + { key: historyKey, file: historyFile, contentType: "application/json" }, + { key: `${destination.prefix}/latest.json`, file: latestFile, contentType: "application/json" }, + { key: `${destination.prefix}/latest-green.json`, file: latestGreenFile, contentType: "application/json" }, + ...["index.html", "dashboard.html", "normalized-results.json"].map(file => ({ key: `${destination.prefix}/${file}`, + file: path.join(landingDirectory, file), contentType: file.endsWith("json") ? "application/json" : "text/html; charset=utf-8" })), + ...["favicon-32x32.png", "InterVariable.woff2"].map(file => ({ key: `${destination.prefix}/assets/${file}`, + file: path.join(s3ReportRoot, "assets", file), contentType: file.endsWith("png") ? "image/png" : "font/woff2" })), + ]; + const metadata = async (file: string) => ({ path: file, bytes: (await stat(file)).size }); + const campaignSize = checkHistoryPayload([...manifest.files, await metadata(manifestFile)], + HISTORY_LIMITS.campaignBytes, HISTORY_LIMITS.campaignFiles, "Public campaign with manifest"); + const mutableMetadata = await Promise.all(mutableFiles.map(file => metadata(file.file))); + const mutableSize = checkHistoryPayload(mutableMetadata, HISTORY_LIMITS.mutableBytes, 8, "Mutable history"); + const pointerMetadata = await Promise.all([historyFile, latestFile, latestGreenFile].map(metadata)); + checkHistoryPayload(pointerMetadata, HISTORY_LIMITS.jsonBytes, 3, "History JSON"); + const pagesSize = checkHistoryPayload([...pagesManifest.files, ...pointerMetadata], + HISTORY_LIMITS.pagesBytes, HISTORY_LIMITS.campaignFiles + 3, "Pages bundle"); + if (!existingManifest) { + await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot, manifest, publisher); + await publisher.upload(bucket, manifestKey, manifestFile, "application/json", "public,max-age=2592000,immutable", true); + } + for (const file of mutableFiles) await publisher.upload(bucket, file.key, file.file, file.contentType, "no-cache"); + console.log(JSON.stringify({ campaign: campaignSize, mutable: mutableSize, pages: pagesSize, billedUsd: null })); + return; + } + if (!existingManifest) { + await uploadImmutableBundle(bucket, campaignPrefix, s3ReportRoot); + await uploadJson(bucket, manifestKey, manifestFile, "public,max-age=31536000,immutable"); + } await Promise.all([ uploadJson(bucket, historyKey, historyFile, "no-cache"), uploadJson( diff --git a/tests/runner-e2e/history.test.ts b/tests/runner-e2e/history.test.ts index c60278769d0..c717441e321 100644 --- a/tests/runner-e2e/history.test.ts +++ b/tests/runner-e2e/history.test.ts @@ -69,6 +69,41 @@ function result(execution: MatrixExecution, status: "passed" | "failed") { } describe("runner E2E campaign history", () => { + it("retains the supervisor trace outcome independently of the worker export through regeneration", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-finalized-trace-")); + temporaryDirectories.push(root); + const execution = runnerMatrix[0]!; + const attempt: RunnerE2EResult = { ...result(execution, "failed"), failureClass: "permanent_infrastructure", + tracing: { projectId: "11111111-1111-4111-8111-111111111111", traceId: "a".repeat(32), spanId: "b".repeat(16), + status: "accepted", verification: "unverified", nativeConfiguration: "configured", + finalization: { spanId: "c".repeat(16), status: "accepted", outcome: "failed", verification: "unverified" } }, + }; + const campaign = buildRunnerCampaign({ campaignId: "finalized-trace", generatedAt: attempt.finishedAt, + expected: [execution.id], results: [attempt], attempts: [attempt] }); + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")); + expect(rebuilt.results[0].tracing).toEqual(attempt.tracing); + expect(rebuilt.accounting.attempts[0].tracing).toEqual(attempt.tracing); + expect(rebuilt.failed).toBe(1); + expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Supervisor outcome: failed; finalization export: accepted"); + }); + + it("preserves all-attempt costs when regenerating a retained report", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-attempt-costs-")); + temporaryDirectories.push(root); + const execution = runnerMatrix[0]!; + const first = result(execution, "failed"); + const last = { ...result(execution, "passed"), attempt: 2, usage: { inputTokens: 200, outputTokens: 50, costUsd: 0.02 } }; + const campaign = buildRunnerCampaign({ campaignId: "retries", generatedAt: last.finishedAt, + expected: [execution.id], results: [last], attempts: [first, last] }); + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await regenerateRunnerDashboard({ bundle: root }); + const regenerated = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")); + expect(regenerated.accounting).toEqual(campaign.accounting); + expect(regenerated.billing.reportedLlmCostUsd).toBe(0.03); + expect(regenerated.results[0].billing.reportedCostUsd).toBe(0.02); + }); it("does not let empty legacy provenance hide a later measured source", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-legacy-source-")); temporaryDirectories.push(root); diff --git a/tests/runner-e2e/history.ts b/tests/runner-e2e/history.ts index 90476f4ee8e..822bd81b7e8 100644 --- a/tests/runner-e2e/history.ts +++ b/tests/runner-e2e/history.ts @@ -1,7 +1,11 @@ import { isIncompleteFirstTaskResult } from "./case-outcome.js"; +import { buildCampaignAccounting } from "./attempt-accounting.js"; +import { validateEvalOperationsReport } from "./operations-report.js"; +import { validateCampaignFeedbackReview } from "./feedback-dispositions.js"; +import { validateBraintrustExperimentReceipt } from "./braintrust-experiment.js"; import { classifyLatency } from "./latency-timing.js"; import { summarizeStartupReuse } from "../../packages/adapter-utils/src/startup-reuse.js"; -import { validateRetainedRunnerResult } from "./result-validation.js"; +import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js"; import { discoverReportCatalog, parseReportExecutionId, @@ -9,12 +13,13 @@ import { } from "./report-catalog.js"; import { runnerMatrix, runnerSuites } from "./catalog.js"; import { - aggregateCampaignBilling, + aggregateBillingSummaries, summarizeExecutionBilling, } from "./billing.js"; import { resolveRunnerE2ESource } from "./source.js"; import type { RunnerE2ECampaign, + RunnerE2ECampaignAccounting, RunnerE2EHistoryCampaign, RunnerE2EHistoryIndex, RunnerE2EResult, @@ -71,6 +76,12 @@ export function buildRunnerCampaign(input: { generatedAt: string; expected: readonly string[]; results: readonly RunnerE2EResult[]; + attempts?: readonly RunnerE2EResult[]; + accounting?: RunnerE2ECampaignAccounting; + operations?: RunnerE2ECampaign["operations"]; + feedbackReview?: RunnerE2ECampaign["feedbackReview"]; + braintrustExperiment?: RunnerE2ECampaign["braintrustExperiment"]; + excludedArtifacts?: number; eventName?: string | null; catalog?: readonly ReportExecution[]; }): RunnerE2ECampaign { @@ -79,6 +90,28 @@ export function buildRunnerCampaign(input: { ...upgradeRunnerResult(result), billing: result.billing ?? summarizeExecutionBilling(result), })); + const accounting = input.accounting ?? buildCampaignAccounting({ + results, + attempts: input.attempts?.map(upgradeRunnerResult), + excludedArtifacts: input.excludedArtifacts, + }); + validateCampaignAccounting(accounting); + if (input.operations) validateEvalOperationsReport(input.operations); + const billingFor = (selectedResults: readonly RunnerE2EResult[]) => { + const ids = new Set(selectedResults.map(result => result.executionId)); + const receipts = accounting.attempts.filter(attempt => ids.has(attempt.executionId)); + const billing = aggregateBillingSummaries([ + ...receipts, + ...selectedResults.filter(result => !receipts.some(attempt => attempt.executionId === result.executionId)) + .map(result => ({ executionId: result.executionId, billing: summarizeExecutionBilling(result) })), + ]); + billing.testsWithCompleteBilling = selectedResults.filter(result => { + const retained = receipts.filter(attempt => attempt.executionId === result.executionId); + return retained.length > 0 && retained.every(attempt => attempt.billing.complete) && + new Set(retained.map(attempt => attempt.attempt)).size >= Math.max(result.attempt, ...retained.map(attempt => attempt.attempt)); + }).length; + return billing; + }; const knownCatalog = input.catalog ?? runnerMatrix; const catalog = discoverReportCatalog({ catalog: knownCatalog, @@ -150,7 +183,7 @@ export function buildRunnerCampaign(input: { (total, result) => total + result.durationMs, 0, ), - billing: aggregateCampaignBilling(suiteResults), + billing: billingFor(suiteResults), }; }) .filter((suite): suite is NonNullable => Boolean(suite)); @@ -175,7 +208,7 @@ export function buildRunnerCampaign(input: { ), ).values(), ]; - return { + const campaign: RunnerE2ECampaign = { schema: "paperclip.runner-e2e.campaign/v2", campaignId: input.campaignId, generatedAt: input.generatedAt, @@ -195,10 +228,17 @@ export function buildRunnerCampaign(input: { ), cleanupPassed: results.every((result) => result.cleanup === "passed"), rankingSnapshots, - billing: aggregateCampaignBilling(results), + billing: billingFor(results), + accounting, + ...(input.operations ? { operations: input.operations } : {}), + ...(input.feedbackReview ? { feedbackReview: input.feedbackReview } : {}), + ...(input.braintrustExperiment ? { braintrustExperiment: input.braintrustExperiment } : {}), suites, results, }; + if (campaign.feedbackReview) validateCampaignFeedbackReview(campaign, campaign.feedbackReview); + if (campaign.braintrustExperiment) validateBraintrustExperimentReceipt(campaign, campaign.braintrustExperiment); + return campaign; } export function campaignHistoryRecord( @@ -253,6 +293,7 @@ export function emptyRunnerHistory(): RunnerE2EHistoryIndex { export function mergeRunnerHistory( current: RunnerE2EHistoryIndex | null | undefined, campaign: RunnerE2EHistoryCampaign, + retention?: { now: number; maxAgeMs: number; maxCampaigns: number }, ): RunnerE2EHistoryIndex { const base = current?.schema === "paperclip.runner-e2e.history/v1" @@ -263,10 +304,14 @@ export function mergeRunnerHistory( ...base.campaigns.filter( (candidate) => candidate.campaignId !== campaign.campaignId, ), - ].sort( + ].filter(candidate => !retention || ( + Number.isFinite(Date.parse(candidate.generatedAt)) && + Date.parse(candidate.generatedAt) > retention.now - retention.maxAgeMs && + Date.parse(candidate.generatedAt) <= retention.now + 60_000 + )).sort( (left, right) => Date.parse(right.generatedAt) - Date.parse(left.generatedAt), - ); + ).slice(0, retention?.maxCampaigns); const latestBySuite: Record = {}; const latestGreenBySuite: Record = {}; for (const candidate of campaigns) { @@ -279,7 +324,7 @@ export function mergeRunnerHistory( } return { schema: "paperclip.runner-e2e.history/v1", - updatedAt: new Date().toISOString(), + updatedAt: new Date(retention?.now ?? Date.now()).toISOString(), latestCampaignId: campaigns[0]?.campaignId ?? null, latestGreenCampaignId: campaigns.find( diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.ts new file mode 100644 index 00000000000..fdb327f4542 --- /dev/null +++ b/tests/runner-e2e/ingestion-usage.test.ts @@ -0,0 +1,108 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { openEvalCostLedger } from "./cost-ledger.js"; +import { startEvalOtlpRelay } from "./eval-otlp-relay.js"; +import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js"; +import { openExperimentIngestion } from "./experiment-ingestion.js"; +import { collectScheduledIngestion } from "./ingestion-usage.js"; + +const projectId = "11111111-1111-4111-8111-111111111111"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture() { + const root = await mkdtemp(path.join(tmpdir(), "scheduled-ingestion-")); roots.push(root); + const input = { ledgerPath: path.join(root, "ledger.jsonl"), campaignId: "campaign", projectId, executionIds: ["fixture.local"] }; + const ledger = await openEvalCostLedger(input.ledgerPath, { campaignId: input.campaignId, maxUsd: "1", + deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 1, maxConcurrent: 1, maxAttemptsPerExecution: 1 }); + try { await ledger.admitAttempt(input.executionIds[0]!, "attempt"); } finally { await ledger.close(); } + const name = createHash("sha256").update("attempt").digest("hex"), files: string[] = []; + for (const kind of ["native", "finalization"] as const) { + const receiptPath = path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`); + files.push(receiptPath); + const relay = await startEvalOtlpRelay({ ...input, attemptId: "attempt", apiKey: "private-fixture", receiptPath, + deadlineAt: Date.now() + 60_000, ...(kind === "finalization" ? { limits: EVAL_FINALIZATION_LIMITS } : {}), + transport: async () => Response.json({}) }); + try { + expect((await fetch(relay.baseUrl + "/v1/traces", { method: "POST", body: "{}", + headers: { authorization: `Bearer ${relay.token}`, "content-type": "application/json" } })).status).toBe(200); + } finally { await relay.close(); } + } + const experiment = await openExperimentIngestion({ ...input, receiptPath: `${input.ledgerPath}.experiment-ingestion.jsonl`, + deadlineAt: Date.now() + 60_000, transport: async () => Response.json({}) }); + const body = JSON.stringify({ project_id: projectId }); + try { await experiment.transport("https://api.braintrust.dev/v1/experiment", { method: "POST", body }); } + finally { await experiment.close(); } + return { input, files, expectedBytes: 4 + Buffer.byteLength(body) }; +} + +it("reconciles real durable receipts for all three transport paths without inventing vendor billing", async () => { + const f = await fixture(), usage = await collectScheduledIngestion(f.input); + expect(usage).toMatchObject({ status: "complete", observedAdmittedPayloadBytes: f.expectedBytes, + totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null, billingStatus: "unreconciled", missingExecutions: [] }); + expect(usage.receipts.map(row => row.kind)).toEqual(["native", "finalization", "experiment"]); + expect(usage.receipts[0]).toMatchObject({ admittedExpandedSourceBytes: 2, admittedRecords: 0 }); + expect(usage.receipts.every(row => "receiptSha256" in row && /^[a-f0-9]{64}$/.test(row.receiptSha256!))).toBe(true); + expect(usage.ledgerHash).toMatch(/^[a-f0-9]{64}$/); + expect(JSON.stringify(usage)).not.toContain("private-fixture"); +}); + +it.each(["missing", "unclosed", "unresolved", "foreign-project", "bad-total", "bad-expanded-total", "missing-record-count", "duplicate-outcome"])( + "retains other measured traffic and an unknown total for a %s receipt", async reason => { + const f = await fixture(), file = f.files[0]!; + if (reason === "missing") await rm(file); + else { + const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + if (reason === "unclosed") rows.pop(); + if (reason === "unresolved") rows[2].outcome = "uncertain"; + if (reason === "foreign-project") rows[0].projectId = "22222222-2222-4222-8222-222222222222"; + if (reason === "bad-total") rows.at(-1).admittedBytes = 0; + if (reason === "bad-expanded-total") rows.at(-1).admittedExpandedBytes = 0; + if (reason === "missing-record-count") delete rows[1].recordCount; + if (reason === "duplicate-outcome") rows.splice(3, 0, rows[2]); + await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + } + const usage = await collectScheduledIngestion(f.input); + expect(usage).toMatchObject({ status: "incomplete", totalAdmittedPayloadBytes: null, billedCostUsd: null }); + expect(usage.observedAdmittedPayloadBytes).toBeGreaterThan(0); + expect(usage.receipts[0]!.status).not.toBe("complete"); + expect(usage.receipts.slice(1).every(row => row.status === "complete")).toBe(true); + }); + +it("binds the expected receipt set to admitted attempts and retains unstarted selections", async () => { + const f = await fixture(); + expect(await collectScheduledIngestion({ ...f.input, executionIds: [...f.input.executionIds, "missing.local"] })) + .toMatchObject({ status: "incomplete", missingExecutions: ["missing.local"], totalAdmittedPayloadBytes: null }); + await expect(collectScheduledIngestion({ ...f.input, executionIds: [] })).rejects.toThrow("selection"); + await expect(collectScheduledIngestion({ ...f.input, executionIds: ["foreign.local"] })).rejects.toThrow("Foreign ingestion attempt"); + await expect(collectScheduledIngestion({ ...f.input, campaignId: "foreign" })).rejects.toThrow("campaign mismatch"); +}); + +it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => { + const f = await fixture(), file = f.files[0]!; + const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + rows[0].version = 1; delete rows[0].sizing; + delete rows[0].limits.maxExpandedBytes; delete rows[0].limits.maxRecords; + for (const row of rows) { + delete row.expandedBytes; delete row.recordCount; + delete row.admittedExpandedBytes; delete row.admittedRecords; + } + await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + const usage = await collectScheduledIngestion(f.input); + expect(usage).toMatchObject({ status: "complete", billedCostUsd: null }); + expect(usage.receipts[0]).toMatchObject({ admittedPayloadBytes: 2, admittedExpandedSourceBytes: null, admittedRecords: null }); +}); + +it.each(["outcome", "closed"])("retains the verified prefix and digest when the final %s line is truncated", async kind => { + const f = await fixture(), file = f.files[0]!; + const lines = (await readFile(file, "utf8")).trim().split("\n"); + const prefix = lines.slice(0, kind === "outcome" ? 2 : 3); + const truncated = prefix.join("\n") + '\n{"type":"' + kind; + await writeFile(file, truncated); + const usage = await collectScheduledIngestion(f.input); + expect(usage).toMatchObject({ status: "incomplete", observedAdmittedPayloadBytes: f.expectedBytes, totalAdmittedPayloadBytes: null }); + expect(usage.receipts[0]).toMatchObject({ status: "incomplete", admittedRequests: 1, admittedPayloadBytes: 2, + unresolvedRequests: kind === "outcome" ? 1 : 0, receiptSha256: createHash("sha256").update(truncated).digest("hex") }); +}); diff --git a/tests/runner-e2e/ingestion-usage.ts b/tests/runner-e2e/ingestion-usage.ts new file mode 100644 index 00000000000..f5c925844cb --- /dev/null +++ b/tests/runner-e2e/ingestion-usage.ts @@ -0,0 +1,98 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js"; +import { EVAL_OTLP_LIMITS } from "./eval-otlp-relay.js"; +import { EVAL_FINALIZATION_LIMITS } from "./eval-finalization-relay.js"; +import { EVAL_EXPERIMENT_LIMITS } from "./experiment-ingestion.js"; + +type Kind = "native" | "finalization" | "experiment"; +interface ReceiptScope { kind: Kind; file: string; campaignId: string; projectId: string; attemptId?: string } +async function readReceipt(scope: ReceiptScope) { + const bytes = await readFile(scope.file); + if (bytes.length > 4 * 1024 * 1024) throw new Error("Oversized ingestion receipt"); + const rows: Array> = []; + let malformed = false; + for (const line of bytes.toString("utf8").trim().split("\n")) { + try { rows.push(JSON.parse(line)); } + catch { malformed = true; break; } + } + const policy = rows[0], closed = rows.at(-1); + const maximum = scope.kind === "native" ? EVAL_OTLP_LIMITS : scope.kind === "finalization" ? EVAL_FINALIZATION_LIMITS : EVAL_EXPERIMENT_LIMITS; + const expanded = scope.kind !== "experiment" && policy?.version === 2; + if (policy?.type !== "policy" || !(policy.version === 1 || expanded && policy.sizing === "otlp-v1.9-expanded-source-json") + || policy.campaignId !== scope.campaignId || policy.projectId !== scope.projectId + || (scope.kind === "experiment" ? policy?.kind !== "experiment" : policy?.attemptId !== scope.attemptId)) { + throw new Error("Invalid ingestion receipt identity"); + } + if (!policy.limits || Object.entries(maximum).filter(([key]) => expanded || !["maxExpandedBytes", "maxRecords"].includes(key)) + .some(([key, max]) => !Number.isSafeInteger(policy.limits[key]) + || policy.limits[key] < 1 || policy.limits[key] > max)) throw new Error("Invalid ingestion receipt limits"); + const admissions = new Map(); + const outcomes = new Map(); + let payloadBytes = 0, scores = 0, expandedBytes = 0, recordCount = 0; + const count = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0; + for (let index = 1; index < rows.length; index++) { + const row = rows[index]; + if (row.type === "admitted") { + const scoreCount = scope.kind === "experiment" ? row.scores : 0; + if (typeof row.id !== "string" || admissions.has(row.id) || !count(row.bodyBytes) || !count(scoreCount) + || scope.kind !== "experiment" && row.bodyBytes > policy.limits.maxRequestBytes + || expanded && (!count(row.expandedBytes) || !count(row.recordCount)) + || !/^[0-9a-f]{64}$/.test(row.payloadSha256)) throw new Error("Invalid ingestion admission"); + admissions.set(row.id, { bodyBytes: row.bodyBytes, scores: scoreCount }); + payloadBytes += row.bodyBytes; scores += scoreCount; + if (expanded) { expandedBytes += row.expandedBytes; recordCount += row.recordCount; } + } else if (row.type === "outcome") { + const allowed = scope.kind === "experiment" ? ["response", "unconfirmed"] : ["accepted", "partial", "rejected", "uncertain", "not_dispatched"]; + if (!admissions.has(row.id) || outcomes.has(row.id) || !allowed.includes(row.outcome) + || ["response", "accepted", "partial", "rejected"].includes(row.outcome) + && (!Number.isInteger(row.httpStatus) || row.httpStatus < 100 || row.httpStatus > 599)) throw new Error("Invalid ingestion outcome"); + outcomes.set(row.id, row.outcome); + } else if (row.type === "closed") { + if (index !== rows.length - 1 || row.admittedRequests !== admissions.size || row.admittedBytes !== payloadBytes + || expanded && (row.admittedExpandedBytes !== expandedBytes || row.admittedRecords !== recordCount) + || scope.kind === "experiment" && row.admittedScores !== scores || row.poisoned) throw new Error("Invalid ingestion closure"); + } else if (row.type !== "quota_exhausted" || scope.kind === "experiment") throw new Error("Unexpected ingestion record"); + } + if (admissions.size > policy.limits.maxRequests || payloadBytes > policy.limits.maxTotalBytes + || expanded && (expandedBytes > policy.limits.maxExpandedBytes || recordCount > policy.limits.maxRecords) + || scope.kind === "experiment" && scores > policy.limits.maxScores) throw new Error("Ingestion allowance exceeded"); + const unresolved = [...admissions.keys()].filter(id => !outcomes.has(id) || ["uncertain", "unconfirmed"].includes(outcomes.get(id)!)).length; + return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}), + receiptSha256: createHash("sha256").update(bytes).digest("hex"), + status: !malformed && closed?.type === "closed" && unresolved === 0 ? "complete" : "incomplete", + admittedRequests: admissions.size, admittedPayloadBytes: payloadBytes, admittedScores: scope.kind === "experiment" ? scores : null, + admittedExpandedSourceBytes: expanded ? expandedBytes : null, admittedRecords: expanded ? recordCount : null, + unresolvedRequests: unresolved }; +} + +export async function collectScheduledIngestion(input: { + ledgerPath: string; campaignId: string; projectId: string; executionIds: string[]; +}) { + if (!input.executionIds.length || new Set(input.executionIds).size !== input.executionIds.length + || !input.executionIds.every(validEvalLedgerId)) throw new Error("Invalid ingestion execution selection"); + const ledger = await readEvalCostLedger(input.ledgerPath); + if (ledger.state.limits.campaignId !== input.campaignId) throw new Error("Ingestion ledger campaign mismatch"); + const attempts = ledger.state.snapshot().admittedAttempts; + if (attempts.some(row => !input.executionIds.includes(row.executionId))) throw new Error("Foreign ingestion attempt"); + const missingExecutions = input.executionIds.filter(id => !attempts.some(row => row.executionId === id)); + const scopes: ReceiptScope[] = attempts.flatMap(row => { + const name = createHash("sha256").update(row.attemptId).digest("hex"); + return (["native", "finalization"] as const).map(kind => ({ kind, attemptId: row.attemptId, + campaignId: input.campaignId, projectId: input.projectId, + file: path.join(`${input.ledgerPath}.otlp`, `${name}${kind === "finalization" ? "-finalization" : ""}.jsonl`) })); + }); + scopes.push({ kind: "experiment", file: `${input.ledgerPath}.experiment-ingestion.jsonl`, campaignId: input.campaignId, projectId: input.projectId }); + const receipts = await Promise.all(scopes.map(async scope => { + try { return await readReceipt(scope); } + catch { return { kind: scope.kind, ...(scope.attemptId ? { attemptId: scope.attemptId } : {}), status: "unavailable" }; } + })); + const complete = missingExecutions.length === 0 && receipts.every(row => row.status === "complete"); + return { version: 1, campaignId: input.campaignId, projectId: input.projectId, ledgerHash: ledger.previous, + status: complete ? "complete" : "incomplete", missingExecutions, receipts, + observedAdmittedPayloadBytes: receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0), + totalAdmittedPayloadBytes: complete ? receipts.reduce((sum, row) => sum + ("admittedPayloadBytes" in row ? row.admittedPayloadBytes : 0), 0) : null, + billedCostUsd: null, billingStatus: "unreconciled", + scope: "Bounded attempt relay, controller finalization and retained experiment requests; not vendor processed bytes or provider model charges" }; +} diff --git a/tests/runner-e2e/installed-cli.test.ts b/tests/runner-e2e/installed-cli.test.ts index b1ab4577a87..83770cd98ac 100644 --- a/tests/runner-e2e/installed-cli.test.ts +++ b/tests/runner-e2e/installed-cli.test.ts @@ -108,10 +108,9 @@ it("uses the direct public Playwright JS bin without bin-shim Node injection", a const f = await fixture(); const cli = join(f.root,"node_modules/@playwright/test/cli.js"); await mkdir(join(f.root,"node_modules/@playwright/test"),{recursive:true}); await writeFile(cli,"console.log(JSON.stringify({nodePath:process.env.NODE_PATH??null,nodeOptions:process.env.NODE_OPTIONS??null,args:process.argv.slice(2)}))"); - const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"],true); + const invocation = runnerE2EPlaywrightInvocation(f.root,["--version"]); expect(invocation.command).toBe(process.execPath); expect(JSON.parse(execFileSync(invocation.command,invocation.args,{env:{},timeout:5000,encoding:"utf8"}))).toEqual({nodePath:null,nodeOptions:null,args:["--version"]}); - expect(runnerE2EPlaywrightInvocation(f.root,["--version"],false)).toEqual({command:"pnpm",args:["exec","playwright","--version"]}); for (const key of ["NODE_PATH","NODE_OPTIONS"]) expect(()=>assertInstalledCliSelection({...f.env,[key]:""},[cells[0]!])).toThrow("ambient Node injection"); }); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 092ce425f9e..30c489ad928 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -34,6 +34,9 @@ associate(["local-session-integrity", "extended-harnesses", "grok-qualification" associate(["question-resume"], { "question-answer-resume": ["questions-and-approvals/questions", "questions-and-approvals/continuation"], }); +associate(["feedback-persistence"], { + "complaint-and-suggestion": ["activity/agent-commentary"], +}); associate(["continuation"], { "answer-updates-scope": ["questions-and-approvals/questions", "questions-and-approvals/continuation"], "clarification-not-approval": ["questions-and-approvals/questions", "questions-and-approvals/confirmations"], diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 1d0971ab97b..934daa1dc86 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,9 +1,15 @@ -import { runnerE2EPlaywrightInvocation } from "./web-server-command.js"; +import { recoverWorkerTiming } from "./worker-timing.js"; +import { openEvalOperations, type EvalOperations } from "./operations.js"; +import { readRecoveredFeedback } from "./feedback-recovery.js"; +import { readInterruptedAttemptTrace } from "./trace-recovery.js"; +import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js"; +import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js"; +import { runnerE2EPlaywrightInvocation, runnerE2EPlaywrightShutdownSignal } from "./web-server-command.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import { assertInstalledCliSelection, assertInstalledStartupOnly, verifyInstalledCli } from "./installed-cli.js"; import { createProcessTreeOwner, stopOwnedProcessTree } from "./process-tree-owner.js"; import { createRunnerE2ETemporaryRoot } from "./server-config.js"; -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { prepareCodexCiSandbox, requiresCodexCiSandbox } from "./codex-ci-sandbox.js"; import { spawn } from "node:child_process"; import { createWriteStream } from "node:fs"; @@ -30,7 +36,9 @@ import { classifyFailure } from "./failure-classifier.js"; import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js"; import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js"; import { buildRunnerCampaign } from "./history.js"; +import { readEvalOperationsReport } from "./operations-report.js"; import { + applyEvalGatewayEnvironment, buildRunnerE2EProcessEnvironment, resolvePaperclipRemoteRunnerBinaryForHarness, resolvePaperclipRunnerBinaryForHarness, @@ -305,7 +313,7 @@ async function runProcess( const cleanupFinished = new Promise(resolve => { cleanupSettled = () => resolve(1); }); const stopChildTree = (_diagnostic?: ProcessTreeDiagnostic) => { if (boundedCleanup) return; - boundedCleanup = stopOwnedProcessTree(child, processOwner) + boundedCleanup = stopOwnedProcessTree(child, processOwner, 45_000, 5_000, runnerE2EPlaywrightShutdownSignal) .then(() => null, error => error instanceof Error ? error.message : String(error)) .finally(cleanupSettled); activeProcessCleanup.set(child.pid!, boundedCleanup); @@ -467,8 +475,12 @@ async function runAttempt(input: { attempt: number; campaignId: string; options: ReturnType; + operations?: EvalOperations; }): Promise { const { executions, attempt, campaignId, options } = input; + const accountingCampaignId = input.operations?.manifest.limits.campaignId ?? process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? campaignId; + const accountingAttemptId = randomBytes(16).toString("hex"); + const traceConfig = evalBraintrustConfig({ ...process.env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined }); const execution = executions[0]; if (!execution) throw new Error("A runner E2E cell must contain a task case"); if ( @@ -494,6 +506,7 @@ async function runAttempt(input: { let attemptSecrets: string[] = []; let processCleanupFailed = false; let startupReceiptPath: string | undefined; + let gateway: Awaited> | undefined; const rawCleanupResults: Array<{ cleanup: string; synthetic: boolean; status: string }> = []; try { const paperclipHome = path.join(temporaryRoot, "paperclip-home"); @@ -523,8 +536,11 @@ async function runAttempt(input: { const decisionSigningSecret = secret(48); const toolActionSigningSecret = secret(48); const betterAuthSecret = secret(48); + gateway = await input.operations?.start(execution, accountingAttemptId, attempt); const credentialSources = { - ...Object.fromEntries(CREDENTIAL_NAMES.map((name) => [name, process.env[name]])), + PAPERCLIP_E2E_GATEWAY_TOKEN: gateway?.token, + PAPERCLIP_E2E_OTLP_RELAY_TOKEN: gateway?.observability?.token, + ...Object.fromEntries([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => [name, process.env[name]])), PAPERCLIP_AGENT_JWT_SECRET: agentJwtSecret, PAPERCLIP_DECISION_SIGNING_SECRET: decisionSigningSecret, PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret, @@ -545,6 +561,8 @@ async function runAttempt(input: { executions.map((candidate) => candidate.id), ), PAPERCLIP_RUNNER_E2E_ATTEMPT: String(attempt), + PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: accountingCampaignId, + PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: accountingAttemptId, PAPERCLIP_RUNNER_E2E_INSTALLED_STARTUP_ONLY: options.installedStartupOnly ? "1" : undefined, PAPERCLIP_RUNNER_E2E_PUBLIC_MCP: executions.some(candidate => candidate.task.flow === "public_mcp") ? "1" : "0", PAPERCLIP_RUNNER_E2E_PORT: String(port), @@ -575,6 +593,9 @@ async function runAttempt(input: { PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: toolActionSigningSecret, BETTER_AUTH_SECRET: betterAuthSecret, }; + if (gateway) { + applyEvalGatewayEnvironment(childEnv, execution.profile.credential, gateway); + } // The database URLs are stripped here and again at the Playwright web-server // boundary so a developer's shell can never redirect this paid test. delete childEnv.DATABASE_URL; @@ -600,7 +621,7 @@ async function runAttempt(input: { ) + 5 * 60_000; const processResult = await runProcess( - runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs, Boolean(installedCli)), + runnerE2EPlaywrightInvocation(repositoryRoot, playwrightArgs), childEnv, watchdog, path.join(privateDir, "playwright.log"), @@ -653,6 +674,16 @@ async function runAttempt(input: { processFailureClass, ); const result = await readResult(resultPath, fallback); + if (!result.workerTiming) result.workerTiming = await recoverWorkerTiming(path.join(path.dirname(resultPath), "snapshots", "worker-timing.json"), + { executionId: candidate.id, attempt, startedAt: result.startedAt, finishedAt: result.finishedAt }); + if (result === fallback) { + result.feedback = await readRecoveredFeedback(path.dirname(resultPath), { + campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id, + }); + if (traceConfig) result.tracing = await readInterruptedAttemptTrace(path.dirname(resultPath), { + projectId: traceConfig.projectId, executionId: candidate.id, + }); + } // Keep cleanup authority before evidence copying/publication can fail. rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status }); return enforceResultProcessIntegrity(result, processResult); @@ -712,7 +743,10 @@ async function runAttempt(input: { } } for (const [index, candidate] of executions.entries()) { - let result = results[index]; + let result = { + ...results[index], + accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId }, + }; if ( isolationError && result.failureClass !== "cleanup_failure" && @@ -836,6 +870,8 @@ async function runAttempt(input: { } return [...publishedResults]; } finally { + let gatewayCleanupError: unknown; + try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; } if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline); let cleanupError: unknown; const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started")) @@ -877,6 +913,7 @@ async function runAttempt(input: { `Refusing to remove unexpected temporary path ${temporaryRoot}`, ); } + if (gatewayCleanupError) cleanupError = new AggregateError([cleanupError, gatewayCleanupError].filter(Boolean), "Eval gateway cleanup failed"); if (startupReceiptPath) { const receipt = JSON.parse(await readFile(startupReceiptPath, "utf8")); receipt.status = cleanupError ? "installed_launch_chain_cleanup_failed" : "installed_launch_chain_passed"; @@ -909,6 +946,26 @@ async function runAttempt(input: { console.error(message); if (options.installedStartupOnly) throw new Error(message); } + for (const result of publishedResults) { + const resultPath = publishedResultPaths.get(result.executionId); + const candidate = executions.find(candidate => candidate.id === result.executionId); + if (!traceConfig || !result.tracing || !resultPath || !candidate) continue; + const finalizationInput = { + receipt: result.tracing, + identity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId, executionId: candidate.id, + provider: candidate.profile.provider, model: candidate.profile.model, environment: candidate.environment.id }, + provenance: { suiteId: candidate.suite.id, suiteDefinitionHash: candidate.suiteDefinitionHash, profileId: candidate.profile.id, + caseId: candidate.task.id, sourceRevision: resolveRunnerE2ESource().sha }, + result, + }; + result.tracing.finalization = input.operations + ? await finalizeEvalAttemptThroughRelay(traceConfig, finalizationInput, + path.join(`${input.operations.manifest.ledgerPath}.otlp`, `${createHash("sha256").update(accountingAttemptId).digest("hex")}-finalization.jsonl`)) + : await finalizeEvalAttemptTrace(traceConfig, finalizationInput); + const safeResult = `${JSON.stringify(sanitizeJson(result, attemptSecrets), null, 2)}\n`; + assertSecretFree(safeResult, attemptSecrets, resultPath); + await writeFile(resultPath, safeResult, "utf8"); + } } } @@ -932,6 +989,8 @@ async function runExecutionWithRetry(input: { execution: MatrixExecution; campaignId: string; options: ReturnType; + onAttempt: (result: RunnerE2EResult) => void; + operations?: EvalOperations; }): Promise { const { execution, campaignId, options } = input; return executeWithAutomaticRetry({ @@ -942,8 +1001,9 @@ async function runExecutionWithRetry(input: { `Retrying ${execution.id} in a fresh isolated harness after ${result.failureClass!.replaceAll("_", " ")}`, ), runAttempt: async attempt => { - const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options }); + const [result] = await runAttempt({ executions: [execution], attempt, campaignId, options, operations: input.operations }); if (!result) throw new Error(`No result produced for ${execution.id} attempt ${attempt}`); + input.onAttempt(result); return result; }, }); @@ -968,7 +1028,9 @@ async function runWithConcurrency( } }, ); - await Promise.all(workers); + const settled = await Promise.allSettled(workers); + const failures = settled.filter((result): result is PromiseRejectedResult => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map(result => result.reason), "Eval campaign workers failed"); return results; } @@ -1011,6 +1073,7 @@ async function main() { return; } + if (process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST && executions.some(execution => execution.task.flow === "provider_connection")) throw new Error("Provider connection campaign has no budgeted routing qualification"); if (executions.some(execution => execution.task.flow === "provider_connection")) { const { runConnectionCampaign } = await import("./connection-launch.js"); await runConnectionCampaign({ executions, catalog: runnerMatrix, configFile: options.connectionConfig, repositoryRoot }); @@ -1041,7 +1104,7 @@ async function main() { await loadLocalEnvironment(process.env); assertRemoteNativeEvidencePrerequisites(executions, process.env); - const missingCredentials = [ + const missingCredentials = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST ? [] : [ ...new Set( executions.flatMap((execution) => execution.requiredCredentials), ), @@ -1084,21 +1147,35 @@ async function main() { console.log( `Running ${executions.length} isolated execution(s) with max parallelism ${requestedParallelism}`, ); - const finalResults = await runWithConcurrency( - executions, - requestedParallelism, - (execution) => runExecutionWithRetry({ execution, campaignId, options }), - ); + const attempts: RunnerE2EResult[] = []; + const operations = process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST + ? await openEvalOperations(process.env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, executions, options.maxAutomaticRetries, process.env) : undefined; + let finalResults: RunnerE2EResult[] = []; + let campaignFailed = false; + try { + if (operations) await writeFile(path.join(summaryDir, "operations-preflight.json"), `${JSON.stringify(operations.manifest, null, 2)}\n`, { mode: 0o600 }); + finalResults = await runWithConcurrency(executions, operations ? Math.min(requestedParallelism, operations.manifest.limits.maxConcurrent) : requestedParallelism, + execution => runExecutionWithRetry({ execution, campaignId, options, operations, onAttempt: result => attempts.push(result) })); + } catch { + campaignFailed = true; + finalResults = [...new Map(attempts.map(result => [result.executionId, result])).values()]; + } finally { + try { await operations?.close(); } catch { campaignFailed = true; } + } const generatedAt = new Date().toISOString(); + const operationsReport = operations ? await readEvalOperationsReport(operations.manifest.ledgerPath) : undefined; + if (operationsReport?.status === "unavailable") campaignFailed = true; const campaign = buildRunnerCampaign({ campaignId, generatedAt, expected: executions.map((execution) => execution.id), results: finalResults, + attempts, + ...(operationsReport ? { operations: operationsReport } : {}), }); const campaignSecrets = normalizedSecrets( - CREDENTIAL_NAMES.map((name) => process.env[name]), + [...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES].map((name) => process.env[name]), ); const safeCampaign = sanitizeJson( campaign, @@ -1138,7 +1215,8 @@ async function main() { console.log( `Campaign ${campaignId}: ${safeCampaign.passed}/${safeCampaign.selected} passed`, ); - if (safeCampaign.failed > 0) process.exitCode = 1; + if (campaignFailed) console.error("Campaign finalization incomplete; inspect retained attempt artifacts and the operations ledger."); + if (campaignFailed || safeCampaign.failed > 0) process.exitCode = 1; } await main().catch((error) => { diff --git a/tests/runner-e2e/ledger-attempt-policy.test.ts b/tests/runner-e2e/ledger-attempt-policy.test.ts new file mode 100644 index 00000000000..138398236c4 --- /dev/null +++ b/tests/runner-e2e/ledger-attempt-policy.test.ts @@ -0,0 +1,120 @@ +import { afterEach, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; +import { adoptLegacyEvalAttemptPolicy, type LegacyAttemptAuditFile } from "./ledger-attempt-policy.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +const limits: EvalSpendLimits = { campaignId: "legacy-pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 3, maxConcurrent: 1 }; +const identity = { executionId: "claude.local.fixture", attemptId: "old-request-attempt", phase: "agent" as const }; +const hash = (value: string) => createHash("sha256").update(value).digest("hex"); + +async function fixture(overrides: Partial = {}) { + const root = await mkdtemp(path.join(os.tmpdir(), "legacy-eval-attempts-")); roots.push(root); + const file = path.join(root, "ledger.jsonl"); + const originalLimits = { ...limits, ...overrides }; + const ledger = await openEvalCostLedger(file, originalLimits); + await ledger.reserve("cancelled-request", identity, "0.6"); + await ledger.observe({ observationId: "partial", requestId: "cancelled-request", source: "provider", provider: "anthropic", + currency: "USD", route: "direct", classification: "reported", costUsd: "0.2", final: false }); + await ledger.finish("cancelled-request", "cancelled"); + await ledger.close(); + const evidencePath = path.join(root, "reviewed-attempts.md"); + const evidence = "Reviewed retained campaigns, including the earlier zero-request startup failure."; + await writeFile(evidencePath, evidence); + const loaded = await readEvalCostLedger(file); + const audit: LegacyAttemptAuditFile = { version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash(evidence), + previousJournalHash: loaded.previous, maxAttemptsPerExecution: 3, + priorAttempts: [identity, { executionId: identity.executionId, attemptId: "old-zero-request-attempt" }] + .map(({ executionId, attemptId }) => ({ executionId, attemptId })) }; + return { file, audit, originalLimits, before: loaded.state.snapshot(), prefix: await readFile(file, "utf8") }; +} + +it("appends an audited policy without rewriting charges, held reservations, or original limits", async () => { + const { file, audit, before, prefix, originalLimits } = await fixture(); + const migrated = await adoptLegacyEvalAttemptPolicy(audit); + expect((await readFile(file, "utf8")).startsWith(prefix)).toBe(true); + expect(migrated.limits).toEqual({ ...originalLimits, maxAttemptsPerExecution: 3 }); + expect({ ...migrated.accounting, admittedAttempts: [] }).toEqual(before); + expect(migrated.accounting.admittedAttempts).toEqual(audit.priorAttempts); + const resumed = await openEvalCostLedger(file, migrated.limits); + try { + await resumed.admitAttempt(identity.executionId, "new-attempt"); + await expect(resumed.admitAttempt(identity.executionId, "extra-attempt")).rejects.toThrow("attempt cap"); + await expect(resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.5")).rejects.toThrow("spending cap"); + await resumed.reserve("new-request", { ...identity, attemptId: "new-attempt" }, "0.4"); + expect(resumed.snapshot().committedUsd).toBe("1.000000000"); + } finally { await resumed.close(); } + const replayed = await readEvalCostLedger(file); + expect(replayed.state.snapshot().admittedAttempts).toHaveLength(3); + expect(replayed.state.snapshot().knownUsd).toBe("0.200000000"); + await expect(openEvalCostLedger(file, originalLimits)).rejects.toThrow("limits changed"); + await expect(openEvalCostLedger(file, { ...migrated.limits, maxUsd: "2" })).rejects.toThrow("limits changed"); +}); + +it("requires all recorded request identities and rejects duplicate or foreign attempts", async () => { + const { file, audit, prefix } = await fixture(); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [] })).rejects.toThrow("omits a recorded request"); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [audit.priorAttempts[0]!, audit.priorAttempts[0]!] })).rejects.toThrow("Invalid legacy attempt identity"); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, executionId: "other-cell" }] })).rejects.toThrow("omits a recorded request"); + expect(await readFile(file, "utf8")).toBe(prefix); +}); + +it("rejects changed evidence and changed journal checkpoints without appending", async () => { + const { file, audit, prefix, originalLimits } = await fixture(); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, evidenceSha256: "a".repeat(64) })).rejects.toThrow("evidence changed"); + expect(await readFile(file, "utf8")).toBe(prefix); + const writer = await openEvalCostLedger(file, originalLimits); + try { + await writer.observe({ observationId: "late", requestId: "cancelled-request", source: "invoice", provider: "anthropic", + currency: "USD", route: "direct", classification: "reconciled", costUsd: "0.1", final: true }); + await expect(writer.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256, + maxAttemptsPerExecution: 3, priorAttempts: audit.priorAttempts })).rejects.toThrow("checkpoint changed"); + } finally { await writer.close(); } + await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("checkpoint changed"); + expect((await readEvalCostLedger(file)).state.limits.maxAttemptsPerExecution).toBeUndefined(); +}); + +it("refuses concurrent writers and unfinished requests", async () => { + const { file, audit, originalLimits } = await fixture(); + const writer = await openEvalCostLedger(file, originalLimits); + try { + await expect(adoptLegacyEvalAttemptPolicy(audit)).rejects.toThrow("writer lock"); + await writer.reserve("inflight", identity, "0.1"); + } finally { await writer.close(); } + const changed = await readEvalCostLedger(file); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: changed.previous })).rejects.toThrow("stopped requests"); + expect((await readEvalCostLedger(file)).state.snapshot().inflight).toBe(1); +}); + +it("does not change established caps or admit extra attempts after an over-cap history", async () => { + const { file, audit } = await fixture(); + const migrated = await adoptLegacyEvalAttemptPolicy({ ...audit, maxAttemptsPerExecution: 1 }); + const writer = await openEvalCostLedger(file, migrated.limits); + try { await expect(writer.admitAttempt(identity.executionId, "extra")).rejects.toThrow("attempt cap"); } + finally { await writer.close(); } + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, previousJournalHash: migrated.journalHash, maxAttemptsPerExecution: 10 })).rejects.toThrow("already established"); +}); + +it("preserves an expired campaign deadline during a historical audit", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "expired-eval-attempts-")); roots.push(root); + const file = path.join(root, "ledger.jsonl"), evidencePath = path.join(root, "audit.md"); + const writer = await openEvalCostLedger(file, { ...limits, deadlineAt: "2000-01-01T00:00:00Z" }); + await writer.close(); await writeFile(evidencePath, "No launched attempts."); + const original = await readEvalCostLedger(file); + const migrated = await adoptLegacyEvalAttemptPolicy({ version: 1, ledgerPath: file, evidencePath, evidenceSha256: hash("No launched attempts."), + previousJournalHash: original.previous, maxAttemptsPerExecution: 1, priorAttempts: [] }); + const resumed = await openEvalCostLedger(file, migrated.limits); + try { await expect(resumed.admitAttempt("cell", "attempt")).rejects.toThrow("deadline"); } + finally { await resumed.close(); } +}); + +it("rejects extra policy fields instead of accepting limit changes or private notes", async () => { + const { file, audit, prefix } = await fixture(); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, maxUsd: "100" } as LegacyAttemptAuditFile)).rejects.toThrow("Invalid legacy attempt audit file"); + await expect(adoptLegacyEvalAttemptPolicy({ ...audit, priorAttempts: [{ ...audit.priorAttempts[0]!, note: "private" }] } as unknown as LegacyAttemptAuditFile)).rejects.toThrow("Unknown ledger field"); + expect(await readFile(file, "utf8")).toBe(prefix); +}); diff --git a/tests/runner-e2e/ledger-attempt-policy.ts b/tests/runner-e2e/ledger-attempt-policy.ts new file mode 100644 index 00000000000..3bcbc286760 --- /dev/null +++ b/tests/runner-e2e/ledger-attempt-policy.ts @@ -0,0 +1,40 @@ +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { openEvalCostLedger, readEvalCostLedger, type EvalLegacyAttemptAudit } from "./cost-ledger.js"; + +export interface LegacyAttemptAuditFile extends EvalLegacyAttemptAudit { + version: 1; + ledgerPath: string; + evidencePath: string; +} + +export async function adoptLegacyEvalAttemptPolicy(audit: LegacyAttemptAuditFile) { + const allowed = ["version", "ledgerPath", "evidencePath", "previousJournalHash", "evidenceSha256", "maxAttemptsPerExecution", "priorAttempts"]; + if (!audit || typeof audit !== "object" || Array.isArray(audit) || Object.keys(audit).some(key => !allowed.includes(key)) + || audit.version !== 1 || typeof audit.ledgerPath !== "string" || !path.isAbsolute(audit.ledgerPath) + || typeof audit.evidencePath !== "string" || !path.isAbsolute(audit.evidencePath)) throw new Error("Invalid legacy attempt audit file"); + const evidence = await readFile(audit.evidencePath); + if (createHash("sha256").update(evidence).digest("hex") !== audit.evidenceSha256) throw new Error("Legacy attempt audit evidence changed"); + const original = await readEvalCostLedger(audit.ledgerPath); + if (original.previous !== audit.previousJournalHash) throw new Error("Legacy attempt audit checkpoint changed"); + const ledger = await openEvalCostLedger(audit.ledgerPath, original.state.limits); + try { + await ledger.adoptLegacyAttemptPolicy({ previousJournalHash: audit.previousJournalHash, evidenceSha256: audit.evidenceSha256, + maxAttemptsPerExecution: audit.maxAttemptsPerExecution, priorAttempts: audit.priorAttempts }); + } finally { await ledger.close(); } + const updated = await readEvalCostLedger(audit.ledgerPath); + return { journalHash: updated.previous, records: updated.sequence, limits: updated.state.limits, accounting: updated.state.snapshot() }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + if (process.argv.length !== 3) throw new Error("Usage: ledger-attempt-policy.ts /absolute/private-audit.json"); + const result = await adoptLegacyEvalAttemptPolicy(JSON.parse(await readFile(process.argv[2]!, "utf8"))); + process.stdout.write(`Legacy attempt policy saved at journal record ${result.records}. Existing costs and limits are preserved.\n`); + } catch { + process.stderr.write("Legacy attempt audit failed. Inspect the private audit and original journal before retrying.\n"); + process.exitCode = 1; + } +} diff --git a/tests/runner-e2e/live-fixtures.ts b/tests/runner-e2e/live-fixtures.ts index 5cc7ff51724..cc861c6232c 100644 --- a/tests/runner-e2e/live-fixtures.ts +++ b/tests/runner-e2e/live-fixtures.ts @@ -1,3 +1,4 @@ +import type { AiConnectionBinding } from "../../packages/shared/src/ai-connections.js"; import { verifyInstalledDaytonaPlugin } from "./installed-daytona-plugin.js"; import { NATIVE_COMPLETION_BUDGET_CENTS } from "./native-completion-defaults.js"; import path from "node:path"; @@ -44,11 +45,7 @@ interface AgentRecord { } interface ManagedAccountFixture { connectionId: string; - binding: { - provider: "openai" | "anthropic" | "github" | "openrouter"; - method: "api_key"; - mode: "responsible_user"; - }; + binding: AiConnectionBinding; } interface ProjectRecord { @@ -143,6 +140,7 @@ export async function setupLiveFixtures(input: { workspacePath: string; credentials: Partial>; daytonaImage?: string; + onCompanyCreated?: (companyId: string) => Promise; }): Promise { const { api, execution } = input; const receiptSmoke = execution.suite.id === "case-handoffs" && execution.task.id === caseReceiptTask.id; @@ -166,15 +164,17 @@ export async function setupLiveFixtures(input: { registry.register({ id: "company", async setup() { - return api.post("/api/companies", { + const company = await api.post("/api/companies", { name: `Runner E2E ${execution.id} ${input.executionNonce}`, description: "Ephemeral paid full-stack runner acceptance fixture", budgetMonthlyCents: receiptSmoke ? CASE_RECEIPT_BUDGET_CENTS : ["native-completion", "native-instruction-consolidation", "native-connection-guidance"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id)) ? NATIVE_COMPLETION_BUDGET_CENTS : execution.suite.id === "task-titles" ? TASK_TITLE_BUDGET_CENTS - : ["stock-harness", "task-workspaces", "case-handoffs"].includes(execution.suite.id) ? 1_000 : 0, + : ["stock-harness", "task-workspaces", "case-handoffs", "feedback-persistence"].includes(execution.suite.id) ? 1_000 : 0, }); + await input.onCompanyCreated?.(company.id); + return company; }, async teardown() { // The isolated instance/database is removed by the launcher. Do not call @@ -275,7 +275,12 @@ export async function setupLiveFixtures(input: { const managedHiring = isManagedHiringCase(execution.suite.id, execution.task.id); const savedCopilot = execution.profile.qualificationCandidate === "copilot"; - if (managedHiring || savedCopilot) { + const gatewayUrl = process.env.PAPERCLIP_E2E_GATEWAY_BASE_URL; + const gatewayToken = process.env.PAPERCLIP_E2E_GATEWAY_TOKEN; + const budgetedGateway = Boolean(gatewayUrl || gatewayToken); + if (budgetedGateway && (!gatewayUrl || !gatewayToken || !/^http:\/\/127\.0\.0\.1:[0-9]+$/.test(gatewayUrl) + || execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id))) throw new Error("Unqualified fixture gateway route"); + if (managedHiring || savedCopilot || budgetedGateway) { registry.register({ id: "ai-connection", dependencies: ["company", ...(savedCopilot ? ["environment"] : [])], @@ -288,22 +293,26 @@ export async function setupLiveFixtures(input: { if (!provider) throw new Error(`Unsupported managed hiring credential ${key}`); const apiKey = input.credentials[key]; if (!apiKey) throw new Error(`Missing credential ${key}`); - const account = await api.postSensitive<{ connectionId: string }>( + const account = await api.postSensitive<{ connectionId: string; grantId: string }>( `/api/companies/${company.id}/ai-connections`, { provider, method: "api_key", name: `Runner E2E account ${input.executionNonce}`, - ownership: "personal", + ownership: budgetedGateway ? "shared" : "personal", + ...(budgetedGateway ? { routing: { kind: "gateway", protocol: provider === "anthropic" ? "messages" : "responses", + baseUrl: provider === "anthropic" ? gatewayUrl : `${gatewayUrl}/v1`, auth: "bearer", models: [{ id: execution.profile.model }] } } : {}), ...(savedCopilot ? { environmentId: value(resolved, "environment").id } : {}), - apiKey, + apiKey: budgetedGateway ? gatewayToken : apiKey, agentIds: [], - allAgents: false, + allAgents: budgetedGateway, }, ); return { connectionId: account.connectionId, - binding: { provider, method: "api_key", mode: "responsible_user" }, + binding: budgetedGateway + ? { provider, method: "api_key", mode: "shared", connectionId: account.connectionId, grantId: account.grantId } + : { provider, method: "api_key", mode: "responsible_user" }, }; }, }); @@ -316,7 +325,7 @@ export async function setupLiveFixtures(input: { "secrets", "environment", ...((grokSubscription || codexSubscription) ? ["subscription-login"] : []), - ...(managedHiring || savedCopilot ? ["ai-connection"] : []), + ...(managedHiring || savedCopilot || budgetedGateway ? ["ai-connection"] : []), ], async setup(resolved) { const company = value(resolved, "company"); @@ -356,7 +365,7 @@ export async function setupLiveFixtures(input: { setupProbe.defaultRunner = selected.defaultRunner; } } - if (["stock-harness", "native-connection-guidance"].includes(execution.suite.id) + if (["stock-harness", "native-connection-guidance", "feedback-persistence"].includes(execution.suite.id) || (execution.suite.id === "everyday-workflows" && ["hire-reuse", "delegate-feedback"].includes(execution.task.id))) { agent.budgetMonthlyCents = 1_000; } @@ -369,7 +378,7 @@ export async function setupLiveFixtures(input: { heartbeat: { ...((runtimeConfig.heartbeat ?? {}) as Record), enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 1, maxDailyCostCents: CASE_RECEIPT_BUDGET_CENTS } }; } - if (managedHiring || savedCopilot) { + if (managedHiring || savedCopilot || budgetedGateway) { const account = value(resolved, "ai-connection"); const config = agent.adapterConfig as Record; delete config.env; diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts new file mode 100644 index 00000000000..c837450da00 --- /dev/null +++ b/tests/runner-e2e/operating-overhead.test.ts @@ -0,0 +1,119 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; +import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js"; + +const now = Date.parse("2026-10-21T00:00:00Z"); +const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet", + verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z", + components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) }); +const currentAllowance = (time = Date.now()): EvalOperatingOverhead => ({ ...allowance(), + verifiedAt: new Date(time - 86_400_000).toISOString(), expiresAt: new Date(time + 30 * 86_400_000).toISOString() }); +const runner = "ubuntu-latest"; +const modelIdentity = { executionId: "claude.local.fixture", attemptId: "attempt-1", phase: "agent" as const }; + +it("keeps operating overhead inside the same cap through model completion and a journal restart", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-")); + const file = path.join(directory, "ledger.jsonl"); + const limits: EvalSpendLimits = { campaignId: "fixture", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z", + maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() }; + try { + const ledger = await openEvalCostLedger(file, limits); + try { + expect((await readEvalCostLedger(file)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000", inflight: 0, unknownRequests: 4 }); + await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId); + await expect(ledger.reserve("too-large", modelIdentity, "17")).rejects.toThrow("spending cap"); + await ledger.reserve("model", modelIdentity, "16"); + await ledger.observe({ observationId: "model-receipt", requestId: "model", source: "provider", provider: "anthropic", + currency: "USD", route: "direct", classification: "reported", costUsd: "1", final: true }); + await ledger.finish("model", "succeeded"); + expect(ledger.snapshot()).toMatchObject({ knownUsd: "1.000000000", heldUsd: "4.000000000", committedUsd: "5.000000000" }); + } finally { await ledger.close(); } + const changed = structuredClone(limits); changed.overhead!.components[0]!.maxUsd = "0"; + await expect(openEvalCostLedger(file, changed)).rejects.toThrow("limits changed"); + const resumed = await openEvalCostLedger(file, limits); + try { + await resumed.finish("overhead:ci_compute", "cancelled"); + expect(resumed.snapshot().heldUsd).toBe("4.000000000"); + await resumed.observe({ observationId: "ci-partial", requestId: "overhead:ci_compute", source: "infrastructure", provider: "aws", + currency: "USD", route: "infrastructure", classification: "estimated", pricingVersion: "fixture", costUsd: "0.2", final: false }); + expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.200000000", heldUsd: "3.800000000", committedUsd: "5.000000000", unknownRequests: 4 }); + await resumed.observe({ observationId: "ci-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", + currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true }); + expect(resumed.snapshot()).toMatchObject({ knownUsd: "1.500000000", heldUsd: "3.000000000", committedUsd: "4.500000000", unknownRequests: 3 }); + } finally { await resumed.close(); } + const report = await readEvalOperationsReport(file); + expect(report).toMatchObject({ requests: 5, unresolvedRequests: 3, knownUsd: "1.500000000", heldUsd: "3.000000000" }); + expect(renderEvalOperationsReport(report)).toContain("campaign-overhead"); + expect(renderEvalOperationsReport(report)).toContain("Held reservations are not charges"); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("stops further model admission when a later infrastructure receipt exceeds the shared cap", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-overrun-")); + try { + const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "overrun", maxUsd: "20", + deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: currentAllowance() }); + try { + await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId); + await ledger.observe({ observationId: "late-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", + currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true }); + expect(ledger.snapshot()).toMatchObject({ overBudget: true, committedUsd: "24.000000000" }); + await expect(ledger.reserve("next", modelIdentity, "0.1")).rejects.toThrow("spending cap"); + } finally { await ledger.close(); } + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("requires each cost category once and rejects unknown policy fields or missing provenance", () => { + expect(validateOperatingOverhead(allowance(), "20")).toBe(4_000_000_000n); + expect(() => validateOperatingOverhead({ ...allowance(), components: allowance().components.slice(1) }, "20")).toThrow("Incomplete"); + const duplicate = allowance(); duplicate.components[0] = duplicate.components[1]!; + expect(() => validateOperatingOverhead(duplicate, "20")).toThrow("component"); + expect(() => validateOperatingOverhead({ ...allowance(), apiKey: "never-allowed" } as never, "20")).toThrow("fields"); + const missing = allowance(); missing.components[0]!.reference = ""; + expect(() => validateOperatingOverhead(missing, "20")).toThrow("component"); + expect(() => validateOperatingOverhead(null as never, "20")).toThrow("fields"); + expect(() => validateOperatingOverhead({ ...allowance(), expiresAt: "2027-10-20T00:00:00Z" }, "20")).toThrow("Incomplete"); + expect(() => validateOperatingOverhead(allowance(), "4")).toThrow("no model allowance"); +}); + +it("requires fresh reviewed allowances tied to the actual CI runner before scheduled work", () => { + const hosted = { ...allowance(), runner: "github-hosted" as const }; + const raw = JSON.stringify(hosted); + expect(scheduledOperatingOverhead(raw, runner, "20", now)).toEqual(hosted); + expect(() => scheduledOperatingOverhead(undefined, runner, "20", now)).toThrow("Missing reviewed"); + expect(() => scheduledOperatingOverhead("not json", runner, "20", now)).toThrow("JSON"); + expect(() => scheduledOperatingOverhead(JSON.stringify(allowance()), runner, "20", now)).toThrow("runner"); + expect(() => scheduledOperatingOverhead(raw, "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", "20", now)).toThrow("runner"); + expect(() => scheduledOperatingOverhead(raw, "unrecognized", "20", now)).toThrow("runner"); + expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-10-19T23:59:59Z"))).toThrow("needs review"); + expect(() => scheduledOperatingOverhead(raw, runner, "20", Date.parse("2026-11-18T20:00:00Z"))).toThrow("needs review"); +}); + +it("does not turn a zero allowance into an observed zero charge", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-overhead-zero-")); + const overhead = allowance(); overhead.components[0]!.maxUsd = "0"; + try { + const ledger = await openEvalCostLedger(path.join(directory, "ledger.jsonl"), { campaignId: "zero", maxUsd: "20", + deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead }); + try { + expect(ledger.snapshot().requests.find(row => row.requestId === "overhead:ci_compute")) + .toMatchObject({ costUsd: null, classification: "unknown", receiptComplete: false, reservedUsd: "0.000000000" }); + } finally { await ledger.close(); } + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("stops new admission after allowance expiry while preserving late invoice reconciliation", () => { + const state = new EvalSpendState({ campaignId: "expired", maxUsd: "20", deadlineAt: "2099-01-01T00:00:00Z", + maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead: allowance() }); + const admitted = { type: "attempt" as const, executionId: modelIdentity.executionId, attemptId: modelIdentity.attemptId }; + state.validate(admitted, now); state.apply(admitted); + const expiredAt = Date.parse(allowance().expiresAt); + expect(() => state.validate({ ...admitted, attemptId: "next" }, expiredAt)).toThrow("needs review"); + expect(() => state.validate({ type: "reserve", requestId: "model", identity: modelIdentity, maxUsd: "1" }, expiredAt)).toThrow("needs review"); + expect(() => state.validate({ type: "observe", observation: { observationId: "late-invoice", requestId: "overhead:ci_compute", + source: "invoice", provider: "aws", currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true } }, expiredAt)).not.toThrow(); +}); diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts new file mode 100644 index 00000000000..08396df0ef3 --- /dev/null +++ b/tests/runner-e2e/operating-overhead.ts @@ -0,0 +1,53 @@ +import { usdToUnits } from "../../packages/shared/src/money.js"; + +export const EVAL_OVERHEAD_COMPONENTS = ["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const; +export interface EvalOperatingOverhead { + version: 1; + reference: string; + runner: "github-hosted" | "aws-fleet"; + verifiedAt: string; + expiresAt: string; + components: Array<{ component: typeof EVAL_OVERHEAD_COMPONENTS[number]; maxUsd: string; reference: string }>; +} + +const object = (value: unknown, allowed: string[]) => { + if (!value || typeof value !== "object" || Array.isArray(value) + || Object.keys(value).some(key => !allowed.includes(key))) throw new Error("Invalid operating overhead fields"); +}; +const reference = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9][A-Za-z0-9_.:/-]{0,511}$/.test(value); + +export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint { + object(value, ["version", "reference", "runner", "verifiedAt", "expiresAt", "components"]); + if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner) + || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string" + || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt)) + || Date.parse(value.expiresAt) <= Date.parse(value.verifiedAt) + || Date.parse(value.expiresAt) - Date.parse(value.verifiedAt) > 31 * 86_400_000 + || !Array.isArray(value.components) || value.components.length !== EVAL_OVERHEAD_COMPONENTS.length) { + throw new Error("Incomplete operating overhead allowance"); + } + const seen = new Set(); + let reserved = 0n; + for (const component of value.components) { + object(component, ["component", "maxUsd", "reference"]); + if (!EVAL_OVERHEAD_COMPONENTS.includes(component.component) || seen.has(component.component) + || !reference(component.reference) || typeof component.maxUsd !== "string" + || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(component.maxUsd)) throw new Error("Invalid operating overhead component"); + seen.add(component.component); + reserved += usdToUnits(component.maxUsd); + } + if (reserved >= usdToUnits(totalUsd)) throw new Error("Operating overhead leaves no model allowance"); + return reserved; +} + +export function scheduledOperatingOverhead(raw: string | undefined, runner: string | undefined, totalUsd: string, now: number): EvalOperatingOverhead { + if (!raw || raw.length > 16_384) throw new Error("Missing reviewed operating overhead allowance"); + let value: EvalOperatingOverhead; + try { value = JSON.parse(raw); } catch { throw new Error("Invalid operating overhead JSON"); } + validateOperatingOverhead(value, totalUsd); + if (runner !== "ubuntu-latest" || value.runner !== "github-hosted") throw new Error("Bounded campaigns require the standard GitHub-hosted runner allowance"); + if (!Number.isFinite(now) || now < Date.parse(value.verifiedAt) || now + 4 * 3_600_000 >= Date.parse(value.expiresAt)) { + throw new Error("Operating overhead allowance needs review before paid admission"); + } + return value; +} diff --git a/tests/runner-e2e/operating-receipts.test.ts b/tests/runner-e2e/operating-receipts.test.ts new file mode 100644 index 00000000000..4ce448949fd --- /dev/null +++ b/tests/runner-e2e/operating-receipts.test.ts @@ -0,0 +1,216 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import * as fs from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import { openEvalCostLedger, readEvalCostLedger } from "./cost-ledger.js"; +import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { importOperatingReceipt, operatingReceiptObservation, type EvalOperatingReceipt } from "./operating-receipts.js"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, open: vi.fn(actual.open) }; +}); + +const evidence = Buffer.from("fixture-only infrastructure receipt"); +const overhead: EvalOperatingOverhead = { version: 1, runner: "github-hosted", reference: "fixture-allowance", + verifiedAt: "2026-10-01T00:00:00Z", expiresAt: "2026-11-01T00:00:00Z", + components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) }; +const receipt = (fields: Partial = {}): EvalOperatingReceipt => ({ version: 2, costBasis: "component_cumulative", campaignId: "fixture", + allowanceReference: overhead.reference, component: "artifact_storage", evidenceSha256: createHash("sha256").update(evidence).digest("hex"), + windowStartedAt: "2026-10-01T00:00:00Z", windowEndedAt: "2026-10-02T00:00:00Z", coverage: "partial", + source: "infrastructure", provider: "fixture", classification: "estimated", costUsd: "0.2", pricingVersion: "fixture-pricing", ...fields }); +const now = Date.parse("2026-10-03T00:00:00Z"); + +it("binds operating observations to the campaign, allowance, evidence and closed observation window", () => { + expect(operatingReceiptObservation(receipt(), "fixture", overhead, evidence, now)).toMatchObject({ requestId: "overhead:artifact_storage", final: false }); + for (const changed of [receipt({ campaignId: "other" }), receipt({ allowanceReference: "other" }), receipt({ evidenceSha256: "0".repeat(64) }), + receipt({ windowEndedAt: "2026-10-04T00:00:00Z" }), receipt({ windowStartedAt: "2026-10-03T00:00:00Z" }), + receipt({ component: "model" as never }), receipt({ version: 1 as never }), receipt({ costBasis: undefined }), + receipt({ costBasis: "window_charge" as never }), { ...receipt(), apiKey: "disallowed" }]) { + expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow(); + } + expect(() => operatingReceiptObservation(receipt(), "fixture", overhead, Buffer.from("different"), now)).toThrow(); +}); + +it("does not upgrade missing or partial evidence to an invoice or completed receipt", () => { + for (const changed of [receipt({ classification: "reconciled" }), receipt({ classification: "unknown" }), + receipt({ classification: "unknown", costUsd: undefined, coverage: "complete" }), receipt({ pricingVersion: undefined }), receipt({ costUsd: "-1" })]) { + expect(() => operatingReceiptObservation(changed, "fixture", overhead, evidence, now)).toThrow(); + } + expect(operatingReceiptObservation(receipt({ classification: "unknown", costUsd: undefined }), "fixture", overhead, evidence, now)) + .toMatchObject({ classification: "unknown", final: false }); + expect(operatingReceiptObservation(receipt({ component: "shared_subscription", classification: "allocated", coverage: "complete" }), "fixture", overhead, evidence, now)) + .toMatchObject({ classification: "allocated", route: "subscription", final: true }); +}); + +async function fixture(maxUsd = "20") { + const directory = await mkdtemp(path.join(tmpdir(), "operating-receipts-")); + const ledgerPath = path.join(directory, "ledger.jsonl"); + const ledger = await openEvalCostLedger(ledgerPath, { campaignId: "fixture", maxUsd, deadlineAt: "2099-01-01T00:00:00Z", + maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead }); + await ledger.close(); + return { directory, ledgerPath }; +} + +it("retains private source evidence, imports idempotently and keeps partial holds until invoice reconciliation", async () => { + const { directory, ledgerPath } = await fixture(); + try { + expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000", unknownRequests: 4 }); + const before = await readFile(ledgerPath, "utf8"); + await importOperatingReceipt(ledgerPath, Object.fromEntries(Object.entries(receipt()).reverse()) as unknown as EvalOperatingReceipt, evidence); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + const retained = path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`); + expect(await readFile(retained)).toEqual(evidence); + expect((await stat(retained)).mode & 0o777).toBe(0o600); + expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2); + expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.3", coverage: "complete" }), evidence)) + .toMatchObject({ knownUsd: "0.300000000", heldUsd: "3.000000000", unknownRequests: 3 }); + expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.4", coverage: "complete" }), evidence)) + .toMatchObject({ knownUsd: "0.400000000", heldUsd: "3.600000000", conflictingRequests: 1 }); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("fails before ledger mutation when retained evidence is corrupt or another writer owns the journal", async () => { + const { directory, ledgerPath } = await fixture(); + try { + await importOperatingReceipt(ledgerPath, receipt(), evidence); + const before = await readFile(ledgerPath, "utf8"); + await writeFile(path.join(`${ledgerPath}.overhead`, `${receipt().evidenceSha256}.evidence`), "corrupted"); + await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.5" }), evidence)).rejects.toThrow("conflicts"); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits); + try { await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("writer lock"); } + finally { await writer.close(); } + expect(await readFile(ledgerPath, "utf8")).toBe(before); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("rejects separate partial windows and accounts for their cumulative total before admitting more work", async () => { + const { directory, ledgerPath } = await fixture("4.5"); + try { + await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8" }), evidence); + const before = await readFile(ledgerPath, "utf8"); + await expect(importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8", + windowStartedAt: "2026-10-02T00:00:00Z", windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("cumulative component window"); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + expect(await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6", + windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).toMatchObject({ knownUsd: "1.600000000", heldUsd: "3.000000000", committedUsd: "4.600000000", overBudget: true }); + const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits); + try { + await writer.admitAttempt("fixture-cell", "next"); + await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01")) + .rejects.toThrow("spending cap reached"); + } finally { await writer.close(); } + await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "0.9" }), evidence)).rejects.toThrow("cumulative component window"); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it.each(["partial", "complete"] as const)("rejects a larger cumulative estimate hidden by an earlier %s invoice", async coverage => { + const { directory, ledgerPath } = await fixture("4.5"); + try { + await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "0.8", coverage }), evidence); + const before = await readFile(ledgerPath, "utf8"), retained = await readdir(`${ledgerPath}.overhead`); + await expect(importOperatingReceipt(ledgerPath, receipt({ costUsd: "1.6", coverage, + windowEndedAt: "2026-10-03T00:00:00Z" }), evidence)).rejects.toThrow("would understate the cumulative total"); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + expect(await readdir(`${ledgerPath}.overhead`)).toEqual(retained); + const updated = await importOperatingReceipt(ledgerPath, receipt({ source: "invoice", classification: "reconciled", costUsd: "1.6", coverage, + windowEndedAt: "2026-10-03T00:00:00Z" }), evidence); + expect(updated).toMatchObject({ knownUsd: "1.600000000", committedUsd: "4.600000000", overBudget: true }); + const loaded = await readEvalCostLedger(ledgerPath), writer = await openEvalCostLedger(ledgerPath, loaded.state.limits); + try { + await writer.admitAttempt("fixture-cell", "next"); + await expect(writer.reserve("next", { executionId: "fixture-cell", attemptId: "next", phase: "agent" }, "0.01")) + .rejects.toThrow(coverage === "complete" ? "receipts conflict" : "spending cap reached"); + } finally { await writer.close(); } + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it.each(["evidence", "receipt.json"])("retries must persist the existing %s file before recording its charge", async suffix => { + const { directory, ledgerPath } = await fixture(); + const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual("node:fs/promises")).open; + const value = receipt(), canonical = Object.fromEntries(Object.entries(value).sort(([a], [b]) => a.localeCompare(b))); + const observation = operatingReceiptObservation(value, "fixture", overhead, evidence, now); + await mkdir(`${ledgerPath}.overhead`, { mode: 0o700 }); + await writeFile(path.join(`${ledgerPath}.overhead`, suffix === "evidence" ? `${value.evidenceSha256}.evidence` + : `${observation.observationId.split(":")[1]}.receipt.json`), suffix === "evidence" ? evidence : `${JSON.stringify(canonical, null, 2)}\n`, { mode: 0o600 }); + let syncs = 0; + const opened = vi.mocked(fs.open).mockImplementation(async (...args) => { + const handle = await originalOpen(...args); + if (String(args[0]).endsWith(`.${suffix}`)) { + const originalSync = handle.sync.bind(handle); + vi.spyOn(handle, "sync").mockImplementation(async () => { + if (++syncs <= 2) throw new Error("fixture file sync failure"); + await originalSync(); + }); + } + return handle; + }); + try { + for (let attempt = 0; attempt < 2; attempt++) { + await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("file sync failure"); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + } + expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" }); + expect(syncs).toBe(3); + } finally { + opened.mockImplementation(originalOpen); + await rm(directory, { recursive: true, force: true }); + } +}); + +it.each(["evidence", "receipt.json"])("recovers an incomplete %s write without publishing corrupt evidence or requiring manual deletion", async suffix => { + const { directory, ledgerPath } = await fixture(); + const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual("node:fs/promises")).open; + let injected = false; + const opened = vi.mocked(fs.open).mockImplementation(async (...args) => { + const handle = await originalOpen(...args); + if (!injected && String(args[0]).includes(`.${suffix}.`) && String(args[0]).endsWith(".tmp")) { + const originalWrite = handle.writeFile.bind(handle); + vi.spyOn(handle, "writeFile").mockImplementation(async () => { + injected = true; + await originalWrite("partial"); + throw new Error("fixture partial write"); + }); + } + return handle; + }); + try { + await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("partial write"); + expect(injected).toBe(true); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + expect((await readdir(`${ledgerPath}.overhead`)).some(name => name.endsWith(`.${suffix}`) || name.endsWith(".tmp"))).toBe(false); + expect(await importOperatingReceipt(ledgerPath, receipt(), evidence)).toMatchObject({ knownUsd: "0.200000000", heldUsd: "3.800000000" }); + expect(await readdir(`${ledgerPath}.overhead`)).toHaveLength(2); + } finally { + opened.mockImplementation(originalOpen); + await rm(directory, { recursive: true, force: true }); + } +}); + +it("retains the hold when the first evidence directory cannot be persisted in its parent", async () => { + const { directory, ledgerPath } = await fixture(); + const before = await readFile(ledgerPath, "utf8"), originalOpen = (await vi.importActual("node:fs/promises")).open; + let parentSyncs = 0; + const opened = vi.mocked(fs.open).mockImplementation(async (...args) => { + const handle = await originalOpen(...args); + if (args[0] === directory && args[1] === "r") { + vi.spyOn(handle, "sync").mockImplementation(async () => { + parentSyncs++; + throw new Error("fixture parent directory sync failure"); + }); + } + return handle; + }); + try { + await expect(importOperatingReceipt(ledgerPath, receipt(), evidence)).rejects.toThrow("parent directory sync failure"); + expect(parentSyncs).toBe(1); + expect(await readFile(ledgerPath, "utf8")).toBe(before); + expect((await readEvalCostLedger(ledgerPath)).state.snapshot()).toMatchObject({ knownUsd: "0.000000000", heldUsd: "4.000000000" }); + } finally { + opened.mockImplementation(originalOpen); + await rm(directory, { recursive: true, force: true }); + } +}); diff --git a/tests/runner-e2e/operating-receipts.ts b/tests/runner-e2e/operating-receipts.ts new file mode 100644 index 00000000000..8806292b0c1 --- /dev/null +++ b/tests/runner-e2e/operating-receipts.ts @@ -0,0 +1,136 @@ +import { createHash, randomUUID } from "node:crypto"; +import { link, mkdir, open, readFile, rm } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { openEvalCostLedger, readEvalCostLedger, validEvalLedgerId, type EvalCostObservation } from "./cost-ledger.js"; +import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { collectGitHubOperatingUsage } from "./github-operating-usage.js"; + +export interface EvalOperatingReceipt { + version: 2; + costBasis: "component_cumulative"; + campaignId: string; + allowanceReference: string; + component: typeof EVAL_OVERHEAD_COMPONENTS[number]; + evidenceSha256: string; + windowStartedAt: string; + windowEndedAt: string; + coverage: "complete" | "partial"; + source: "infrastructure" | "invoice"; + provider: string; + classification: EvalCostObservation["classification"]; + costUsd?: string; + pricingVersion?: string; +} +const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +const canonical = (receipt: EvalOperatingReceipt) => Object.fromEntries(Object.entries(receipt).sort(([left], [right]) => left.localeCompare(right))); + +export function operatingReceiptObservation(receipt: EvalOperatingReceipt, campaignId: string, overhead: EvalOperatingOverhead, + evidence: Buffer, now = Date.now()): EvalCostObservation { + if (!receipt || typeof receipt !== "object" || Array.isArray(receipt) || Object.keys(receipt).some(key => ![ + "version", "costBasis", "campaignId", "allowanceReference", "component", "evidenceSha256", "windowStartedAt", "windowEndedAt", + "coverage", "source", "provider", "classification", "costUsd", "pricingVersion", + ].includes(key))) throw new Error("Invalid operating receipt fields"); + if (receipt.version !== 2 || receipt.costBasis !== "component_cumulative" + || receipt.campaignId !== campaignId || receipt.allowanceReference !== overhead.reference + || !EVAL_OVERHEAD_COMPONENTS.includes(receipt.component) || !validEvalLedgerId(receipt.provider) + || !["infrastructure", "invoice"].includes(receipt.source) || !["complete", "partial"].includes(receipt.coverage) + || receipt.evidenceSha256 !== hash(evidence) || !/^[a-f0-9]{64}$/.test(receipt.evidenceSha256) + || !Number.isFinite(now) || typeof receipt.windowStartedAt !== "string" || typeof receipt.windowEndedAt !== "string" + || !Number.isFinite(Date.parse(receipt.windowStartedAt)) || !Number.isFinite(Date.parse(receipt.windowEndedAt)) + || Date.parse(receipt.windowStartedAt) > Date.parse(receipt.windowEndedAt) || Date.parse(receipt.windowEndedAt) > now) { + throw new Error("Operating receipt scope, evidence or window mismatch"); + } + if (!["reported", "estimated", "allocated", "reconciled", "unknown"].includes(receipt.classification) + || receipt.classification === "reconciled" && receipt.source !== "invoice" + || receipt.classification === "unknown" && (receipt.costUsd !== undefined || receipt.coverage === "complete") + || receipt.classification !== "unknown" && (typeof receipt.costUsd !== "string" || !/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(receipt.costUsd)) + || ["estimated", "allocated"].includes(receipt.classification) && !validEvalLedgerId(receipt.pricingVersion) + || receipt.pricingVersion !== undefined && !validEvalLedgerId(receipt.pricingVersion)) throw new Error("Invalid operating cost classification or provenance"); + return { observationId: `operating-receipt:${hash(JSON.stringify(canonical(receipt)))}`, requestId: `overhead:${receipt.component}`, + source: receipt.source, provider: receipt.provider, currency: "USD", + route: receipt.component === "shared_subscription" ? "subscription" : "infrastructure", + classification: receipt.classification, ...(receipt.costUsd === undefined ? {} : { costUsd: receipt.costUsd }), + ...(receipt.pricingVersion === undefined ? {} : { pricingVersion: receipt.pricingVersion }), final: receipt.coverage === "complete" }; +} + +async function retain(file: string, bytes: Buffer) { + const temporary = `${file}.${randomUUID()}.tmp`; + const handle = await open(temporary, "wx", 0o600); + try { + try { await handle.writeFile(bytes); await handle.sync(); } finally { await handle.close(); } + // Hard-link publication is atomic and cannot replace previously retained evidence. + try { await link(temporary, file); } + catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const existing = await open(file, "r+"); + try { + if (!(await existing.readFile()).equals(bytes)) throw new Error("Retained operating evidence conflicts"); + await existing.sync(); + } finally { await existing.close(); } + } + } finally { await rm(temporary, { force: true }); } +} + +export async function importOperatingReceipt(ledgerPath: string, receipt: EvalOperatingReceipt, evidence: Buffer) { + if (evidence.length < 1 || evidence.length > 8 * 1024 * 1024) throw new Error("Operating evidence must be between one byte and 8 MiB"); + const loaded = await readEvalCostLedger(ledgerPath); + if (!loaded.state.limits.overhead) throw new Error("Campaign has no declared operating overhead"); + const observation = operatingReceiptObservation(receipt, loaded.state.limits.campaignId, loaded.state.limits.overhead, evidence); + const ledger = await openEvalCostLedger(ledgerPath, loaded.state.limits); + try { + const directory = `${ledgerPath}.overhead`; + const current = (await readEvalCostLedger(ledgerPath)).state; + const alreadyImported = current.observations.has(observation.observationId); + if (!alreadyImported) { + for (const previous of current.observations.values()) { + if (previous.requestId !== observation.requestId) continue; + if (!/^operating-receipt:[a-f0-9]{64}$/.test(previous.observationId)) { + throw new Error("Operating receipt lacks cumulative window evidence"); + } + const prior = JSON.parse(await readFile(path.join(directory, `${previous.observationId.split(":")[1]}.receipt.json`), "utf8")) as EvalOperatingReceipt; + if (previous.observationId !== `operating-receipt:${hash(JSON.stringify(canonical(prior)))}` + || prior.version !== 2 || prior.costBasis !== "component_cumulative") { + throw new Error("Retained operating receipt lacks verified cumulative window evidence"); + } + if (Date.parse(receipt.windowStartedAt) !== Date.parse(prior.windowStartedAt) + || Date.parse(receipt.windowEndedAt) < Date.parse(prior.windowEndedAt) + || prior.coverage === "complete" && receipt.coverage !== "complete") { + throw new Error("Operating receipts must cover one cumulative component window; combine separate charges before importing"); + } + } + // Preview the ledger's own precedence rules before retaining or appending a receipt. + current.validate({ type: "observe", observation }); + current.apply({ type: "observe", observation }); + const counted = current.snapshot().requests.find(request => request.requestId === observation.requestId)?.costUsd; + if (observation.costUsd !== undefined && (counted == null || usdToUnits(counted) < usdToUnits(observation.costUsd))) { + throw new Error("Operating receipt would understate the cumulative total; reconcile the entire component window before importing"); + } + } + await mkdir(directory, { recursive: true, mode: 0o700 }); + await retain(path.join(directory, `${receipt.evidenceSha256}.evidence`), evidence); + await retain(path.join(directory, `${observation.observationId.split(":")[1]}.receipt.json`), Buffer.from(`${JSON.stringify(canonical(receipt), null, 2)}\n`)); + for (const durableDirectory of [directory, path.dirname(directory)]) { + const directoryHandle = await open(durableDirectory, "r"); + try { await directoryHandle.sync(); } finally { await directoryHandle.close(); } + } + if (!alreadyImported) await ledger.observe(observation); + return ledger.snapshot(); + } finally { await ledger.close(); } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const [command, first, second, third, ...rest] = process.argv.slice(2); + if (command === "github" && first && second && !third) { + const usage = await collectGitHubOperatingUsage(first); + await retain(path.resolve(second), Buffer.from(`${JSON.stringify(usage, null, 2)}\n`)); + console.log(JSON.stringify({ source: usage.workflowUrl, summedJobSeconds: usage.summedJobSeconds, + retainedArtifactBytes: usage.artifacts.retainedBytes, billing: "unknown", output: path.resolve(second) })); + } else if (command === "import" && first && second && third && !rest.length) { + const receipt = JSON.parse(await readFile(second, "utf8")) as EvalOperatingReceipt; + const result = await importOperatingReceipt(path.resolve(first), receipt, await readFile(third)); + console.log(JSON.stringify({ campaignId: result.campaignId, knownUsd: result.knownUsd, heldUsd: result.heldUsd, + committedUsd: result.committedUsd, unknownRequests: result.unknownRequests, conflictingRequests: result.conflictingRequests })); + } else throw new Error("Usage: operating-receipts.ts github | import "); +} diff --git a/tests/runner-e2e/operations-report.test.ts b/tests/runner-e2e/operations-report.test.ts new file mode 100644 index 00000000000..a310ddd35e9 --- /dev/null +++ b/tests/runner-e2e/operations-report.test.ts @@ -0,0 +1,93 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { openEvalCostLedger } from "./cost-ledger.js"; +import { readEvalOperationsReport, renderEvalOperationsReport, validateEvalOperationsReport } from "./operations-report.js"; +import { buildRunnerCampaign } from "./history.js"; +import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; + +it("reports shared setup/retry spending once, preserves unknown holds and omits private provider receipts", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")); + const file = path.join(root, "ledger.jsonl"); + const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 }); + try { + await ledger.reserve("setup", { executionId: "cell", attemptId: "first", phase: "setup" }, "0.2"); + await ledger.observe({ observationId: "gateway", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "gateway", + currency: "USD", route: "direct", classification: "estimated", pricingVersion: "fixture", costUsd: "0.05", final: true }); + await ledger.observe({ observationId: "native", requestId: "setup", provider: "fixture", providerRequestId: "private-provider-id", source: "native_otlp", + currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true }); + await ledger.finish("setup", "failed"); + await ledger.reserve("retry", { executionId: "cell", attemptId: "retry", phase: "agent" }, "0.2"); + await ledger.finish("retry", "cancelled"); + const report = await readEvalOperationsReport(file); + validateEvalOperationsReport(report); + if (report.status === "unavailable") throw new Error("Expected readable fixture ledger"); + expect(report).toMatchObject({ knownUsd: "0.050000000", heldUsd: "0.200000000", committedUsd: "0.250000000", requests: 2, + unresolvedRequests: 1, inflight: 0, breakdown: { reported: "0.050000000", estimated: "0.000000000" }, + attempts: [{ attemptId: "first", requests: 1, knownUsd: "0.050000000" }, { attemptId: "retry", unresolvedRequests: 1, heldUsd: "0.200000000" }] }); + expect(JSON.stringify(report)).not.toContain("private-provider-id"); + const campaign = buildRunnerCampaign({ campaignId: "retry-campaign", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report }); + expect(campaign.operations).toEqual(report); + expect(campaign.billing.reportedLlmCostUsd).toBe(0); + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")); + expect(rebuilt.operations).toEqual(report); + expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("$0.200000000 additional held reservations"); + expect(renderEvalOperationsReport(report)).toContain("Do not add its model subtotal to heartbeat totals"); + expect(renderEvalOperationsReport(report)).toContain("Unrecorded infrastructure, storage, subscriptions"); + expect(() => validateEvalOperationsReport({ ...report, secret: "forbidden" } as never)).toThrow("Invalid retained"); + expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.060000000" })).toThrow("Invalid retained"); + expect(() => validateEvalOperationsReport({ ...report, attempts: [] })).toThrow("Invalid retained"); + expect(() => validateEvalOperationsReport({ ...report, deadlineAt: "Tue, 20 Oct 2026 01:00:00 GMT (private note)" })).toThrow("Invalid retained"); + } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); } +}); + +it("publishes the parsed deadline without private text accepted by the journal's date parser", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl"); + const deadlineAt = "Thu, 01 Jan 2099 00:00:00 GMT (private budget rationale)"; + const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt, maxRequests: 10, maxConcurrent: 2 }); + try { + const report = await readEvalOperationsReport(file); + expect(report).toMatchObject({ deadlineAt: "2099-01-01T00:00:00.000Z" }); + expect(JSON.stringify(report)).not.toContain("private budget rationale"); + expect(await readFile(file, "utf8")).toContain(deadlineAt); + } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); } +}); + +it("preserves reports with explicit unknown accounting after a partial journal write", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl"); + const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 }); + await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "setup" }, "0.2"); + await ledger.close(); + try { + await writeFile(file, '{"partial-record":', { flag: "a" }); + const damaged = await readFile(file, "utf8"); + const report = await readEvalOperationsReport(file); + expect(report).toEqual({ version: 1, scope: "shared_gateway_ledger", status: "unavailable" }); + validateEvalOperationsReport(report); + expect(renderEvalOperationsReport(report)).toContain("Request amounts, counts, and held reservations are unknown"); + expect(renderEvalOperationsReport(report)).not.toContain("$0"); + expect(() => validateEvalOperationsReport({ ...report, knownUsd: "0.000000000" } as never)).toThrow("Invalid retained"); + const campaign = buildRunnerCampaign({ campaignId: "damaged-ledger", generatedAt: new Date().toISOString(), expected: [], results: [], operations: report }); + await writeFile(path.join(root, "normalized-results.json"), JSON.stringify(campaign)); + await regenerateRunnerDashboard({ bundle: root, historyFile: null }); + const rebuilt = JSON.parse(await readFile(path.join(root, "normalized-results.json"), "utf8")); + expect(rebuilt.operations).toEqual(report); + expect(await readFile(path.join(root, "dashboard.html"), "utf8")).toContain("Request ledger accounting unavailable"); + expect(await readFile(file, "utf8")).toBe(damaged); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("keeps an in-flight final receipt reserved until the forwarding request finishes", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-operations-report-")), file = path.join(root, "ledger.jsonl"); + const ledger = await openEvalCostLedger(file, { campaignId: "pilot", maxUsd: "1", deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 2 }); + try { + await ledger.reserve("request", { executionId: "cell", attemptId: "attempt", phase: "agent" }, "0.2"); + await ledger.observe({ observationId: "usage", requestId: "request", provider: "fixture", source: "provider", currency: "USD", route: "direct", classification: "reported", costUsd: "0.05", final: true }); + const report = await readEvalOperationsReport(file); + validateEvalOperationsReport(report); + expect(report).toMatchObject({ inflight: 1, knownUsd: "0.050000000", heldUsd: "0.150000000", committedUsd: "0.200000000", unresolvedRequests: 0 }); + } finally { await ledger.close(); await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/operations-report.ts b/tests/runner-e2e/operations-report.ts new file mode 100644 index 00000000000..97d8a3df776 --- /dev/null +++ b/tests/runner-e2e/operations-report.ts @@ -0,0 +1,114 @@ +import { readEvalCostLedger, validEvalLedgerId } from "./cost-ledger.js"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { html, isPublicReportTimestamp } from "./report-format.js"; + +const money = (n: bigint) => `${n / 1_000_000_000n}.${String(n % 1_000_000_000n).padStart(9, "0")}`; +interface EvalOperationsAvailableReport { + version: 1; + scope: "shared_gateway_ledger"; + status?: never; + ledgerCampaignId: string; + journalHash: string; + records: number; + deadlineAt: string; + limitUsd: string; + knownUsd: string; + heldUsd: string; + committedUsd: string; + requests: number; + unresolvedRequests: number; + conflictingRequests: number; + inflight: number; + breakdown: { reported: string; estimated: string; allocated: string; reconciled: string }; + attempts: Array<{ executionId: string; attemptId: string; requests: number; unresolvedRequests: number; knownUsd: string; heldUsd: string }>; +} +export type EvalOperationsReport = EvalOperationsAvailableReport | { + version: 1; scope: "shared_gateway_ledger"; status: "unavailable"; +}; + +export function validateEvalOperationsReport(report: EvalOperationsReport): void { + const fail = (): never => { throw new Error("Invalid retained eval operations report"); }; + const fields = (value: unknown, allowed: string[]) => { + if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(key => !allowed.includes(key))) fail(); + }; + if (report?.status === "unavailable") { + fields(report, ["version", "scope", "status"]); + if (report.version !== 1 || report.scope !== "shared_gateway_ledger") fail(); + return; + } + const dollars = (value: unknown): bigint => typeof value === "string" && /^(?:0|[1-9][0-9]*)\.[0-9]{9}$/.test(value) ? usdToUnits(value) : fail(); + const count = (value: unknown) => { if (!Number.isSafeInteger(value) || Number(value) < 0) fail(); }; + fields(report, ["version", "scope", "ledgerCampaignId", "journalHash", "records", "deadlineAt", "limitUsd", "knownUsd", "heldUsd", "committedUsd", "requests", "unresolvedRequests", "conflictingRequests", "inflight", "breakdown", "attempts"]); + if (report.version !== 1 || report.scope !== "shared_gateway_ledger" || !validEvalLedgerId(report.ledgerCampaignId) + || !/^[a-f0-9]{64}$/.test(report.journalHash) || !isPublicReportTimestamp(report.deadlineAt) || !Array.isArray(report.attempts)) fail(); + [report.records, report.requests, report.unresolvedRequests, report.conflictingRequests, report.inflight].forEach(count); + if (report.records < 1 || report.unresolvedRequests > report.requests || report.conflictingRequests > report.unresolvedRequests || report.inflight > report.requests) fail(); + dollars(report.limitUsd); + const known = dollars(report.knownUsd), held = dollars(report.heldUsd); + if (known + held !== dollars(report.committedUsd)) fail(); + fields(report.breakdown, ["reported", "estimated", "allocated", "reconciled"]); + if (["reported", "estimated", "allocated", "reconciled"].reduce((sum, key) => sum + dollars(report.breakdown[key as keyof typeof report.breakdown]), 0n) !== known) fail(); + let requests = 0, unresolved = 0, rowKnown = 0n, rowHeld = 0n; + const seen = new Set(); + for (const attempt of report.attempts) { + fields(attempt, ["executionId", "attemptId", "requests", "unresolvedRequests", "knownUsd", "heldUsd"]); + if (!validEvalLedgerId(attempt.executionId) || !validEvalLedgerId(attempt.attemptId)) fail(); + const key = JSON.stringify([attempt.executionId, attempt.attemptId]); if (seen.has(key)) fail(); seen.add(key); + count(attempt.requests); count(attempt.unresolvedRequests); + if (attempt.requests < 1 || attempt.unresolvedRequests > attempt.requests) fail(); + requests += attempt.requests; unresolved += attempt.unresolvedRequests; + rowKnown += dollars(attempt.knownUsd); rowHeld += dollars(attempt.heldUsd); + } + if (requests !== report.requests || unresolved !== report.unresolvedRequests || rowKnown !== known || rowHeld !== held) fail(); +} + +export async function readEvalOperationsReport(file: string): Promise { + try { return await readAvailableEvalOperationsReport(file); } + catch { return { version: 1, scope: "shared_gateway_ledger", status: "unavailable" }; } +} + +async function readAvailableEvalOperationsReport(file: string): Promise { + const loaded = await readEvalCostLedger(file), snapshot = loaded.state.snapshot(); + const groups = new Map(); + for (const request of snapshot.requests) { + const key = JSON.stringify([request.executionId, request.attemptId]); + const group = groups.get(key) ?? { executionId: request.executionId, attemptId: request.attemptId, + requests: 0, unresolvedRequests: 0, knownUsd: money(0n), heldUsd: money(0n) }; + const cost = usdToUnits(request.costUsd ?? "0"), reserved = usdToUnits(request.reservedUsd); + const held = (!request.finished || !request.receiptComplete) && reserved > cost ? reserved - cost : 0n; + group.requests++; if (!request.receiptComplete) group.unresolvedRequests++; + group.knownUsd = money(usdToUnits(group.knownUsd) + cost); + group.heldUsd = money(usdToUnits(group.heldUsd) + held); + groups.set(key, group); + } + const report: EvalOperationsAvailableReport = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: snapshot.campaignId, + journalHash: loaded.previous, records: loaded.sequence, deadlineAt: new Date(loaded.state.limits.deadlineAt).toISOString(), + limitUsd: snapshot.limitUsd, knownUsd: snapshot.knownUsd, heldUsd: snapshot.heldUsd, committedUsd: snapshot.committedUsd, + requests: snapshot.requests.length, unresolvedRequests: snapshot.unknownRequests, conflictingRequests: snapshot.conflictingRequests, + inflight: snapshot.inflight, breakdown: snapshot.breakdown as EvalOperationsAvailableReport["breakdown"], attempts: [...groups.values()] }; + validateEvalOperationsReport(report); + return report; +} + +export function operationsReportText(report: EvalOperationsReport): string[] { + if (report.status === "unavailable") return [ + "Request ledger accounting unavailable: the journal could not be read or validated. Request amounts, counts, and held reservations are unknown.", + "Retained test results are preserved. Recover and validate the original journal before resuming paid work; do not reset the allowance or treat missing amounts as zero.", + ]; + return [ + `Shared request ledger ${report.ledgerCampaignId}: $${report.knownUsd} known subtotal; $${report.heldUsd} additional held reservations; $${report.committedUsd} committed against $${report.limitUsd}.`, + `Charges: $${report.breakdown.reconciled} reconciled; $${report.breakdown.reported} reported; $${report.breakdown.estimated} estimated; $${report.breakdown.allocated} allocated.`, + `${report.requests} recorded requests or overhead reservations; ${report.unresolvedRequests} unresolved; ${report.conflictingRequests} conflicting; ${report.inflight} in flight. Deadline ${report.deadlineAt}.`, + "This shared ledger includes earlier attempts and setup probes routed through the gateway, plus any declared campaign-overhead reservations. Do not add its model subtotal to heartbeat totals. Held reservations are not charges. Unrecorded infrastructure, storage, subscriptions and requests outside the gateway still need accounting; zero recorded requests does not prove zero cost.", + `Journal checkpoint: ${report.records} records; SHA-256 ${report.journalHash}.`, + ]; +} + +export function renderEvalOperationsReport(report: EvalOperationsReport): string { + validateEvalOperationsReport(report); + if (report.status === "unavailable") return `

Request ledger accounting unavailable

${operationsReportText(report).map(line => `

${html(line)}

`).join("")}
`; + return `

Request ledger accounting

${operationsReportText(report).map(line => `

${html(line)}

`).join("")} + + ${report.attempts.map(attempt => ``).join("")} +
ExecutionAttempt or componentRequests / reservationsKnown subtotalAdditional heldUnresolved
${html(attempt.executionId)}${html(attempt.attemptId)}${attempt.requests}$${html(attempt.knownUsd)}$${html(attempt.heldUsd)}${attempt.unresolvedRequests}
`; +} diff --git a/tests/runner-e2e/operations.test.ts b/tests/runner-e2e/operations.test.ts new file mode 100644 index 00000000000..7193837a012 --- /dev/null +++ b/tests/runner-e2e/operations.test.ts @@ -0,0 +1,173 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { access, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import { runnerMatrix } from "./catalog.js"; +import { validateEvalOperationsManifest, openEvalOperations, type EvalOperationsManifest } from "./operations.js"; +import { setupLiveFixtures } from "./live-fixtures.js"; +import { createAiConnectionSchema, aiConnectionBindingSchema } from "../../packages/shared/src/ai-connections.js"; +import type { RunnerApi } from "./api.js"; +import { beginEvalAgentPhase, readEvalAttemptPhase } from "./attempt-phase.js"; +import { applyEvalGatewayEnvironment, buildPaperclipServerEnvironment } from "./harness-env.js"; +const cell = runnerMatrix.find(e => e.suite.id === "core-compatibility" && e.profile.id === "runner-codex" && e.environment.id === "local")!; +function manifest(): EvalOperationsManifest { + return { version: 1, limits: { campaignId: "pilot", maxUsd: "100", deadlineAt: new Date(Date.now() + 60_000).toISOString(), maxRequests: 20, maxConcurrent: 1 }, + ledgerPath: "/tmp/fixture-not-opened.jsonl", executionIds: [cell.id], maxAttemptsPerExecution: 2, route: "direct", + models: [{ provider: "openai", maximumInputTokens: 1000, maximumOutputTokens: 100, timeoutMs: 1000, + price: { provider: "openai", model: cell.profile.model, version: "fixture-only", sourceUrl: "https://provider.invalid/fixture", inputUsdPerMillion: "2", cachedReadUsdPerMillion: "1", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" } }] }; +} +afterEach(() => { vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); +it("admits only the exact bounded local selection and explicitly priced models", () => { + expect(validateEvalOperationsManifest(manifest(), [cell], 1)).toMatchObject({ route: "direct" }); + expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [] }, [cell], 1)).toThrow("selection"); + expect(() => validateEvalOperationsManifest(manifest(), [cell], 2)).toThrow("limits"); + const remote = runnerMatrix.find(e => e.suite.id === cell.suite.id && e.profile.id === cell.profile.id && e.environment.id === "daytona")!; + expect(() => validateEvalOperationsManifest({ ...manifest(), executionIds: [remote.id] }, [remote], 1)).toThrow("Unqualified"); + const changed = manifest(); changed.models[0]!.price.model = "other"; + expect(() => validateEvalOperationsManifest(changed, [cell], 1)).toThrow("Missing exact model"); +}); +it("rejects credentials in the manifest and unknown pricing/billing policy fields", () => { + expect(() => validateEvalOperationsManifest({ ...manifest(), apiKey: "private" } as never, [cell], 0)).toThrow("fields"); + const changed = manifest(); Object.assign(changed.models[0]!, { apiKey: "private" }); + expect(() => validateEvalOperationsManifest(changed, [cell], 0)).toThrow("fields"); + expect(() => validateEvalOperationsManifest({ ...manifest(), route: "braintrust" }, [cell], 0)).toThrow("project UUID"); +}); + +it("rejects an unaffordable selected request before opening a ledger or checking credentials", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-unaffordable-preflight-")); + try { + const value = manifest(); value.limits.maxUsd = "0.002999999"; value.ledgerPath = path.join(root, "ledger.jsonl"); + const file = path.join(root, "manifest.json"); await writeFile(file, JSON.stringify(value)); + await expect(openEvalOperations(file, [cell], 0, {})).rejects.toThrow("Insufficient initial model allowance"); + await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(`${value.ledgerPath}.requests`)).rejects.toMatchObject({ code: "ENOENT" }); + value.limits.maxUsd = "0.003"; + expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("subtracts declared operating holds and checks only models used by the selection", () => { + const value = manifest(); value.limits.maxUsd = "0.007"; value.limits.maxAttemptsPerExecution = 2; + value.limits.overhead = { version: 1, reference: "fixture", runner: "github-hosted", + verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z", + components: (["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const).map(component => ({ + component, maxUsd: "0.001", reference: "fixture" })) }; + value.models.push({ ...structuredClone(value.models[0]!), maximumInputTokens: 1_000_000, + price: { ...value.models[0]!.price, model: "unused-model" } }); + expect(validateEvalOperationsManifest(value, [cell], 0)).toBe(value); + value.limits.maxUsd = "0.006999999"; + expect(() => validateEvalOperationsManifest(value, [cell], 0)).toThrow("Insufficient initial model allowance"); +}); +it("retains a campaign summary and refuses changing its budget on restart", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-ops-test-")); + try { + const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json"); + await writeFile(file, JSON.stringify(value)); + const operations = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" }); + const endpoint = await operations.start(cell, "attempt", 1); + expect(endpoint.baseUrl).toMatch(/^http:\/\/127\.0\.0\.1:/); + expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "setup" }); + await beginEvalAgentPhase(cell.id, { PAPERCLIP_E2E_GATEWAY_PHASE_PATH: endpoint.phasePath, PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID: "attempt" }); + expect(await readEvalAttemptPhase(endpoint.phasePath, { executionId: cell.id, attemptId: "attempt" })).toMatchObject({ phase: "agent" }); + await endpoint.close(); await operations.close(); + expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8"))).toMatchObject({ limitUsd: "100.000000000", knownUsd: "0.000000000", requests: [] }); + const resumed = await openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" }); + const retry = await resumed.start(cell, "retry-after-restart", 1); + await retry.close(); + await expect(resumed.start(cell, "third-attempt", 1)).rejects.toThrow("attempt cap"); + await resumed.close(); + const summary = JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8")); + expect(summary.admittedAttempts).toEqual([{ executionId: cell.id, attemptId: "attempt" }, { executionId: cell.id, attemptId: "retry-after-restart" }]); + await writeFile(file, JSON.stringify({ ...value, maxAttemptsPerExecution: 3 })); + await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed"); + await writeFile(file, JSON.stringify({ ...value, limits: { ...value.limits, maxUsd: "101" } })); + await expect(openEvalOperations(file, [cell], 0, { OPENAI_API_KEY: "private-fixture" })).rejects.toThrow("limits changed"); + } finally { await rm(root, { recursive: true, force: true }); } +}); + +it("keeps both upstream keys on the controller and owns the trace relay lifetime", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-")); + const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl") }, file = path.join(root, "manifest.json"); + const env = { OPENAI_API_KEY: "provider-private", BRAINTRUST_API_KEY: "braintrust-private", BRAINTRUST_OTHER_KEY: "other-private", + GITHUB_SHA: "a".repeat(40), PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "11111111-1111-4111-8111-111111111111", + PAPERCLIP_E2E_OTLP_RELAY_URL: "https://unowned.invalid/otel" }; + let operations: Awaited> | undefined; + try { + await writeFile(file, JSON.stringify(value)); + operations = await openEvalOperations(file, [cell], 0, env); + const endpoint = await operations.start(cell, "with-observability", 1); + const worker: NodeJS.ProcessEnv = { ...env }; + applyEvalGatewayEnvironment(worker, cell.profile.credential, endpoint); + expect(worker).toMatchObject({ OPENAI_API_KEY: endpoint.token, BRAINTRUST_API_KEY: endpoint.observability!.token, + PAPERCLIP_E2E_OTLP_RELAY_URL: endpoint.observability!.baseUrl, GITHUB_SHA: env.GITHUB_SHA }); + expect(JSON.stringify(worker)).not.toMatch(/provider-private|braintrust-private|other-private/); + expect(buildPaperclipServerEnvironment(worker).BRAINTRUST_API_KEY).toBeUndefined(); + expect((await fetch(endpoint.observability!.baseUrl + "/v1/traces", { method: "POST" })).status).toBe(401); + await operations.close(); operations = undefined; + await expect(fetch(endpoint.baseUrl)).rejects.toThrow(); + await expect(fetch(endpoint.observability!.baseUrl)).rejects.toThrow(); + const receipts = (await readFile(endpoint.observability!.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + expect(receipts.map(row => row.type)).toEqual(["policy", "closed"]); + expect(receipts[0]).toMatchObject({ campaignId: "pilot", attemptId: "with-observability", projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID }); + const withoutTracing: NodeJS.ProcessEnv = { ...env }; + applyEvalGatewayEnvironment(withoutTracing, cell.profile.credential, { ...endpoint, observability: undefined }); + expect(withoutTracing.BRAINTRUST_API_KEY).toBeUndefined(); + expect(withoutTracing.PAPERCLIP_E2E_OTLP_RELAY_URL).toBeUndefined(); + } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); } +}); + +it("rejects mismatched trace projects before opening the journal and retains failed relay admission", async () => { + const root = await mkdtemp(path.join(tmpdir(), "eval-ops-relay-failure-")); + const value = { ...manifest(), ledgerPath: path.join(root, "ledger.jsonl"), route: "braintrust" as const, + projectId: "11111111-1111-4111-8111-111111111111", maxAttemptsPerExecution: 1 }, file = path.join(root, "manifest.json"); + const env = { BRAINTRUST_API_KEY: "braintrust-private", PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "22222222-2222-4222-8222-222222222222" }; + let operations: Awaited> | undefined; + try { + await writeFile(file, JSON.stringify(value)); + await expect(openEvalOperations(file, [cell], 0, env)).rejects.toThrow("projects differ"); + await expect(access(value.ledgerPath)).rejects.toMatchObject({ code: "ENOENT" }); + await mkdir(`${value.ledgerPath}.otlp`); + const receipt = path.join(`${value.ledgerPath}.otlp`, `${createHash("sha256").update("collision").digest("hex")}.jsonl`); + await writeFile(receipt, "retained-receipt"); + operations = await openEvalOperations(file, [cell], 0, { ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: value.projectId }); + await expect(operations.start(cell, "collision", 1)).rejects.toMatchObject({ code: "EEXIST" }); + await expect(operations.start(cell, "replacement", 1)).rejects.toThrow("attempt cap"); + await operations.close(); operations = undefined; + expect(await readFile(receipt, "utf8")).toBe("retained-receipt"); + expect(JSON.parse(await readFile(`${value.ledgerPath}.summary.json`, "utf8")).admittedAttempts) + .toEqual([{ executionId: cell.id, attemptId: "collision" }]); + } finally { await operations?.close(); await rm(root, { recursive: true, force: true }); } +}); +it.each([ + cell.id, + "feedback-persistence.runner-codex.local.complaint-and-suggestion", + "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", +])("routes %s through a shared connection without ambient provider bindings", async executionId => { + const selected = runnerMatrix.find(execution => execution.id === executionId)!; + const anthropic = selected.profile.credential === "ANTHROPIC_API_KEY"; + const admitted = manifest(); admitted.executionIds = [selected.id]; + admitted.models[0]!.provider = anthropic ? "anthropic" : "openai"; + admitted.models[0]!.price.provider = admitted.models[0]!.provider; + admitted.models[0]!.price.model = selected.profile.model; + if (anthropic) Object.assign(admitted.models[0]!.price, { cacheWrite5mUsdPerMillion: "2", cacheWrite1hUsdPerMillion: "2" }); + expect(validateEvalOperationsManifest(admitted, [selected], 0)).toBe(admitted); + vi.stubEnv("PAPERCLIP_E2E_GATEWAY_BASE_URL", "http://127.0.0.1:43123"); + vi.stubEnv("PAPERCLIP_E2E_GATEWAY_TOKEN", "ephemeral-fixture-token"); + const id = "11111111-1111-4111-8111-111111111111", grantId = "22222222-2222-4222-8222-222222222222"; + let connection: unknown, agent: Record | undefined, company: Record | undefined; + const api = { + async get() { return [{ id, driver: "local" }]; }, + async postSensitive(url: string, body: unknown) { if (url.endsWith("/ai-connections")) { connection = body; return { connectionId: id, grantId }; } return { id }; }, + async post(url: string, body: Record) { if (url.endsWith("/agents")) agent = body; if (url === "/api/companies") company = body; return { id, name: "fixture" }; }, + } as unknown as RunnerApi; + const fixtures = await setupLiveFixtures({ api, execution: selected, executionNonce: "fixture", workspacePath: "/tmp/fixture", credentials: { [selected.profile.credential]: "ephemeral-fixture-token" } }); + await fixtures.teardown(); + expect(createAiConnectionSchema.parse(connection)).toMatchObject({ ownership: "shared", apiKey: "ephemeral-fixture-token", routing: { kind: "gateway", protocol: anthropic ? "messages" : "responses", baseUrl: `http://127.0.0.1:43123${anthropic ? "" : "/v1"}`, auth: "bearer" } }); + expect(aiConnectionBindingSchema.parse(agent!.runtimeConfig.aiConnection)).toEqual({ provider: anthropic ? "anthropic" : "openai", method: "api_key", mode: "shared", connectionId: id, grantId }); + expect(agent!.adapterConfig.env).toBeUndefined(); + if (selected.suite.id === "feedback-persistence") { + expect(company!.budgetMonthlyCents).toBe(1_000); + expect(agent!.budgetMonthlyCents).toBe(1_000); + } +}); diff --git a/tests/runner-e2e/operations.ts b/tests/runner-e2e/operations.ts new file mode 100644 index 00000000000..2ab81eb2676 --- /dev/null +++ b/tests/runner-e2e/operations.ts @@ -0,0 +1,123 @@ +import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { createEvalAttemptPhase, readEvalAttemptPhase } from "./attempt-phase.js"; +import { EvalSpendState, openEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; +import { evalRequestReservation, startEvalGateway, type EvalGatewayRoute, type EvalGatewayReceipt } from "./eval-gateway.js"; +import type { MatrixExecution } from "./types.js"; +import { evalBraintrustConfig } from "./braintrust.js"; +import { startEvalOtlpRelay } from "./eval-otlp-relay.js"; +import { usdToUnits } from "../../packages/shared/src/money.js"; + +export interface EvalOperationsManifest { + version: 1; + limits: EvalSpendLimits; + ledgerPath: string; + executionIds: string[]; + maxAttemptsPerExecution: number; + route: "direct" | "braintrust"; + projectId?: string; + models: Array>; +} +const fields = (value: object, names: string[]) => { + if (!value || typeof value !== "object" || Array.isArray(value) || Object.keys(value).some(name => !names.includes(name))) throw new Error("Invalid eval operations manifest fields"); +}; +export function validateEvalOperationsManifest(value: EvalOperationsManifest, executions: readonly MatrixExecution[], maxRetries: number) { + fields(value, ["version", "limits", "ledgerPath", "executionIds", "maxAttemptsPerExecution", "route", "projectId", "models"]); + const initial = new EvalSpendState(value.limits).snapshot(); + const initialModelAllowance = usdToUnits(value.limits.maxUsd) - usdToUnits(initial.committedUsd); + if (value.version !== 1 || !["direct", "braintrust"].includes(value.route) || !path.isAbsolute(value.ledgerPath) + || !Array.isArray(value.executionIds) || new Set(value.executionIds).size !== value.executionIds.length + || JSON.stringify([...value.executionIds].sort()) !== JSON.stringify(executions.map(e => e.id).sort()) + || !Number.isSafeInteger(value.maxAttemptsPerExecution) || value.maxAttemptsPerExecution < 1 + || value.limits.maxAttemptsPerExecution !== undefined && value.limits.maxAttemptsPerExecution !== value.maxAttemptsPerExecution + || maxRetries + 1 > value.maxAttemptsPerExecution || !Array.isArray(value.models) || value.models.length === 0) throw new Error("Eval operations selection or limits mismatch"); + if (value.route === "braintrust" && !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(value.projectId ?? "")) throw new Error("Braintrust gateway requires an explicit project UUID"); + const models = new Map(); + for (const model of value.models) { + fields(model, ["provider", "price", "maximumInputTokens", "maximumOutputTokens", "timeoutMs"]); + const rateFields = ["inputUsdPerMillion", "cachedReadUsdPerMillion", "outputUsdPerMillion", "cacheWrite5mUsdPerMillion", "cacheWrite1hUsdPerMillion", "cacheWrite30mUsdPerMillion", "cacheWrite12hUsdPerMillion"]; + fields(model.price, ["provider", "model", "version", "sourceUrl", "longContext", ...rateFields]); + if (model.price.longContext) { fields(model.price.longContext, ["aboveInputTokens", "rates"]); fields(model.price.longContext.rates, rateFields); } + const url = new URL(model.price.sourceUrl); + if (!["openai", "anthropic"].includes(model.provider) || url.protocol !== "https:" || url.username || url.password || url.search || url.hash) throw new Error("Invalid model price provenance"); + const key = `${model.provider}:${model.price.model}`; + if (models.has(key)) throw new Error("Duplicate eval model price"); + models.set(key, usdToUnits(evalRequestReservation({ ...model, mode: value.route, apiKey: "validation" }))); + } + for (const execution of executions) { + if (execution.environment.id !== "local" || !["core-compatibility", "feedback-persistence"].includes(execution.suite.id) + || !["runner-codex", "runner-acpx-claude", "legacy-codex", "legacy-claude"].includes(execution.profile.id) + || execution.requiredCredentials.length !== 1) throw new Error(`Unqualified budgeted eval route: ${execution.id}`); + const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai"; + if (!models.has(`${provider}:${execution.profile.model}`)) throw new Error(`Missing exact model pricing: ${execution.id}`); + if (models.get(`${provider}:${execution.profile.model}`)! > initialModelAllowance) { + throw new Error(`Insufficient initial model allowance for selected eval: ${execution.id}`); + } + } + return value; +} + +export async function openEvalOperations(file: string, executions: readonly MatrixExecution[], maxRetries: number, env: NodeJS.ProcessEnv) { + const manifest = validateEvalOperationsManifest(JSON.parse(await readFile(file, "utf8")), executions, maxRetries); + if (Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval campaign deadline reached"); + if (manifest.limits.overhead && (Date.now() < Date.parse(manifest.limits.overhead.verifiedAt) + || Date.parse(manifest.limits.deadlineAt) >= Date.parse(manifest.limits.overhead.expiresAt))) { + throw new Error("Operating overhead allowance does not cover the campaign window"); + } + const required = manifest.route === "braintrust" ? ["BRAINTRUST_API_KEY"] : executions.map(execution => execution.profile.credential); + if (required.some(name => !env[name]?.trim())) throw new Error("Missing eval gateway upstream credential"); + const observability = evalBraintrustConfig({ ...env, PAPERCLIP_E2E_OTLP_RELAY_URL: undefined }); + if (observability && manifest.route === "braintrust" && observability.projectId !== manifest.projectId) throw new Error("Eval gateway and OTLP projects differ"); + const receiptDirectory = `${manifest.ledgerPath}.requests`; + await mkdir(receiptDirectory, { recursive: true, mode: 0o700 }); + const phaseDirectory = `${manifest.ledgerPath}.attempts`; + await mkdir(phaseDirectory, { recursive: true, mode: 0o700 }); + const ledger = await openEvalCostLedger(manifest.ledgerPath, { ...manifest.limits, maxAttemptsPerExecution: manifest.maxAttemptsPerExecution }); + const active = new Set<{ close(): Promise }>(); + const atomicJson = async (file: string, value: unknown) => { + await writeFile(`${file}.tmp`, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); + await rename(`${file}.tmp`, file); + }; + return { + manifest, + async start(execution: MatrixExecution, attemptId: string, attempt: number) { + if (!Number.isSafeInteger(attempt) || attempt < 1 || attempt > manifest.maxAttemptsPerExecution || Date.now() >= Date.parse(manifest.limits.deadlineAt)) throw new Error("Eval operations attempt/deadline limit reached"); + if (!manifest.executionIds.includes(execution.id)) throw new Error("Eval operation is outside the admitted selection"); + const provider = execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai"; + const model = manifest.models.find(m => m.provider === provider && m.price.model === execution.profile.model)!; + const route: EvalGatewayRoute = { ...model, mode: manifest.route, projectId: manifest.projectId, + endpointName: provider === "openai" ? "OPENAI_API_KEY" : "ANTHROPIC_API_KEY", + apiKey: env[manifest.route === "braintrust" ? "BRAINTRUST_API_KEY" : execution.profile.credential] ?? "" }; + if (!route.apiKey) throw new Error("Missing eval gateway upstream credential"); + await ledger.admitAttempt(execution.id, attemptId); + const identity = { executionId: execution.id, attemptId, phase: "setup" as const }; + const phasePath = path.join(phaseDirectory, `${createHash("sha256").update(attemptId).digest("hex")}.json`); + await createEvalAttemptPhase(phasePath, { executionId: execution.id, attemptId }); + const gateway = await startEvalGateway({ ledger, route, + identity, resolveIdentity: () => readEvalAttemptPhase(phasePath, identity), deadlineAt: Date.parse(manifest.limits.deadlineAt), + persist: (receipt: EvalGatewayReceipt) => atomicJson(path.join(receiptDirectory, `${receipt.requestId}.json`), receipt), + }); + let relay: Awaited> | undefined; + try { + if (observability) relay = await startEvalOtlpRelay({ projectId: observability.projectId, apiKey: observability.apiKey, + campaignId: manifest.limits.campaignId, attemptId, deadlineAt: Math.min(Date.parse(manifest.limits.deadlineAt), Date.now() + 4 * 3_600_000), + receiptPath: path.join(`${manifest.ledgerPath}.otlp`, `${createHash("sha256").update(attemptId).digest("hex")}.jsonl`) }); + } catch (error) { await gateway.close(); throw error; } + let closing: Promise | undefined; + const endpoint = { ...gateway, phasePath, observability: relay, close: () => closing ??= (async () => { + const results = await Promise.allSettled([gateway.close(), relay?.close()]); + active.delete(endpoint); + if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway or OTLP relay cleanup failed"); + })() }; + active.add(endpoint); + return endpoint; + }, + async close() { + const results = await Promise.allSettled([...active].map(gateway => gateway.close())); active.clear(); + try { await atomicJson(`${manifest.ledgerPath}.summary.json`, ledger.snapshot()); } finally { await ledger.close(); } + if (results.some(result => result.status === "rejected")) throw new Error("Eval gateway cleanup failed"); + }, + }; +} +export type EvalOperations = Awaited>; diff --git a/tests/runner-e2e/paid-runner-routing.test.mjs b/tests/runner-e2e/paid-runner-routing.test.mjs new file mode 100644 index 00000000000..669ade64813 --- /dev/null +++ b/tests/runner-e2e/paid-runner-routing.test.mjs @@ -0,0 +1,51 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { expect, it } from "vitest"; + +const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8"); +const step = workflow.split(" - name: Select paid test runner\n")[1]?.split("\n target_lock:")[0]; +if (!step?.includes(" run: |\n")) throw new Error("Missing actual workflow runner selection"); +const script = step.split(" run: |\n")[1].split("\n").map(line => line.replace(/^ /, "")).join("\n"); + +async function select(overrides) { + const root = await mkdtemp(path.join(tmpdir(), "eval-runner-routing-")); + try { + const output = path.join(root, "outputs"); + const result = spawnSync("bash", ["-c", script], { encoding: "utf8", timeout: 5000, env: { + PATH: process.env.PATH, GITHUB_OUTPUT: output, AWS_PAID_RUNNER_ENABLED: "true", + EVENT_NAME: "workflow_dispatch", EVAL_CADENCE: "manual", REPOSITORY_PRIVATE: "false", ...overrides, + } }); + const saved = await readFile(output, "utf8").catch(error => { if (error.code === "ENOENT") return ""; throw error; }); + return { ...result, outputs: Object.fromEntries(saved.trim().split("\n").filter(Boolean).map(line => { + const separator = line.indexOf("="); return [line.slice(0, separator), line.slice(separator + 1)]; + })) }; + } finally { await rm(root, { recursive: true, force: true }); } +} + +it.each([ + { EVENT_NAME: "schedule" }, + { EVAL_CADENCE: "daily" }, + { EVAL_CADENCE: "weekly" }, +])("routes bounded $EVENT_NAME/$EVAL_CADENCE campaigns to standard hosted compute despite the AWS switch", async trigger => { + const result = await select(trigger); + expect(result.status).toBe(0); + expect(result.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" }); +}); + +it.each(["true", "", "unknown"])("refuses bounded campaigns when repository privacy is %s", async REPOSITORY_PRIVATE => { + const result = await select({ EVAL_CADENCE: "daily", REPOSITORY_PRIVATE }); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain("require a public repository"); + expect(result.outputs).toEqual({}); +}); + +it("preserves the explicit manual AWS and GitHub options", async () => { + const aws = await select({ REPOSITORY_PRIVATE: "true" }); + expect(aws.status).toBe(0); + expect(aws.outputs).toMatchObject({ runner: "runs-on/fleet=paperclip-public-pr-x64/env=public-ci", playwright_channel: "chrome" }); + const hosted = await select({ AWS_PAID_RUNNER_ENABLED: "false" }); + expect(hosted.status).toBe(0); + expect(hosted.outputs).toMatchObject({ runner: "ubuntu-latest", playwright_channel: "" }); +}); diff --git a/tests/runner-e2e/pi-controls-evidence.test.ts b/tests/runner-e2e/pi-controls-evidence.test.ts index 599d8c6fc25..bae48344569 100644 --- a/tests/runner-e2e/pi-controls-evidence.test.ts +++ b/tests/runner-e2e/pi-controls-evidence.test.ts @@ -180,7 +180,7 @@ describe("Pi controls catalog admission", () => { expect(cells).toHaveLength(4); expect(cells.every(c => c.profile.qualificationCandidate === "pi" && c.task.expectedRunCount === 1)).toBe(true); expect(cells.map(c => `${c.environment.id}/${c.task.id}`).sort()).toEqual(["daytona/pending-permission-stop", "daytona/same-turn-steering", "local/pending-permission-stop", "local/same-turn-steering"]); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(c => c.suite.id === "pi-controls")).toBe(false); - expect(validateRunnerCatalog()).toHaveLength(761); + expect(validateRunnerCatalog()).toHaveLength(763); for (const cell of cells) expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_ACPX_QUALIFICATION).toBeUndefined(); expect(() => assertRemoteNativeEvidencePrerequisites(cells, {})).toThrow(); }); diff --git a/tests/runner-e2e/private-history-archive.py b/tests/runner-e2e/private-history-archive.py new file mode 100644 index 00000000000..02c3271234b --- /dev/null +++ b/tests/runner-e2e/private-history-archive.py @@ -0,0 +1,148 @@ +import json +import os +from pathlib import Path, PurePosixPath +import shutil +import stat +import struct +import sys +import tempfile +import zipfile + + +MAX_BYTES = 256 * 1024 * 1024 +MAX_ARCHIVE_BYTES = MAX_BYTES + 8 * 1024 * 1024 +MAX_FILES = 4097 + + +def safe_path(name): + parts = name.split("/") + if (not name or len(name) > 512 or "\\" in name or ":" in name + or any(ord(c) < 32 or ord(c) == 127 for c in name) + or any(p in ("", ".", "..") for p in parts)): + raise ValueError("Unsafe private archive path") + return PurePosixPath(name) + + +def pack(source, destination): + source, destination = Path(source), Path(destination) + if source.is_symlink() or not source.is_dir(): + raise ValueError("Private archive source must be a directory") + root = source.resolve() + if destination.resolve().is_relative_to(root): + raise ValueError("Private archive must be outside its source") + files = [] + names_seen = set() + total = 0 + visited = 0 + for directory, directories, names in os.walk(source, followlinks=False): + visited += len(directories) + len(names) + if visited > MAX_FILES: + raise ValueError("Private archive exceeds entry allowance") + for name in directories: + entry = Path(directory) / name + safe_path(entry.relative_to(source).as_posix()) + if entry.is_symlink(): + raise ValueError("Private archive cannot contain symlinks") + for name in names: + entry = Path(directory) / name + relative = entry.relative_to(source).as_posix() + safe_path(relative) + if relative.casefold() in names_seen: + raise ValueError("Private archive has colliding paths") + names_seen.add(relative.casefold()) + metadata = entry.lstat() + if not stat.S_ISREG(metadata.st_mode) or not entry.resolve().is_relative_to(root): + raise ValueError("Private archive requires regular files") + total += metadata.st_size + if total > MAX_BYTES: + raise ValueError("Private archive exceeds byte allowance") + files.append((entry, relative, metadata)) + if not files: + raise ValueError("Private archive is empty") + output = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(output, "wb") as stream, zipfile.ZipFile(stream, "w", zipfile.ZIP_STORED, allowZip64=False) as archive: + for entry, relative, before in sorted(files): + with os.fdopen(os.open(entry, os.O_RDONLY | os.O_NOFOLLOW), "rb") as data: + observed = os.fstat(data.fileno()) + if (observed.st_ino, observed.st_size, observed.st_mtime_ns) != (before.st_ino, before.st_size, before.st_mtime_ns): + raise ValueError("Private archive source changed") + info = zipfile.ZipInfo(relative) + info.external_attr = (stat.S_IFREG | 0o600) << 16 + copied = 0 + with archive.open(info, "w") as target: + while chunk := data.read(64 * 1024): + copied += len(chunk) + if copied > before.st_size: + raise ValueError("Private archive source grew") + target.write(chunk) + after = os.fstat(data.fileno()) + if copied != before.st_size or (after.st_size, after.st_mtime_ns, after.st_ctime_ns) != (before.st_size, before.st_mtime_ns, before.st_ctime_ns): + raise ValueError("Private archive source changed") + if destination.stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("Private archive exceeds transfer allowance") + except BaseException: + destination.unlink(missing_ok=True) + raise + return {"files": len(files), "sourceBytes": total, "archiveBytes": destination.stat().st_size} + + +def unpack(source, destination): + source, destination = Path(source), Path(destination) + if source.is_symlink() or not source.is_file() or source.stat().st_size > MAX_ARCHIVE_BYTES: + raise ValueError("Invalid private archive file") + if destination.exists() or destination.is_symlink(): + raise ValueError("Private archive destination already exists") + with source.open("rb") as data: + if source.stat().st_size < 22: + raise ValueError("Private archive is truncated") + data.seek(-22, os.SEEK_END) + signature, disk, start_disk, disk_entries, entries, central_bytes, central_offset, comment_bytes = struct.unpack("<4s4H2LH", data.read(22)) + if (signature != b"PK\x05\x06" or disk or start_disk or comment_bytes + or entries != disk_entries or not 0 < entries <= MAX_FILES + or central_bytes > 8 * 1024 * 1024 + or central_offset + central_bytes != source.stat().st_size - 22): + raise ValueError("Private archive directory exceeds its boundary") + destination.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=".private-eval-", dir=destination.parent)) + try: + with zipfile.ZipFile(source) as archive: + entries = archive.infolist() + if not entries or len(entries) != disk_entries or len(entries) > MAX_FILES: + raise ValueError("Private archive exceeds entry allowance") + paths = set() + total = 0 + for entry in entries: + relative = safe_path(entry.filename) + mode = entry.external_attr >> 16 + if (entry.compress_type != zipfile.ZIP_STORED or entry.flag_bits & 1 + or stat.S_IFMT(mode) not in (0, stat.S_IFREG) + or str(relative).casefold() in paths): + raise ValueError("Private archive has unsupported or duplicate entries") + paths.add(str(relative).casefold()) + total += entry.file_size + if total > MAX_BYTES: + raise ValueError("Private archive exceeds byte allowance") + for entry in entries: + target = temporary.joinpath(*safe_path(entry.filename).parts) + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + copied = 0 + with archive.open(entry) as data, os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output: + while chunk := data.read(64 * 1024): + copied += len(chunk) + if copied > entry.file_size: + raise ValueError("Private archive entry exceeded its declared size") + output.write(chunk) + if copied != entry.file_size: + raise ValueError("Private archive entry was truncated") + os.rename(temporary, destination) + except BaseException: + shutil.rmtree(temporary) + raise + return {"files": len(entries), "sourceBytes": total} + + +if __name__ == "__main__": + if len(sys.argv) != 4 or sys.argv[1] not in ("pack", "unpack"): + raise SystemExit("Usage: private-history-archive.py pack|unpack SOURCE DESTINATION") + print(json.dumps((pack if sys.argv[1] == "pack" else unpack)(sys.argv[2], sys.argv[3]))) diff --git a/tests/runner-e2e/private-history.test.ts b/tests/runner-e2e/private-history.test.ts new file mode 100644 index 00000000000..9af459a01b6 --- /dev/null +++ b/tests/runner-e2e/private-history.test.ts @@ -0,0 +1,162 @@ +import { execFile } from "node:child_process"; +import { copyFile, lstat, mkdir, mkdtemp, readFile, rm, symlink, truncate, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { afterEach, expect, it, vi } from "vitest"; +import { BoundedHistoryPublisher } from "./history-budget.js"; +import { archivePrivateEvidence, PRIVATE_HISTORY_PREFIX, privateArtifactKey, retainPrivateTransfer, transferPrivateArtifact, verifyPrivateHistory } from "./private-history.js"; + +const execute = promisify(execFile); +const roots: string[] = []; +async function temporary() { + const root = await mkdtemp(path.join(os.tmpdir(), "private-history-test-")); roots.push(root); return root; +} +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); + +function configurations() { + return { + "get-bucket-versioning": { Status: "Enabled" }, + "get-bucket-lifecycle-configuration": { Rules: [ + { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { Days: 30 }, + NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } }, + { Status: "Enabled", Filter: { Prefix: `${PRIVATE_HISTORY_PREFIX}/` }, Expiration: { ExpiredObjectDeleteMarker: true } }, + ] }, + "get-public-access-block": { PublicAccessBlockConfiguration: { BlockPublicAcls: true, IgnorePublicAcls: true, BlockPublicPolicy: true, RestrictPublicBuckets: true } }, + "get-bucket-ownership-controls": { OwnershipControls: { Rules: [{ ObjectOwnership: "BucketOwnerEnforced" }] } }, + "get-bucket-policy": { Policy: JSON.stringify({ Statement: [{ Effect: "Deny" }] }) }, + }; +} +function configurationRunner(config: Record) { + return async (args: string[]) => { + if (args[1] === "get-object-lock-configuration" || args[1] === "get-bucket-replication") { + throw Object.assign(new Error("Not configured"), { stderr: `(${args[1] === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError"})` }); + } + if (!(args[1]! in config)) throw new Error(`Unexpected AWS operation ${args[1]}`); + return { stdout: JSON.stringify(config[args[1]!]) }; + }; +} + +it("round trips binary evidence and a ledger without exposing unrelated files", async () => { + const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip"), destination = path.join(root, "restored"); + await mkdir(path.join(source, "scheduled"), { recursive: true }); + await writeFile(path.join(source, "scheduled/ledger.jsonl"), '{"cost":"unknown"}\n'); + await writeFile(path.join(source, "binary.png"), Buffer.from([0, 255, 17, 0])); + await writeFile(path.join(root, "outside"), "private outside material"); + expect(await archivePrivateEvidence("pack", source, archive)).toMatchObject({ files: 2 }); + expect(await archivePrivateEvidence("unpack", archive, destination)).toMatchObject({ files: 2 }); + expect(await readFile(path.join(destination, "binary.png"))).toEqual(Buffer.from([0, 255, 17, 0])); + expect(await readFile(path.join(destination, "scheduled/ledger.jsonl"), "utf8")).toBe('{"cost":"unknown"}\n'); + expect((await lstat(path.join(destination, "binary.png"))).mode & 0o777).toBe(0o600); + await expect(lstat(path.join(destination, "outside"))).rejects.toThrow(); + await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow("already exists"); +}); + +it("rejects symlink sources and oversized source files before producing an archive", async () => { + const root = await temporary(), source = path.join(root, "source"), archive = path.join(root, "evidence.zip"); + await mkdir(source); await writeFile(path.join(root, "outside"), "outside"); + await symlink(path.join(root, "outside"), path.join(source, "link")); + await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("regular files"); + await rm(path.join(source, "link")); await writeFile(path.join(source, "huge"), ""); + await truncate(path.join(source, "huge"), 256 * 1024 * 1024 + 1); + await expect(archivePrivateEvidence("pack", source, archive)).rejects.toThrow("byte allowance"); + await expect(lstat(archive)).rejects.toThrow(); +}); + +it.each(["traversal", "absolute", "symlink", "compressed", "duplicate", "case_collision", "comment", "many_entries", "truncated"])("rejects %s archives without retaining a partial extraction", async kind => { + const root = await temporary(), archive = path.join(root, "bad.zip"), destination = path.join(root, "restored"); + await execute("python3", ["-c", ` +import sys,zipfile,stat +p,kind=sys.argv[1:] +with zipfile.ZipFile(p,'w') as z: + name={'traversal':'../outside','absolute':'/tmp/outside'}.get(kind,'safe.txt') + info=zipfile.ZipInfo(name) + if kind=='symlink': info.external_attr=(stat.S_IFLNK|0o777)<<16 + if kind=='compressed': info.compress_type=zipfile.ZIP_DEFLATED + z.writestr(info,'data') + if kind=='duplicate': z.writestr(name,'other') + if kind=='case_collision': z.writestr('SAFE.txt','other') + if kind=='comment': z.comment=b'extra' + if kind=='many_entries': + for n in range(4097): z.writestr(str(n),'') +if kind=='truncated': + with open(p,'r+b') as f: f.truncate(12) +`, archive, kind]); + await expect(archivePrivateEvidence("unpack", archive, destination)).rejects.toThrow(); + await expect(lstat(destination)).rejects.toThrow(); + await expect(lstat(path.join(root, "outside"))).rejects.toThrow(); +}); + +it("fails closed on public access, ACLs, a reader grant, or unavailable inspection", async () => { + const root = await temporary(); + await verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "allowed.json"), configurationRunner(configurations())), "private-bucket"); + for (const operation of ["get-public-access-block", "get-bucket-ownership-controls", "get-bucket-policy"]) { + const config: Record = configurations(); + config[operation] = operation === "get-bucket-policy" ? { Policy: '{"Statement":[{"Effect":"Allow"}]}' } : {}; + const runner = vi.fn(configurationRunner(config)); + await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, `${operation}.json`), runner), "private-bucket")).rejects.toThrow(); + expect(runner.mock.calls.some(([args]) => args[1] === "put-object")).toBe(false); + } + await expect(verifyPrivateHistory(new BoundedHistoryPublisher(path.join(root, "denied.json"), async () => { throw new Error("AccessDenied"); }), "private-bucket")).rejects.toThrow("AccessDenied"); +}); + +it("uses immutable private keys and restores the uploaded archive through the actual transfer path", async () => { + const root = await temporary(), source = path.join(root, "source"), stored = path.join(root, "stored.zip"), destination = path.join(root, "restored"); + await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "retained attempt\n"); + const config = configurationRunner(configurations()); + const runner = vi.fn(async (args: string[]) => { + if (args[1] === "put-object") { + expect(args).toEqual(expect.arrayContaining(["--if-none-match", "*", "--cache-control", "private, no-store"])); + expect(args[args.indexOf("--key") + 1]).toBe(privateArtifactKey("evidence", "12345")); + await copyFile(args[args.indexOf("--body") + 1]!, stored); + return { stdout: '{"VersionId":"retained-version","ETag":"tag"}' }; + } + if (args[1] === "get-object") { + expect(args).toContain(`bytes=0-${264 * 1024 * 1024}`); + await copyFile(stored, args[args.indexOf("--cli-connect-timeout") - 1]!); + return { stdout: '{"VersionId":"retained-version"}' }; + } + return config(args); + }); + const upload = new BoundedHistoryPublisher(path.join(root, "upload.json"), runner); + expect(await transferPrivateArtifact({ operation: "upload", kind: "evidence", runId: "12345", directory: source, bucket: "private-bucket", publisher: upload })) + .toMatchObject({ private: true, versionId: "retained-version", files: 1, sha256: expect.stringMatching(/^[0-9a-f]{64}$/) }); + await transferPrivateArtifact({ operation: "download", kind: "evidence", runId: "12345", directory: destination, bucket: "private-bucket", publisher: new BoundedHistoryPublisher(path.join(root, "download.json"), runner) }); + expect(await readFile(path.join(destination, "ledger.jsonl"), "utf8")).toBe("retained attempt\n"); + expect(runner).toHaveBeenCalledTimes(16); +}); + +it("rejects unsafe run identities and other artifact kinds", () => { + for (const id of ["../123", "0", "1/2", "123-2"]) expect(() => privateArtifactKey("report", id)).toThrow(); + expect(() => privateArtifactKey("public", "123")).toThrow(); +}); + +it.each([false, true])("retains an unknown transfer without retrying it when receipt upload fails=%s", async receiptFails => { + const root = await temporary(), source = path.join(root, "source"), receipt = path.join(root, "transfer.json"); + await mkdir(source); await writeFile(path.join(source, "ledger.jsonl"), "original evidence\n"); + const config = configurationRunner(configurations()); + const runner = vi.fn(async (args: string[]) => { + if (args[1] !== "put-object") return config(args); + await copyFile(args[args.indexOf("--body") + 1]!, path.join(root, "accepted.zip")); + throw new Error("Response lost after server accepted archive"); + }); + const retain = vi.fn(async (args: string[]) => { + expect(args[args.indexOf("--key") + 1]).toBe(`${privateArtifactKey("evidence", "12345")}.upload-receipt.json`); + const snapshot = JSON.parse(await readFile(args[args.indexOf("--body") + 1]!, "utf8")); + expect(snapshot.transfer.status).toBe("failed_or_unknown"); + expect(snapshot.requests.at(-1)).toMatchObject({ operation: "put-object", status: "failed_or_unknown" }); + expect(snapshot.billedUsd).toBeNull(); + expect(snapshot.receiptUpload.status).toBe("pending_at_snapshot"); + if (receiptFails) throw new Error("Receipt response also lost"); + return { stdout: '{"VersionId":"receipt-version"}' }; + }); + await expect(retainPrivateTransfer({ operation: "upload", kind: "evidence", runId: "12345", directory: source, + bucket: "private-bucket", publisher: new BoundedHistoryPublisher(receipt, runner) }, receipt, + () => new BoundedHistoryPublisher(`${receipt}.retention.json`, retain))) + .rejects.toThrow(receiptFails ? "receipt retention failed" : "Response lost"); + expect(runner.mock.calls.filter(([args]) => args[1] === "put-object")).toHaveLength(1); + expect(retain).toHaveBeenCalledTimes(1); + expect((await lstat(`${receipt}.transfer.json`)).mode & 0o777).toBe(0o600); + expect(JSON.parse(await readFile(`${receipt}.retention.json`, "utf8")).requests[0].status) + .toBe(receiptFails ? "failed_or_unknown" : "accepted"); +}); diff --git a/tests/runner-e2e/private-history.ts b/tests/runner-e2e/private-history.ts new file mode 100644 index 00000000000..85224bf997d --- /dev/null +++ b/tests/runner-e2e/private-history.ts @@ -0,0 +1,142 @@ +import { execFile } from "node:child_process"; +import { createReadStream } from "node:fs"; +import { createHash } from "node:crypto"; +import { lstat, mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { pathToFileURL } from "node:url"; +import { BoundedHistoryPublisher } from "./history-budget.js"; + +const execFileAsync = promisify(execFile); +export const PRIVATE_HISTORY_PREFIX = "runner-e2e-private/bounded-v1"; +export const PRIVATE_ARCHIVE_BYTES = 264 * 1024 * 1024; + +export function privateArtifactKey(kind: string, runId: string) { + if (!["evidence", "report"].includes(kind) || !/^[1-9][0-9]{0,19}$/.test(runId)) { + throw new Error("Private history needs an evidence/report kind and numeric workflow run ID"); + } + return `${PRIVATE_HISTORY_PREFIX}/campaigns/gha-${runId}-1/${kind}.zip`; +} + +export async function archivePrivateEvidence(operation: "pack" | "unpack", source: string, destination: string) { + const result = await execFileAsync("python3", [path.join(import.meta.dirname, "private-history-archive.py"), operation, source, destination], { + timeout: 60_000, maxBuffer: 64 * 1024, + }); + return JSON.parse(result.stdout) as { files: number; sourceBytes: number; archiveBytes?: number }; +} + +export async function verifyPrivateHistory(publisher: BoundedHistoryPublisher, bucket: string) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(bucket)) throw new Error("Invalid private history bucket"); + await publisher.verifyRetention(bucket, PRIVATE_HISTORY_PREFIX); + const configuration = async (operation: string) => JSON.parse((await publisher.run(["s3api", operation, "--bucket", bucket])).stdout); + const block = (await configuration("get-public-access-block")).PublicAccessBlockConfiguration; + if (!["BlockPublicAcls", "IgnorePublicAcls", "BlockPublicPolicy", "RestrictPublicBuckets"].every(key => block?.[key] === true)) { + throw new Error("Private history requires all bucket public-access blocks"); + } + const ownership = (await configuration("get-bucket-ownership-controls")).OwnershipControls?.Rules; + if (!Array.isArray(ownership) || ownership.length !== 1 || ownership[0]?.ObjectOwnership !== "BucketOwnerEnforced") { + throw new Error("Private history requires disabled object ACLs"); + } + const policy = JSON.parse((await configuration("get-bucket-policy")).Policy); + if (!Array.isArray(policy.Statement) || policy.Statement.some((statement: { Effect?: string }) => statement.Effect !== "Deny")) { + throw new Error("Private history bucket policy must not grant access to external readers or a CDN"); + } +} + +type TransferOptions = { + operation: "upload" | "download"; + kind: "evidence" | "report"; + runId: string; + directory: string; + bucket: string; + publisher: BoundedHistoryPublisher; +}; + +export async function transferPrivateArtifact(options: TransferOptions) { + privateArtifactKey(options.kind, options.runId); + await verifyPrivateHistory(options.publisher, options.bucket); + return transferVerifiedArtifact(options); +} + +async function transferVerifiedArtifact(options: TransferOptions) { + const key = privateArtifactKey(options.kind, options.runId); + const temporary = await mkdtemp(path.join(os.tmpdir(), "paperclip-private-history-")); + const archive = path.join(temporary, "archive.zip"); + try { + if (options.operation === "upload") { + const metadata = await archivePrivateEvidence("pack", options.directory, archive); + const hash = createHash("sha256"); + for await (const chunk of createReadStream(archive)) hash.update(chunk); + const result = await options.publisher.upload(options.bucket, key, archive, "application/zip", "private, no-store", true); + const response = JSON.parse(result.stdout) as { VersionId?: string }; + if (!response.VersionId) throw new Error("Private archive upload did not return a version identity"); + return { private: true, ...metadata, uri: `s3://${options.bucket}/${key}`, versionId: response.VersionId, sha256: hash.digest("hex") }; + } + await options.publisher.run(["s3api", "get-object", "--bucket", options.bucket, "--key", key, + "--range", `bytes=0-${PRIVATE_ARCHIVE_BYTES}`, archive]); + if ((await lstat(archive)).size > PRIVATE_ARCHIVE_BYTES) throw new Error("Private archive exceeds transfer allowance"); + const metadata = await archivePrivateEvidence("unpack", archive, options.directory); + return { ...metadata, uri: `s3://${options.bucket}/${key}` }; + } finally { + await rm(temporary, { recursive: true, force: true }); + } +} + +export async function retainPrivateTransfer(options: TransferOptions, receipt: string, receiptPublisher?: () => BoundedHistoryPublisher) { + const key = privateArtifactKey(options.kind, options.runId); + await verifyPrivateHistory(options.publisher, options.bucket); + let result: Awaited> | undefined; + let failure: unknown; + try { + result = await transferVerifiedArtifact(options); + } catch (error) { + failure = error; + } + if (receiptPublisher) { + await options.publisher.save(); + const snapshot = JSON.stringify({ ...JSON.parse(await readFile(receipt, "utf8")), + transfer: result ? { status: "complete", ...result } : { status: "failed_or_unknown" }, + receiptUpload: { status: "pending_at_snapshot", maxBytes: 1024 * 1024 } }); + if (Buffer.byteLength(snapshot) > 1024 * 1024) throw new Error("Private transfer receipt exceeds allowance"); + await writeFile(`${receipt}.transfer.json`, snapshot, { mode: 0o600, flag: "wx" }); + try { + await receiptPublisher().upload(options.bucket, `${key}.${options.operation}-receipt.json`, `${receipt}.transfer.json`, "application/json", "private, no-store", true); + } catch (retentionError) { + throw new AggregateError(failure ? [failure, retentionError] : [retentionError], "Private transfer receipt retention failed; local receipts remain authoritative"); + } + } + if (failure) throw failure; + return result!; +} + +async function main() { + const [operation, kind, runId, directory] = process.argv.slice(2); + if (!["verify", "upload", "download"].includes(operation ?? "") || !kind || !runId || !directory) { + throw new Error("Usage: private-history.ts verify|upload|download evidence|report RUN_ID DIRECTORY"); + } + privateArtifactKey(kind, runId); + if (process.env.GITHUB_ACTIONS === "true" && (process.env.GITHUB_RUN_ATTEMPT !== "1" || process.env.GITHUB_RUN_ID !== runId)) { + throw new Error("Private bounded history accepts only the initial attempt of this workflow run"); + } + const bucket = process.env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? ""; + const receipt = path.resolve(`private-history-${operation}-${kind}-${runId}.json`); + await (await open(receipt, "wx", 0o600)).close(); + const publisher = new BoundedHistoryPublisher(receipt); + if (operation === "verify") { + await verifyPrivateHistory(publisher, bucket); + console.log("Private retention and access checks passed"); + return; + } + const receiptPublisher = operation === "upload" || process.env.GITHUB_ACTIONS === "true" + ? () => new BoundedHistoryPublisher(`${receipt}.retention.json`) : undefined; + const result = await retainPrivateTransfer({ operation: operation as "upload" | "download", kind: kind as "evidence" | "report", runId, directory, bucket, publisher }, receipt, receiptPublisher); + console.log(JSON.stringify(process.env.GITHUB_ACTIONS === "true" ? { version: 1, privateTransferCompleted: true } : result)); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().catch(error => { + console.error(process.env.GITHUB_ACTIONS === "true" ? "Private history transfer failed; inspect private retained evidence." : error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/process-tree-owner.ts b/tests/runner-e2e/process-tree-owner.ts index 86b44902bcc..3dcae17791f 100644 --- a/tests/runner-e2e/process-tree-owner.ts +++ b/tests/runner-e2e/process-tree-owner.ts @@ -177,13 +177,13 @@ export function createProcessTreeOwner(root: ChildProcess, options: { } let graceComplete = false; let directGraceDelivered = false; - async function signalGracefully() { + async function signalGracefully(signal: "SIGTERM" | "SIGINT" = "SIGTERM") { if (graceComplete) return; await observe(); // Choose and deliver from one validated table. ESRCH grants no ownership // coverage: the next grace poll can select that branch's surviving owner. const selected = new Set([...gracefulRoots()].filter(pid => !covered.has(pid))); - const delivered = signalSnapshot("SIGTERM", selected); + const delivered = signalSnapshot(signal, selected); for (const pid of delivered) { if (pid === root.pid) directGraceDelivered = true; covered.add(pid); @@ -220,9 +220,10 @@ export async function stopKnownDirectChild(child: ChildProcess, phase: { gracefu } export async function incompleteTreeFallback(error: unknown, child: ChildProcess, - phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number) { + phase: { gracefulSent: boolean; forcedDeadline?: number }, deadline: number, forcedMs: number, + signal: "SIGTERM" | "SIGINT" = "SIGTERM") { let fallbackError: unknown; - try { await stopKnownDirectChild(child, phase, deadline, forcedMs); } + try { await stopKnownDirectChild(child, phase, deadline, forcedMs, signal); } catch (failure) { fallbackError = failure; } throw new Error(`Owned tree cleanup incomplete: ${String(error)}${fallbackError ? `; ${String(fallbackError)}` : ""}`); } @@ -234,13 +235,14 @@ export async function stopOwnedProcessTree( owner: ReturnType, gracefulMs = 45_000, forcedMs = 5_000, + gracefulSignal: "SIGTERM" | "SIGINT" = "SIGTERM", ): Promise { const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms)); const deadline = Date.now() + gracefulMs; const phase: { gracefulSent: boolean; forcedDeadline?: number } = { gracefulSent: false }; try { if (process.platform === "win32") { - await stopKnownDirectChild(child, phase, deadline, forcedMs); + await stopKnownDirectChild(child, phase, deadline, forcedMs, gracefulSignal); return; } const stopped = async () => { @@ -248,7 +250,7 @@ export async function stopOwnedProcessTree( return (child.exitCode !== null || child.signalCode !== null) && owner.liveGroups().length === 0; }; do { - await owner.signalGracefully(); + await owner.signalGracefully(gracefulSignal); phase.gracefulSent ||= owner.directGraceDelivered(); if (await stopped()) return; await wait(50); @@ -262,6 +264,6 @@ export async function stopOwnedProcessTree( throw new Error("Observed launcher descendants remained after bounded cleanup"); } catch (error) { phase.gracefulSent ||= owner.directGraceDelivered(); - await incompleteTreeFallback(error, child, phase, deadline, forcedMs); + await incompleteTreeFallback(error, child, phase, deadline, forcedMs, gracefulSignal); } } diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index 82762c533fc..5e90e3c2e23 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -143,6 +143,8 @@ describe("deployment journey contracts", () => { onboarding.runIds=["run-1"]; onboarding.usage={runs:[{runId:"run-1",usage:{costUsd:0.5,inputTokens:10,outputTokens:10}}]}; onboarding.productJourney!.operations={campaignId:"campaign",attemptId:"attempt",modelCostUsd:0.7,setupCostUsd:0.2,requestCount:2,feedback:{collection:"complete",complaints:0,suggestions:1},traceIds:[]}; expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7); + onboarding.usage={runs:[{runId:"run-1",usage:{costUsdExact:"0.5",costStatus:"estimated",inputTokens:10,outputTokens:10}}]}; + expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeCloseTo(0.7); expect(()=>validateJourneyCollection({journeyId:"cloud-onboarding",runIds:["run"],usage:{runs:[{runId:"run",usage:{apiKey:"not-retainable"} as any}]}})).toThrow("invalid_collection_receipt"); onboarding.productJourney!.operations.modelCostUsd=null; expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull(); diff --git a/tests/runner-e2e/provider-usage.test.ts b/tests/runner-e2e/provider-usage.test.ts new file mode 100644 index 00000000000..b651c9e8a89 --- /dev/null +++ b/tests/runner-e2e/provider-usage.test.ts @@ -0,0 +1,113 @@ +import { expect, it } from "vitest"; +import { estimateProviderCost, normalizeProviderUsage, ProviderUsageAccumulator, type EvalModelPrice } from "./provider-usage.js"; +const price: EvalModelPrice = { provider: "openai", model: "fixture", version: "fixture-1", sourceUrl: "https://provider.test/prices", inputUsdPerMillion: "2.5", cachedReadUsdPerMillion: "0.25", outputUsdPerMillion: "10", cacheWrite30mUsdPerMillion: "2", cacheWrite12hUsdPerMillion: "2" }; + +it("excludes cached reads from OpenAI input and treats reasoning as a subset of output", () => { + const usage = normalizeProviderUsage("openai", { id: "response", model: "fixture", usage: { input_tokens: 100, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 40 }, output_tokens: 20, output_tokens_details: { reasoning_tokens: 15 } } }); + expect(usage).toMatchObject({ complete: true, tokens: { input: 60, cachedRead: 40, cachedWrite: 0, output: 20, reasoning: 15 } }); + expect(estimateProviderCost(usage, price)).toBe("0.000360000"); +}); +it("never turns missing usage or incompatible model/pricing into a free charge", () => { + const absent = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, output_tokens: 0 } }); + expect(absent.complete).toBe(false); + expect(estimateProviderCost(absent, price)).toBeNull(); + const zero = normalizeProviderUsage("openai", { model: "fixture", usage: { input_tokens: 0, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 0 } }); + expect(estimateProviderCost(zero, price)).toBe("0.000000000"); + expect(estimateProviderCost(zero, { ...price, model: "other" })).toBeNull(); +}); +it("does not add Anthropic cumulative output counters and prices cache writes by TTL", () => { + const stream = new ProviderUsageAccumulator("anthropic"); + stream.observe({ type: "message_start", message: { id: "message", model: "fixture", usage: { input_tokens: 10, output_tokens: 1, cache_read_input_tokens: 20, cache_creation_input_tokens: 30, cache_creation: { ephemeral_5m_input_tokens: 20, ephemeral_1h_input_tokens: 10 } } } }); + stream.observe({ type: "message_delta", usage: { output_tokens: 5 } }); + stream.observe({ type: "message_delta", usage: { output_tokens: 8 } }); + expect(stream.receipt().final).toBe(false); + stream.observe({ type: "message_stop" }); + expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 10, cachedRead: 20, cachedWrite: 30, output: 8 } }); + expect(estimateProviderCost(stream.receipt(), { ...price, provider: "anthropic", cacheWrite5mUsdPerMillion: "3.125", cacheWrite1hUsdPerMillion: "5" })).toBe("0.000222500"); +}); +it("retains missing cache TTL and malformed token subsets as unknown", () => { + const write = normalizeProviderUsage("anthropic", { model: "fixture", usage: { input_tokens: 1, cache_read_input_tokens: 0, cache_creation_input_tokens: 4, output_tokens: 1 } }); + expect(estimateProviderCost(write, { ...price, provider: "anthropic" })).toBeNull(); + expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 2 }, output_tokens: 1 } }).complete).toBe(false); + expect(normalizeProviderUsage("openai", { usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1, output_tokens_details: { reasoning_tokens: 2 } } }).complete).toBe(false); +}); +it("uses one terminal Responses usage receipt regardless of streamed text or repeat notifications", () => { + const stream = new ProviderUsageAccumulator("openai"); + stream.observe({ type: "response.output_text.delta", delta: "not a usage receipt" }); + expect(stream.receipt().complete).toBe(false); + const terminal = { type: "response.completed", response: { id: "response", model: "fixture", usage: { input_tokens: 1, input_tokens_details: { cache_write_tokens: 0, cached_tokens: 0 }, output_tokens: 1 } } }; + stream.observe(terminal); stream.observe(terminal); + expect(stream.receipt()).toMatchObject({ final: true, complete: true, tokens: { input: 1, output: 1 } }); + expect(estimateProviderCost(stream.receipt(), price)).toBe("0.000012500"); +}); + +const modernPrice: EvalModelPrice = { ...price, inputUsdPerMillion: "4", cachedReadUsdPerMillion: "0.4", outputUsdPerMillion: "20", + cacheWrite30mUsdPerMillion: "5", cacheWrite12hUsdPerMillion: "10", + longContext: { aboveInputTokens: 272_000, rates: { inputUsdPerMillion: "8", cachedReadUsdPerMillion: "0.8", outputUsdPerMillion: "30", cacheWrite30mUsdPerMillion: "10", cacheWrite12hUsdPerMillion: "20" } } }; +const modernUsage = (total = 12_000, read = 4000, shortWrite = 3000, longWrite = 4000) => ({ model: "fixture", usage: { + input_tokens: total, input_tokens_details: { cached_tokens: read, cache_write_tokens: shortWrite + longWrite, + cache_write_tokens_details: { ttl_30m_tokens: shortWrite, ttl_12h_tokens: longWrite } }, output_tokens: 100, +} }); +it("separates modern OpenAI ordinary input and mixed-TTL writes without adding the base rate twice", () => { + const receipt = normalizeProviderUsage("openai", modernUsage()); + expect(receipt).toMatchObject({ complete: true, tokens: { input: 1000, cachedRead: 4000, cachedWrite: 7000 }, cacheWrite30m: 3000, cacheWrite12h: 4000 }); + expect(estimateProviderCost(receipt, modernPrice)).toBe("0.062600000"); +}); +it("applies long-context pricing to the entire request using total input including cache", () => { + expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_000, 270_000, 0, 1000)), modernPrice)).toBe("0.124000000"); + expect(estimateProviderCost(normalizeProviderUsage("openai", modernUsage(272_001, 270_000, 0, 1000)), modernPrice)).toBe("0.247008000"); +}); +it("keeps missing modern write counters or missing/inconsistent TTL splits unknown", () => { + const missing = modernUsage(); delete (missing.usage.input_tokens_details as any).cache_write_tokens; + expect(normalizeProviderUsage("openai", missing).complete).toBe(false); + expect(estimateProviderCost(normalizeProviderUsage("openai", missing), modernPrice)).toBeNull(); + const noTtl = modernUsage(); delete (noTtl.usage.input_tokens_details as any).cache_write_tokens_details; + expect(estimateProviderCost(normalizeProviderUsage("openai", noTtl), modernPrice)).toBeNull(); + const mismatch = modernUsage(); mismatch.usage.input_tokens_details.cache_write_tokens_details.ttl_12h_tokens++; + expect(normalizeProviderUsage("openai", mismatch).complete).toBe(false); +}); + +const policyPrice = { ...modernPrice, model: "gpt-5.6-sol" }; +function requestPolicyReceipt(request: Record = {}, details: Record = {}) { + const stream = new ProviderUsageAccumulator("openai", { model: policyPrice.model, input: "hello", ...request }); + stream.observe({ type: "response.completed", response: { model: policyPrice.model, usage: { + input_tokens: 12_000, input_tokens_details: { cached_tokens: 4000, cache_write_tokens: 7000, ...details }, output_tokens: 100, + } } }); + return stream.receipt(); +} +it.each([ + [{}, "30m", "0.042600000"], + [{ prompt_cache_retention: "24h" }, "30m", "0.042600000"], + [{ prompt_cache_options: { mode: "implicit", ttl: "12h" } }, "12h", "0.077600000"], +] as const)("prices omitted write TTLs from a documented visible request policy: %j", (request, ttl, cost) => { + const receipt = requestPolicyReceipt(request); + expect(receipt).toMatchObject({ final: true, complete: true, cacheWriteAttribution: { + source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl, + } }); + expect(estimateProviderCost(receipt, policyPrice)).toBe(cost); +}); +it.each([ + { model: "unqualified-model" }, { previous_response_id: "stored-response" }, { conversation: "stored-conversation" }, + { prompt: { id: "stored-prompt" } }, { context_management: [] }, + { input: [{ type: "item_reference", id: "stored-item" }] }, + { input: [{ type: "compaction", encrypted_content: "opaque" }] }, + { input: [{ role: "user", content: [{ type: "input_text", text: "hello", prompt_cache_breakpoint: { mode: "explicit", ttl: "12h" } }] }] }, + { prompt_cache_options: { mode: "explicit" } }, { prompt_cache_options: { ttl: "unknown" } }, + { prompt_cache_options: { ttl: null } }, { prompt_cache_options: null }, + { prompt_cache_options: { future_option: true } }, +])("keeps hidden, explicit, or unqualified cache policies unknown: %j", request => { + const receipt = requestPolicyReceipt(request); + expect(receipt.cacheWriteAttribution).toBeUndefined(); + expect(estimateProviderCost(receipt, policyPrice)).toBeNull(); +}); +it("prefers provider TTL splits and never repairs malformed or partial provider counters", () => { + const reported = requestPolicyReceipt({}, { cache_write_tokens_details: { ttl_30m_tokens: 3000, ttl_12h_tokens: 4000 } }); + expect(reported.cacheWriteAttribution).toBeUndefined(); + expect(estimateProviderCost(reported, policyPrice)).toBe("0.062600000"); + for (const details of [{ cache_write_tokens: undefined }, { cache_write_tokens_details: null }, + { cache_write_tokens_details: { ttl_30m_tokens: 7000 } }, { cache_write_tokens_details: { ttl_30m_tokens: 7000, ttl_12h_tokens: 1 } }]) { + const receipt = requestPolicyReceipt({}, details); + expect(receipt.cacheWriteAttribution).toBeUndefined(); + expect(estimateProviderCost(receipt, policyPrice)).toBeNull(); + } +}); diff --git a/tests/runner-e2e/provider-usage.ts b/tests/runner-e2e/provider-usage.ts new file mode 100644 index 00000000000..bb3453a761d --- /dev/null +++ b/tests/runner-e2e/provider-usage.ts @@ -0,0 +1,154 @@ +import { usdToUnits } from "../../packages/shared/src/money.js"; +import type { EvalCostObservation } from "./cost-ledger.js"; + +type Tokens = NonNullable; +type Row = Record; +const row = (value: unknown): Row => value && typeof value === "object" && !Array.isArray(value) ? value as Row : {}; +const count = (value: unknown): number | undefined => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined; +export interface ProviderUsageReceipt { + provider: "openai" | "anthropic"; + tokens: Tokens; + complete: boolean; + providerResponseId?: string; + model?: string; + cacheWrite5m?: number; + cacheWrite1h?: number; + cacheWrite30m?: number; + cacheWrite12h?: number; + cacheWriteAttribution?: { source: "request_policy"; version: "openai-prompt-cache-2026-10-20"; ttl: "30m" | "12h" }; +} + +function openAiRequestCachePolicy(payload: unknown): { model: string; ttl: "30m" | "12h" } | undefined { + const body = row(payload); + if (body.model !== "gpt-5.6-sol" || body.previous_response_id !== undefined || body.conversation !== undefined || body.prompt !== undefined + || body.context_management !== undefined || !(typeof body.input === "string" || Array.isArray(body.input))) return; + const options = row(body.prompt_cache_options); + if (body.prompt_cache_options !== undefined && body.prompt_cache_options !== options) return; + if (Object.keys(options).some(key => !["mode", "ttl"].includes(key)) || options.mode !== undefined && options.mode !== "implicit") return; + const ttl = options.ttl === undefined ? "30m" : options.ttl; + if (ttl !== "30m" && ttl !== "12h") return; + const hasUnqualifiedInput = (value: unknown): boolean => { + if (Array.isArray(value)) return value.some(hasUnqualifiedInput); + if (!value || typeof value !== "object") return false; + const item = row(value); + return "prompt_cache_breakpoint" in item || item.type === "compaction" || item.type === "item_reference" + || Object.values(item).some(hasUnqualifiedInput); + }; + if (hasUnqualifiedInput(body.input)) return; + return { model: body.model, ttl }; +} + +export function normalizeProviderUsage(provider: "openai" | "anthropic", payload: unknown): ProviderUsageReceipt { + const body = row(payload), usage = row(body.usage); + const output = count(usage.output_tokens); + let tokens: Tokens; + let cacheWrite5m: number | undefined, cacheWrite1h: number | undefined; + let cacheWrite30m: number | undefined, cacheWrite12h: number | undefined; + if (provider === "openai") { + const totalInput = count(usage.input_tokens); + const details = row(usage.input_tokens_details); + const cachedRead = count(details.cached_tokens); + let cachedWrite = count(details.cache_write_tokens); + const writes = row(details.cache_write_tokens_details); + cacheWrite30m = count(writes.ttl_30m_tokens); + cacheWrite12h = count(writes.ttl_12h_tokens); + if (cachedWrite !== undefined && cacheWrite30m !== undefined && cacheWrite12h !== undefined && cacheWrite30m + cacheWrite12h !== cachedWrite) cachedWrite = undefined; + const reasoning = count(row(usage.output_tokens_details).reasoning_tokens); + tokens = { input: totalInput !== undefined && cachedRead !== undefined && cachedWrite !== undefined && totalInput >= cachedRead + cachedWrite ? totalInput - cachedRead - cachedWrite : undefined, + cachedRead, cachedWrite, output, reasoning }; + } else { + const cache = row(usage.cache_creation); + cacheWrite5m = count(cache.ephemeral_5m_input_tokens); + cacheWrite1h = count(cache.ephemeral_1h_input_tokens); + tokens = { input: count(usage.input_tokens), cachedRead: count(usage.cache_read_input_tokens), cachedWrite: count(usage.cache_creation_input_tokens), output }; + if (tokens.cachedWrite !== undefined && cacheWrite5m !== undefined && cacheWrite1h !== undefined && cacheWrite5m + cacheWrite1h !== tokens.cachedWrite) tokens.cachedWrite = undefined; + } + const complete = tokens.input !== undefined && tokens.cachedRead !== undefined && tokens.cachedWrite !== undefined && output !== undefined + && !(tokens.reasoning !== undefined && tokens.reasoning > output); + return { provider, tokens: Object.fromEntries(Object.entries(tokens).filter(([, value]) => value !== undefined)), complete, + ...(typeof body.id === "string" ? { providerResponseId: body.id } : {}), + ...(typeof body.model === "string" ? { model: body.model } : {}), + ...(cacheWrite5m !== undefined ? { cacheWrite5m } : {}), ...(cacheWrite1h !== undefined ? { cacheWrite1h } : {}), + ...(cacheWrite30m !== undefined ? { cacheWrite30m } : {}), ...(cacheWrite12h !== undefined ? { cacheWrite12h } : {}) }; +} + +export class ProviderUsageAccumulator { + private body: Row = {}; + private usage: Row = {}; + private final = false; + private readonly cachePolicy: ReturnType; + constructor(readonly provider: "openai" | "anthropic", request?: unknown) { + this.cachePolicy = provider === "openai" ? openAiRequestCachePolicy(request) : undefined; + } + observe(event: unknown) { + const value = row(event); + if (this.provider === "openai") { + const response = row(value.response); + if (["response.completed", "response.incomplete", "response.failed"].includes(String(value.type))) { + this.body = response; + this.usage = row(response.usage); + this.final = true; + } else if (value.object === "response") { + this.body = value; + this.usage = row(value.usage); + this.final = ["completed", "incomplete", "failed"].includes(String(value.status)); + } + } else { + if (value.type === "message_start") { + this.body = row(value.message); + this.usage = { ...this.usage, ...row(this.body.usage) }; + } else if (value.type === "message_delta") this.usage = { ...this.usage, ...row(value.usage) }; + else if (value.type === "message_stop") this.final = true; + else if (value.type === "message") { this.body = value; this.usage = row(value.usage); this.final = true; } + } + } + receipt() { + const receipt = normalizeProviderUsage(this.provider, { ...this.body, usage: this.usage }); + const details = row(this.usage.input_tokens_details); + if (this.cachePolicy && receipt.model === this.cachePolicy.model && receipt.complete && receipt.tokens.cachedWrite! > 0 + && !("cache_write_tokens_details" in details)) { + receipt.cacheWrite30m = this.cachePolicy.ttl === "30m" ? receipt.tokens.cachedWrite : 0; + receipt.cacheWrite12h = this.cachePolicy.ttl === "12h" ? receipt.tokens.cachedWrite : 0; + receipt.cacheWriteAttribution = { source: "request_policy", version: "openai-prompt-cache-2026-10-20", ttl: this.cachePolicy.ttl }; + } + return { ...receipt, final: this.final }; + } +} + +export interface EvalTokenRates { + inputUsdPerMillion: string; + cachedReadUsdPerMillion: string; + outputUsdPerMillion: string; + cacheWrite5mUsdPerMillion?: string; + cacheWrite1hUsdPerMillion?: string; + cacheWrite30mUsdPerMillion?: string; + cacheWrite12hUsdPerMillion?: string; +} +export interface EvalModelPrice extends EvalTokenRates { + provider: "openai" | "anthropic"; + model: string; + version: string; + sourceUrl: string; + longContext?: { aboveInputTokens: number; rates: EvalTokenRates }; +} + +export function estimateProviderCost(receipt: ProviderUsageReceipt, price: EvalModelPrice): string | null { + if (!receipt.complete || receipt.model !== price.model || receipt.provider !== price.provider) return null; + const { tokens } = receipt; + const totalInput = tokens.input! + tokens.cachedRead! + tokens.cachedWrite!; + if (!Number.isSafeInteger(totalInput)) return null; + const rates = price.longContext && totalInput > price.longContext.aboveInputTokens ? price.longContext.rates : price; + const priced = (count: number, rate: string) => { + if (!/^(?:0|[1-9][0-9]*)(?:\.[0-9]{1,9})?$/.test(rate)) throw new Error("Invalid nonnegative model price"); + return BigInt(count) * usdToUnits(rate); + }; + let numerator = priced(tokens.input!, rates.inputUsdPerMillion) + priced(tokens.cachedRead!, rates.cachedReadUsdPerMillion) + priced(tokens.output!, rates.outputUsdPerMillion); + if (tokens.cachedWrite! > 0) { + const counts = receipt.provider === "openai" ? [receipt.cacheWrite30m, receipt.cacheWrite12h] : [receipt.cacheWrite5m, receipt.cacheWrite1h]; + const writeRates = receipt.provider === "openai" ? [rates.cacheWrite30mUsdPerMillion, rates.cacheWrite12hUsdPerMillion] : [rates.cacheWrite5mUsdPerMillion, rates.cacheWrite1hUsdPerMillion]; + if (counts[0] === undefined || counts[1] === undefined || counts[0] + counts[1] !== tokens.cachedWrite || writeRates[0] === undefined || writeRates[1] === undefined) return null; + numerator += priced(counts[0], writeRates[0]) + priced(counts[1], writeRates[1]); + } + const units = (numerator + 999_999n) / 1_000_000n; + return `${units / 1_000_000_000n}.${String(units % 1_000_000_000n).padStart(9, "0")}`; +} diff --git a/tests/runner-e2e/report-format.ts b/tests/runner-e2e/report-format.ts index 4b50371ed53..7237d456972 100644 --- a/tests/runner-e2e/report-format.ts +++ b/tests/runner-e2e/report-format.ts @@ -1,6 +1,12 @@ import type { RunnerE2EBillingSummary } from "./types.js"; import { billingCoverageLabel } from "./billing.js"; +export function isPublicReportTimestamp(value: unknown): value is string { + return typeof value === "string" && value === value.trim() + && /^\d{4}-\d{2}-\d{2}T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d{1,9})?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/.test(value) + && Number.isFinite(Date.parse(value)); +} + export function html(value: unknown) { return String(value ?? "") .replaceAll("&", "&") @@ -10,7 +16,8 @@ export function html(value: unknown) { .replaceAll("'", "'"); } -export function durationLabel(durationMs: number) { +export function durationLabel(durationMs: number | null) { + if (durationMs === null) return "Unknown"; if (durationMs < 1_000) return `${durationMs}ms`; const seconds = Math.round(durationMs / 1_000); if (seconds < 60) return `${seconds}s`; @@ -23,7 +30,7 @@ export function tokenLabel(value: number, llm?: RunnerE2EBillingSummary["llm"]) } export function llmCostLabel(value: number, llm: RunnerE2EBillingSummary["llm"]) { - return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount); + return billingCoverageLabel(usdLabel(value), llm.runsWithReportedCost, llm.runCount, (llm.runsWithPartialReportedCost ?? 0) > 0); } export function usdLabel(value: number | null) { diff --git a/tests/runner-e2e/report-playwright.config.ts b/tests/runner-e2e/report-playwright.config.ts new file mode 100644 index 00000000000..390ee132084 --- /dev/null +++ b/tests/runner-e2e/report-playwright.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "@playwright/test"; + +export default defineConfig({ + testDir: "../e2e", testMatch: "runner-e2e-dashboard.spec.ts", workers: 1, retries: 0, + timeout: 30_000, outputDir: "../../test-results/eval-report-navigation", + use: { browserName: "chromium", headless: true }, reporter: "list", +}); diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index 2677ec7d3b6..f3c516fbd88 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -1,3 +1,4 @@ +import { createWorkerTiming } from "./worker-timing.js"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; @@ -20,6 +21,50 @@ afterEach(async () => { }); describe("runner E2E report aggregation", () => { + it("publishes retained passing results but exits unsuccessfully when the request journal is damaged", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-damaged-ledger-report-")); + cleanupDirectories.push(root); + const executionId = "core-compatibility.runner-codex.local.message-marker"; + const directory = path.join(root, "attempt-1"); + await mkdir(directory); + const result: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility", + attempt: 1, status: "passed", profileId: "runner-codex", environmentId: "local", + caseId: "message-marker", provider: "codex", model: "fixture-model", runtimeMode: "native", + startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed", + }; + const phaseStart = Date.parse(result.startedAt); + const timing = await createWorkerTiming(path.join(directory, "worker-timing.json"), result, phaseStart); + await timing.enter("execution", phaseStart + 200); + await timing.enter("cleanup", phaseStart + 800); + await timing.enter("finalization", phaseStart + 900); + result.workerTiming = await timing.finish(phaseStart + 1000); + const original = JSON.stringify(result); + await writeFile(path.join(directory, "result.json"), original); + await writeFile(path.join(directory, "final-state.png"), "fake-png"); + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] })); + const journal = path.join(root, "journal.jsonl"), output = path.join(root, "merged"); + await writeFile(journal, '{"damaged-journal":'); + await expect(execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]), PAPERCLIP_E2E_OPERATIONS_LEDGER: journal }, + })).rejects.toMatchObject({ code: 1 }); + const normalized = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")); + expect(normalized).toMatchObject({ passed: 1, failed: 0, operations: { status: "unavailable" } }); + expect(normalized.results[0]).toMatchObject({ executionId, status: "passed" }); + expect(normalized.accounting.attempts[0].workerTiming).toEqual(result.workerTiming); + expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Worker setup time: 200ms."); + expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("200msWorker setup time"); + expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("Worker phase timing: complete."); + expect(normalized.billing.agentRunDurationMs).toBeNull(); + expect(await readFile(path.join(output, "index.html"), "utf8")) + .toContain("UnknownAgent execution time"); + for (const file of ["index.html", "summary.md"]) { + expect(await readFile(path.join(output, file), "utf8")).toContain("Request ledger accounting unavailable"); + } + expect(await readFile(path.join(directory, "result.json"), "utf8")).toBe(original); + }); + it("corrects overconfident timing verdicts without changing retained bounds or original files", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "runner-latency-report-")); cleanupDirectories.push(root); @@ -331,13 +376,14 @@ describe("runner E2E report aggregation", () => { }, }); expect(normalized.billing).toMatchObject({ - reportedLlmCostUsd: 0.0125, + reportedLlmCostUsd: 0.025, llm: { - inputTokens: 1_250, - outputTokens: 75, - runsWithReportedCost: 1, + inputTokens: 2_500, + outputTokens: 150, + runsWithReportedCost: 2, }, }); + expect(normalized.accounting.attempts).toHaveLength(2); expect(normalized.results[0]).toMatchObject({ attempt: 2, evidenceValid: true, @@ -353,6 +399,10 @@ describe("runner E2E report aggregation", () => { "utf8", ); expect(dashboard).toContain("Runner Full-Stack E2E"); + expect(dashboard).toMatch(/\$0\.0250*<\/strong>LLM reported subtotal/); + expect(dashboard).toContain("2sSummed attempt time"); + expect(dashboard).toContain("Carried-forward evidence can span older runs"); + expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("Summed attempt time: 2s; 2 retained attempts; 0 missing attempts."); expect(dashboard).toContain(executionId); expect(dashboard).toContain("case-passed"); expect(dashboard).not.toContain( @@ -668,3 +718,45 @@ describe("runner E2E report aggregation", () => { expect(junit).not.toContain(" { + const { selectRerunArtifacts } = await import("./select-rerun-artifacts.js"); + const root = await mkdtemp(path.join(os.tmpdir(), "runner-partial-rerun-")); + cleanupDirectories.push(root); + const artifactRoot = path.join(root, "downloaded"), selectedRoot = path.join(root, "selected"); + const executions = ["core-compatibility.runner-codex.local.message-marker", "core-compatibility.runner-claude.local.message-marker"]; + const source = { sha: "a".repeat(40), ref: "master", workflowRunUrl: "https://github.com/paperclipai/paperclip/actions/runs/12345" }; + for (const [index, executionId] of executions.entries()) { + const workflowAttempt = index + 1; + const artifactName = `runner-e2e-12345-${workflowAttempt}-${executionId}`; + const directory = path.join(artifactRoot, artifactName, `gha-12345-${workflowAttempt}-${executionId}`, "attempt-1"); + await mkdir(directory, { recursive: true }); + const result: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v2", executionId, suiteId: "core-compatibility", source, + accountingIdentity: { campaignId: `gha-12345-${workflowAttempt}`, attemptId: `attempt-${index}` }, + attempt: 1, status: "passed", profileId: index ? "runner-claude" : "runner-codex", environmentId: "local", + caseId: "message-marker", provider: index ? "claude" : "codex", model: "fixture", runtimeMode: "native", + startedAt: "2026-10-20T00:00:00Z", finishedAt: "2026-10-20T00:00:01Z", durationMs: 1000, cleanup: "passed", + usage: { inputTokens: 100, outputTokens: 10, costUsd: 0.01 * workflowAttempt }, + }; + await writeFile(path.join(directory, "result.json"), JSON.stringify(result)); + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: ["final-state.png"], leaks: [], missing: [] })); + await writeFile(path.join(directory, "final-state.png"), "fixture-png"); + } + await selectRerunArtifacts({ artifactRoot, selectedRoot, expectedExecutionIds: executions, workflowRunId: "12345", workflowRunAttempt: 2, + sourceSha: source.sha, sourceRef: source.ref, workflowRunUrl: source.workflowRunUrl, + jobs: { attempts: [{ run_attempt: 1, run_started_at: "2026-10-20T00:00:00Z" }, { run_attempt: 2, run_started_at: "2026-10-20T01:00:00Z" }], + jobs: [{ name: executions[0]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" }, + { name: executions[1]!, run_attempt: 1, started_at: "2026-10-20T00:01:00Z" }, + { name: executions[1]!, run_attempt: 2, started_at: "2026-10-20T01:01:00Z" }] }, + }); + const output = path.join(root, "report"); + await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, env: { ...process.env, PAPERCLIP_E2E_CAMPAIGN_ID: "gha-12345-2", PAPERCLIP_RUNNER_E2E_REPORT_ROOT: selectedRoot, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify(executions) }, + }); + const campaign = JSON.parse(await readFile(path.join(output, "normalized-results.json"), "utf8")); + expect(campaign).toMatchObject({ passed: 2, failed: 0, billing: { reportedLlmCostUsd: 0.03 }, accounting: { excludedArtifacts: 0 } }); + expect(campaign.accounting.attempts).toHaveLength(2); + expect(campaign.results[0].accountingIdentity.campaignId).toBe("gha-12345-1"); +}); diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 81103993b2a..a8a9664c3ac 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -1,3 +1,4 @@ +import { campaignTimingText, summarizeCampaignTiming, summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js"; import { stageDashboardBrandAssets } from "./report-assets.js"; import { discoverReportCatalog } from "./report-catalog.js"; import path from "node:path"; @@ -12,6 +13,10 @@ import { } from "node:fs/promises"; import { runnerMatrix } from "./catalog.js"; import { billingCoverageLabel, summarizeExecutionBilling } from "./billing.js"; +import { isSelectedCarryForward, retainedCampaignAttempts } from "./attempt-accounting.js"; +import { summarizeCampaignFeedback } from "./feedback.js"; +import { campaignFeedbackReviewText } from "./feedback-dispositions.js"; +import { operationsReportText, readEvalOperationsReport } from "./operations-report.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { buildRunnerCampaign, @@ -195,6 +200,12 @@ async function main() { const resultFiles = (await walk(root)).filter( (file) => path.basename(file) === "result.json", ); + const requestedCampaignId = process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? + (process.env.GITHUB_RUN_ID ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}` : undefined); + const selection = await readFile(path.join(root, "selected-artifacts.json"), "utf8") + .then(value => JSON.parse(value) as unknown) + .catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }); + let excludedArtifacts = 0; const candidates = new Map(); for (const resultFile of resultFiles) { const parsed = JSON.parse( @@ -206,6 +217,13 @@ async function main() { ) continue; const result = upgradeRunnerResult(parsed); + if (requestedCampaignId && result.accountingIdentity && + result.accountingIdentity.campaignId !== requestedCampaignId && + !isSelectedCarryForward({ requestedCampaignId, result, selection, + relativeResultPath: path.relative(root, resultFile).split(path.sep).join("/") })) { + excludedArtifacts += 1; + continue; + } const directory = path.dirname(resultFile); const evidence = await readFile( path.join(directory, "evidence-manifest.json"), @@ -314,17 +332,24 @@ async function main() { billing: summarizeExecutionBilling(entry.result), })); const generatedAt = new Date().toISOString(); + const retained = retainedCampaignAttempts( + selected.map(entry => entry.result), + [...candidates.values()].flatMap(entries => entries.map(entry => entry.result)), + ); const campaign = buildRunnerCampaign({ campaignId: - process.env.PAPERCLIP_E2E_CAMPAIGN_ID ?? - (process.env.GITHUB_RUN_ID - ? `gha-${process.env.GITHUB_RUN_ID}-${process.env.GITHUB_RUN_ATTEMPT ?? "1"}` - : `report-${generatedAt.replace(/[:.]/g, "-")}`), + requestedCampaignId ?? `report-${generatedAt.replace(/[:.]/g, "-")}`, generatedAt, expected, results: resolvedResults, + attempts: retained.attempts, + excludedArtifacts: excludedArtifacts + retained.excludedArtifacts, + ...(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER ? { operations: await readEvalOperationsReport(process.env.PAPERCLIP_E2E_OPERATIONS_LEDGER) } : {}), + ...(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW ? { feedbackReview: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_FEEDBACK_REVIEW, "utf8")) } : {}), + ...(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT ? { braintrustExperiment: JSON.parse(await readFile(process.env.PAPERCLIP_E2E_BRAINTRUST_EXPERIMENT, "utf8")) } : {}), }); const billing = campaign.billing; + const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); const normalized = { ...campaign, results: selected.map((entry, index) => ({ @@ -386,7 +411,15 @@ async function main() { : []), `Tokens: ${billingCoverageLabel(`${billing.llm.inputTokens} input / ${billing.llm.outputTokens} output / ${billing.llm.cachedInputTokens} cached`, billing.llm.runsWithTokenUsage, billing.llm.runCount)}`, "", - `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount)}`, + `Provider-reported LLM cost: ${billingCoverageLabel(`$${billing.reportedLlmCostUsd.toFixed(6)}`, billing.llm.runsWithReportedCost, billing.llm.runCount, (billing.llm.runsWithPartialReportedCost ?? 0) > 0)}`, + `Estimated LLM cost: ${billingCoverageLabel(`$${(billing.estimatedLlmCostUsd ?? 0).toFixed(6)}`, billing.llm.runsWithEstimatedCost ?? 0, billing.llm.runCount)}`, + `Accounting: ${campaign.accounting!.attempts.length} retained attempts; ${campaign.accounting!.missingAttempts} missing attempts; ${campaign.accounting!.excludedArtifacts} out-of-scope artifacts excluded. Costs cover captured usage, not unobserved setup or external charges.`, + ...campaignTimingText(summarizeCampaignTiming(campaign.results, campaign.accounting)), + ...workerPhaseTimingText(summarizeWorkerPhaseTiming(campaign.results, campaign.accounting)), + ...(campaign.operations ? operationsReportText(campaign.operations) : []), + "", + `Feedback across retained attempts: ${feedback.complaints} complaints; ${feedback.suggestions} suggestions; ${feedback.incomplete} attempts with incomplete collection. Tool submissions: ${feedback.failedSubmissions} failed, ${feedback.unknownSubmissions} unresolved. Full bodies are in access-controlled snapshots/agent-feedback.json or snapshots/agent-feedback-recovered.json.`, + campaignFeedbackReviewText(campaign), "", `Estimated Daytona list-price runtime cost: $${billing.estimatedRuntimeCostUsd.toFixed(6)}`, ...(billing.judge ? [`Estimated judge cost: ${billing.judge.estimatedCostUsd === null ? "unknown" : `$${billing.judge.estimatedCostUsd.toFixed(6)}`}; ${billing.judge.attempts} attempts; ${billing.judge.attemptsWithUnknownUsage} with unknown usage; $${billing.judge.reservedCostUsd.toFixed(6)} reserved`] : []), @@ -401,7 +434,7 @@ async function main() { const cell = publicCampaignUrl ? `[${resolved.executionId}](${publicCampaignUrl}#execution-${encodeURIComponent(resolved.executionId)})` : resolved.executionId; - return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; + return `| ${cell} | ${resolved.attempt} | ${entry.valid ? "pass" : "fail"} | ${resolved.runtimeMode} | ${Math.round(resolved.durationMs / 1000)}s | ${billingCoverageLabel(`${cellBilling.llm.inputTokens}/${cellBilling.llm.outputTokens}`, cellBilling.llm.runsWithTokenUsage, cellBilling.llm.runCount)} | ${billingCoverageLabel(`$${cellBilling.reportedCostUsd.toFixed(6)}`, cellBilling.llm.runsWithReportedCost, cellBilling.llm.runCount, (cellBilling.llm.runsWithPartialReportedCost ?? 0) > 0)} | ${runtimeCost === undefined ? cellBilling.runtime.costStatus : `$${runtimeCost.toFixed(6)} est.`} | ${detail} |`; }), "", ]; @@ -419,7 +452,7 @@ async function main() { const junit = `${cases}\n`; await writeFile(path.join(output, "junit.xml"), junit); console.log(summaryLines.join("\n")); - if (failures > 0 || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1; + if (failures > 0 || campaign.operations?.status === "unavailable" || (dependable && (!dependable.complete || dependable.unclassifiedResults.length > 0))) process.exitCode = 1; } await main().catch((error) => { diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index eb06745b108..460855184f4 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -1,6 +1,7 @@ +import { validateWorkerTiming } from "./worker-timing.js"; import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js"; import { STARTUP_REUSE_LAYERS, STARTUP_REUSE_REASONS } from "../../packages/adapter-utils/src/startup-reuse.js"; -import type { RunnerE2EResult } from "./types.js"; +import type { RunnerE2ECampaignAccounting, RunnerE2EResult } from "./types.js"; import { DEPENDABLE_CELLS } from "./dependable-qualification.js"; type Rule = (value: unknown, at: string) => void; @@ -117,7 +118,7 @@ const relativeFile: Rule = (value, at) => { }; const runtime = shape({ provider: oneOf("local", "daytona"), - agentRunDurationMs: number, + agentRunDurationMs: nullable(number), leaseDurationMs: nullable(number), leaseCount: integer, cpuCores: optional(number), @@ -138,12 +139,14 @@ const llm = shape({ runCount: integer, runsWithTokenUsage: integer, runsWithReportedCost: integer, + runsWithEstimatedCost: optional(integer), inputTokens: number, outputTokens: number, cachedInputTokens: number, totalTokens: number, reportedCostUsd: number, - costStatus: oneOf("reported", "partial", "unpriced", "unavailable"), + estimatedCostUsd: optional(number), + costStatus: oneOf("reported", "estimated", "partial", "unpriced", "unavailable"), }); const billing = shape({ judge: optional(shape({ inputTokens: nullable(number), outputTokens: nullable(number), estimatedCostUsd: nullable(number), reservedCostUsd: number })), @@ -155,6 +158,43 @@ const billing = shape({ observedAndEstimatedCostUsd: nullable(number), complete: boolean, }); +const tracing = shape({ + projectId: string, traceId: string, spanId: string, status: oneOf("pending", "accepted", "failed"), + verification: oneOf("unverified"), nativeConfiguration: oneOf("not_started", "configured", "failed"), + companyId: optional(string), revision: optional(string), errorCode: optional(string), httpStatus: optional(number), + finalization: optional(shape({ spanId: string, status: oneOf("accepted", "failed"), outcome: oneOf("passed", "failed"), + verification: oneOf("unverified"), errorCode: optional(string), httpStatus: optional(number) })), +}); +const feedback = shape({ + version: (value, at) => { if (value !== 1) invalid(at); }, + collection: oneOf("complete", "partial", "unavailable", "not_started"), + invocationCollection: oneOf("complete", "partial", "unavailable", "not_started"), + toolAvailability: oneOf("observed", "unknown"), + complaints: integer, suggestions: integer, bodiesCollected: integer, runsInspected: integer, + invocations: shape({ completed: integer, failed: integer, unknown: integer }), + review: array(shape({ fingerprint: (value, at) => { if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) invalid(at); }, + kind: oneOf("complaint", "suggestion"), occurrences: integer, disposition: oneOf("needs_investigation") })), +}); + +export function validateCampaignAccounting(value: RunnerE2ECampaignAccounting) { + shape({ + version: (version, at) => { if (version !== 1) invalid(at); }, + scope: oneOf("retained_attempts", "selected_results"), + missingAttempts: integer, + excludedArtifacts: integer, + attempts: array(shape({ + id: string, executionId: string, attempt: integer, + startedAt: date, finishedAt: date, durationMs: number, + status: oneOf("passed", "failed"), cleanup: oneOf("not_started", "passed", "failed"), billing, feedback: optional(feedback), tracing: optional(tracing), + })), + })(value, "accounting"); + if (new Set(value.attempts.map(attempt => attempt.id)).size !== value.attempts.length) invalid("accounting.attempts duplicate id"); + for (const attempt of value.attempts) { + if (attempt.workerTiming !== undefined) { + try { validateWorkerTiming(attempt.workerTiming, attempt); } catch { invalid("accounting.attempts.workerTiming"); } + } + } +} const matcher: Rule = (value, at) => { object(value, at); const item = value as Record; @@ -182,6 +222,7 @@ const matcher: Rule = (value, at) => { rule(value, at); }; const fields = { + accountingIdentity: optional(shape({ campaignId: string, attemptId: string })), publicMcp: optional(assistant), schema: oneOf( "paperclip.runner-e2e.result/v1", @@ -268,11 +309,13 @@ const fields = { startedAt: date, finishedAt: date, durationMs: number, + workerTiming: optional(object), // Provider usage is intentionally opaque. Billing reads only finite numbers; // the dashboard shows opaque diagnostics through escaped JSON. usage: optional(nullable(object)), runtimeUsage: optional(runtime), billing: optional(billing), + feedback: optional(feedback), tracing: optional(tracing), matcherResults: optional( array(shape({ matcher, passed: boolean, detail: string })), ), @@ -344,6 +387,9 @@ export function validateRetainedRunnerResult( ): asserts value is RunnerE2EResult { result(value, "result"); const checked = value as RunnerE2EResult; + if (checked.workerTiming !== undefined) { + try { validateWorkerTiming(checked.workerTiming, checked); } catch { invalid("result.workerTiming"); } + } if (checked.suiteId === "product-journeys" && checked.status === "passed") { const evidence = checked.productJourney; if (!evidence) return invalid("result.productJourney"); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index d2a1f01848a..ee7e6dfd1f2 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -20,7 +20,7 @@ delete env.NODE_OPTIONS; for (const [name, command] of [ ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], - ["Node tests", ["--test", "tests/runner-e2e/native-completion-checks.test.mjs", + ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"]], ]) { const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index b4b08246536..420d8c97cf5 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,3 +1,6 @@ +import { createWorkerTiming } from "./worker-timing.js"; +import { FEEDBACK_PERSISTENCE_SUITE, gradeFeedbackPersistence } from "./feedback-persistence.js"; +import { EvalAttemptTrace, evalBraintrustConfig, EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js"; import { captureFollowupStopFailureApiState } from "./followup-stop-evidence.js"; import { runFollowupStopResumeFlow } from "./followup-stop-resume-flow.js"; import { runFinalizationRecoveryFlow } from "./finalization-recovery-flow.js"; @@ -58,6 +61,8 @@ import path from "node:path"; import { expect, test, type Page } from "@playwright/test"; import { RunnerApi, pollUntil } from "./api.js"; import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js"; +import { collectAttemptFeedback, emptyFeedbackSummary } from "./feedback.js"; +import { finishBudgetedEvalSetup } from "./budgeted-setup.js"; import { establishPublicMcpSession, runPublicMcpFlow } from "./public-mcp-flow.js"; import { assistantUsage } from "./public-mcp-model.js"; import { runnerExecutionById } from "./catalog.js"; @@ -359,7 +364,9 @@ async function writeSanitizedJson( name: string, value: unknown, secrets: readonly string[], + signal?: AbortSignal, ) { + signal?.throwIfAborted(); const raw = `${JSON.stringify(value)}\n`; // Public API payloads may contain harmless provider-shaped fixtures or // opaque generated tokens. Reject exact campaign credentials before writing, @@ -368,8 +375,9 @@ async function writeSanitizedJson( const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`; const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets); if (leak) throw new Error(`Secret leak in ${name}: ${leak}`); + signal?.throwIfAborted(); await mkdir(directory, { recursive: true }); - await writeFile(path.join(directory, name), safe, "utf8"); + await writeFile(path.join(directory, name), safe, { encoding: "utf8", signal }); } async function submitTaskRevision(page: Page, body: string): Promise { @@ -593,6 +601,7 @@ for (const execution of executions) { } const startedAtMs = Date.now(); const startedAt = new Date(startedAtMs).toISOString(); + const workerTiming = await createWorkerTiming(path.join(snapshotsDir, "worker-timing.json"), { executionId: execution.id, attempt }, startedAtMs); const nonce = `${randomBytes(6).toString("hex")}-${attempt}`; const marker = execution.task.buildVisibleMarker(nonce); const title = execution.task.buildTitle(nonce); @@ -609,7 +618,20 @@ for (const execution of executions) { verifyNativeInstructionPreflight(process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV]); } const credentials = credentialValues(); - const secrets = normalizedSecrets(Object.values(credentials)); + const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]); + const traceConfig = evalBraintrustConfig(process.env); + const trace = traceConfig ? new EvalAttemptTrace(traceConfig, { + campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID ?? "", + attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ?? "", + executionId: execution.id, provider: execution.profile.provider, model: execution.profile.model, environment: execution.environment.id, + }, { + suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, + profileId: execution.profile.id, caseId: execution.task.id, sourceRevision: resolveRunnerE2ESource().sha, + }) : undefined; + const saveTrace = async () => { if (trace) await writeSanitizedJson(snapshotsDir, "braintrust-trace.json", trace.snapshot(), secrets); }; + const onCompanyCreated = async (id: string) => { + if (trace) { try { await trace.configureCompany(api, id); } finally { await saveTrace(); } } + }; const api = new RunnerApi(request); const companyRunFlow = execution.suite.id === "task-titles" || ["plan_task_guidance", "blocker_guidance", "continuation_accounting", "continuation", "context_integrity", "repository_skills", "agent_chat", "everyday_workflow", "first_task", "instruction_persistence", "pi_native", "pi_controls", "copilot_protection", "cursor_native", "native_active_stop", "native_provider_loss", "public_mcp", "case_handoff"].includes(execution.task.flow); const publicMcpUsage = execution.task.flow === "public_mcp" ? assistantUsage(execution.profile.provider === "claude" ? "anthropic" : "openai", execution.profile.model) : undefined; @@ -626,6 +648,7 @@ for (const execution of executions) { let issue: IssueRecord | undefined; let workflowCase: { id: string; identifier?: string } | undefined; let selectedRuns: RunRecord[] = []; + let feedback = emptyFeedbackSummary(); let runtimeLeases: EnvironmentLeaseRecord[] = []; let matcherResults: MatcherResult[] = []; let downloadedResponseProof: Awaited> | undefined; @@ -891,6 +914,8 @@ for (const execution of executions) { }); try { + trace?.phase("setup"); + await saveTrace(); if (execution.task.flow === "public_mcp") publicMcpUserId = await establishPublicMcpSession(api, page, secrets); if (execution.suite.id === "stock-harness") { const receipt = verifyStockHarnessPreflight(process.env[STOCK_PREFLIGHT_ENV]); @@ -920,7 +945,7 @@ for (const execution of executions) { await setupGitStreamingWorkspace(workspacePath); } fixtures = execution.task.flow === "first_task" - ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; } }) + ? await setupFirstTaskFixtures({ page, api, execution, nonce, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, deadlineAt: startedAtMs + deadlineMs, observe: value => { fixtures = value; }, onCompanyCreated }) : await setupLiveFixtures({ api, execution, @@ -928,7 +953,13 @@ for (const execution of executions) { workspacePath, credentials, daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE, + onCompanyCreated, }); + await finishBudgetedEvalSetup({ executionId: execution.id, agentId: fixtures.agent.id, + companyId: fixtures.company.id, deadlineAt: startedAtMs + deadlineMs, api, + observe: observations => writeSanitizedJson(snapshotsDir, "budgeted-setup.json", { observations }, secrets) }); + await workerTiming.enter("execution"); + trace?.phase("execution"); if (["native-completion", NATIVE_INSTRUCTION_SUITE].includes(execution.suite.id)) { const receipt = await captureNativeDefault({ api, agentId: fixtures.agent.id, companyId: fixtures.company.id }); @@ -3078,6 +3109,8 @@ for (const execution of executions) { } } } finally { + await workerTiming.enter("cleanup"); + trace?.phase("cleanup", Boolean(primaryError)); await responseTiming?.stop(); await reviewProvider?.close(); try { @@ -3136,7 +3169,29 @@ for (const execution of executions) { } }, assertions: beforeEnvironmentTeardownAssertions, - teardown: () => fixtures!.teardown(), + teardown: async () => { + await workerTiming.enter("feedback"); + trace?.phase("feedback"); + try { + feedback = await collectAttemptFeedback({ api, companyId: fixtures!.company.id, + onCheckpoint: summary => { feedback = summary; }, + persist: async (evidence, signal) => { + await writeSanitizedJson(snapshotsDir, "agent-feedback.pending.json", evidence, secrets, signal); + signal.throwIfAborted(); + await rename(path.join(snapshotsDir, "agent-feedback.pending.json"), path.join(snapshotsDir, "agent-feedback.json")); + signal.throwIfAborted(); + }, + }); + } catch { + feedback.collection = feedback.bodiesCollected ? "partial" : "unavailable"; + feedback.invocationCollection = "unavailable"; + networkDiagnostics.push({ feedbackCaptureError: "feedback_evidence_write_failed" }); + } finally { + await workerTiming.enter("cleanup"); + trace?.phase("cleanup", !["complete", "not_started"].includes(feedback.collection)); + await fixtures!.teardown(); + } + }, }); matcherResults.push(...verification.checks.map(check => ({ matcher: { kind: "json_path" as const, path: `remoteCleanup.${check.id}`, expected: true }, passed: check.passed, detail: check.detail, @@ -3202,6 +3257,20 @@ for (const execution of executions) { } } + await workerTiming.enter("finalization"); + if (execution.suite.id === FEEDBACK_PERSISTENCE_SUITE) { + let evidence: unknown; + try { evidence = JSON.parse(await readFile(path.join(snapshotsDir, "agent-feedback.json"), "utf8")); } catch { evidence = undefined; } + const checks = gradeFeedbackPersistence({ evidence, nonce, companyId: fixtures?.company.id, + agentId: fixtures?.agent.id, taskId: issue?.id, runIds: selectedRuns.map(run => run.id) }); + matcherResults.push(...checks.map(check => ({ matcher: { kind: "json_path" as const, + path: `feedbackPersistence.${check.id}`, expected: true }, passed: check.passed, detail: check.detail }))); + if (checks.some(check => !check.passed)) primaryError ??= new Error("Feedback persistence matcher failed"); + } + + const traceReceipt = await trace?.export({ failed: Boolean(primaryError), runIds: selectedRuns.map(run => run.id), + cleanup, complaints: feedback.complaints, suggestions: feedback.suggestions }); + await saveTrace(); const finishedAtMs = Date.now(); const runtimeUsage = fixtureRuntimeCostCoverage(buildRuntimeUsage({ environmentId: execution.environment.id, @@ -3211,6 +3280,12 @@ for (const execution of executions) { }), fixtures ?? {}); const resultWithoutBilling: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", + ...(process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID && process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID ? { + accountingIdentity: { + campaignId: process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID, + attemptId: process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID, + }, + } : {}), executionId: execution.id, suiteId: execution.suite.id, suiteDefinitionHash: execution.suiteDefinitionHash, @@ -3248,6 +3323,7 @@ for (const execution of executions) { startedAt, finishedAt: new Date(finishedAtMs).toISOString(), durationMs: finishedAtMs - startedAtMs, + workerTiming: await workerTiming.finish(finishedAtMs), usage: selectedRuns.length === 1 ? (selectedRuns[0]?.usageJson ?? null) @@ -3258,6 +3334,8 @@ for (const execution of executions) { })), }, runtimeUsage, + feedback, + ...(traceReceipt ? { tracing: traceReceipt } : {}), matcherResults, screenshots, cleanup, diff --git a/tests/runner-e2e/scheduled-finalization.test.ts b/tests/runner-e2e/scheduled-finalization.test.ts new file mode 100644 index 00000000000..8b90c9fd377 --- /dev/null +++ b/tests/runner-e2e/scheduled-finalization.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vitest"; +import { buildRunnerCampaign } from "./history.js"; +import { runnerMatrix } from "./catalog.js"; +import { emptyFeedbackSummary, feedbackReview, type FeedbackEvidence } from "./feedback.js"; +import { expectedFixtureFeedback } from "./feedback-persistence.js"; +import { reviewScheduledFeedback, scheduledCampaignFindings } from "./scheduled-finalization.js"; +import type { RunnerE2EResult } from "./types.js"; + +function fixture() { + const cell = runnerMatrix.find(row => row.id === "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion")!; + const nonce = "abcdef123-1", bodies = expectedFixtureFeedback(nonce); + const entries = ["complaint", "suggestion"].map(kind => ({ kind, body: bodies[kind as keyof typeof bodies] })) as FeedbackEvidence["entries"]; + const feedback = { version: 1, companyId: "company", entries, invocations: [], diagnostics: [], summary: emptyFeedbackSummary() } as FeedbackEvidence; + const result: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: cell.id, + suiteId: cell.suite.id, suiteDefinitionHash: cell.suiteDefinitionHash, + accountingIdentity: { campaignId: "scheduled-fixture", attemptId: "one" }, source: { sha: "a".repeat(40), ref: null, workflowRunUrl: null }, + attempt: 1, status: "passed", cleanup: "passed", profileId: cell.profile.id, caseId: cell.task.id, + environmentId: cell.environment.id, provider: cell.profile.provider, model: cell.profile.model, runtimeMode: cell.profile.expectedRuntimeMode, + startedAt: "2026-10-21T00:00:00.000Z", finishedAt: "2026-10-21T00:00:01.000Z", durationMs: 1000, + feedback: { ...emptyFeedbackSummary(), collection: "complete", invocationCollection: "complete", complaints: 1, suggestions: 1, + bodiesCollected: 2, review: feedbackReview(entries) }, + matcherResults: [{ matcher: { kind: "message_exact", expected: `PAPERCLIP_E2E_OK_${nonce}` }, passed: true, detail: "matched" }, + ...["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool", + "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"].map(id => ({ + matcher: { kind: "json_path" as const, path: `feedbackPersistence.${id}`, expected: true }, passed: true, detail: "matched" }))] }; + const campaign = buildRunnerCampaign({ campaignId: "scheduled-fixture", generatedAt: result.finishedAt, + expected: [cell.id], results: [result], attempts: [result] }); + return { result, feedback, campaign }; +} + +it("reviews independently retained exact synthetic bodies without a model call", () => { + const { result, feedback, campaign } = fixture(); + const review = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt); + expect(review.decisions.map(row => row.disposition)).toEqual(["unsupported", "unsupported"]); + expect(JSON.stringify(review)).not.toContain("Synthetic eval"); +}); + +it.each(["body-changed", "wrong-suite", "missing-evidence", "failed-grade", "partial-collection"])("leaves %s feedback for investigation", reason => { + const { result, feedback, campaign } = fixture(); + if (reason === "body-changed") feedback.entries[0]!.body = "A real product problem"; + if (reason === "wrong-suite") result.suiteId = "core-compatibility"; + if (reason === "failed-grade") result.matcherResults![2]!.passed = false; + if (reason === "partial-collection") result.feedback!.collection = "partial"; + const review = reviewScheduledFeedback(campaign, [{ result, feedback: reason === "missing-evidence" ? null : feedback }], result.finishedAt); + expect(review.decisions.every(row => row.disposition === "needs_investigation")).toBe(true); +}); + +it("keeps incomplete accounting and trace export visible independently of a passing product grade", () => { + const { result, feedback, campaign } = fixture(); + campaign.feedbackReview = reviewScheduledFeedback(campaign, [{ result, feedback }], result.finishedAt); + expect(scheduledCampaignFindings(campaign)).toEqual(["accounting_unavailable", "trace_export_incomplete"]); + campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation"; + expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review"); +}); diff --git a/tests/runner-e2e/scheduled-finalization.ts b/tests/runner-e2e/scheduled-finalization.ts new file mode 100644 index 00000000000..b9653b17723 --- /dev/null +++ b/tests/runner-e2e/scheduled-finalization.ts @@ -0,0 +1,117 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { evalBraintrustConfig } from "./braintrust.js"; +import { openExperimentIngestion } from "./experiment-ingestion.js"; +import { collectScheduledIngestion } from "./ingestion-usage.js"; +import { importBraintrustExperiment, type BraintrustExperimentReceipt } from "./braintrust-experiment.js"; +import { verifyBraintrustIndexing } from "./braintrust-indexing.js"; +import { feedbackReviewTemplate, validateCampaignFeedbackReview, type CampaignFeedbackReview } from "./feedback-dispositions.js"; +import { expectedFixtureFeedback } from "./feedback-persistence.js"; +import { feedbackReview, summarizeCampaignFeedback, type FeedbackEvidence } from "./feedback.js"; +import type { EvalOperationsManifest } from "./operations.js"; +import type { RunnerE2ECampaign, RunnerE2EResult } from "./types.js"; + +export function reviewScheduledFeedback(campaign: RunnerE2ECampaign, + evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }>, now: string): CampaignFeedbackReview { + const synthetic = new Set(); + for (const { result, feedback } of evidence) { + if (result.suiteId !== "feedback-persistence" || !feedback || result.feedback?.collection !== "complete" + || result.feedback.invocationCollection !== "complete") continue; + const marker = result.matcherResults?.find(row => row.matcher.kind === "message_exact")?.matcher; + const nonce = marker?.kind === "message_exact" ? /^PAPERCLIP_E2E_OK_([a-z0-9]+-[1-9][0-9]*)$/.exec(marker.expected)?.[1] : undefined; + if (!nonce) continue; + const checks = ["complete-retained-evidence", "one-of-each", "complaint-body-and-attribution", "complaint-completed-tool", + "suggestion-body-and-attribution", "suggestion-completed-tool", "only-expected-submissions"]; + if (!checks.every(id => result.matcherResults?.some(row => row.matcher.kind === "json_path" + && row.matcher.path === `feedbackPersistence.${id}` && row.passed))) continue; + const bodies = expectedFixtureFeedback(nonce); + if (feedback.entries.length !== 2 || !["complaint", "suggestion"].every(kind => feedback.entries.filter(entry => + entry.kind === kind && entry.body === bodies[kind as keyof typeof bodies]).length === 1)) continue; + for (const group of feedbackReview(feedback.entries)) synthetic.add(group.fingerprint); + } + const template = feedbackReviewTemplate(campaign); + const review: CampaignFeedbackReview = { ...template, version: 1, reviewedAt: now, reviewedBy: "agent", + decisions: template.decisions.map(row => ({ fingerprint: row.fingerprint, + disposition: synthetic.has(row.fingerprint) ? "unsupported" : "needs_investigation" })) }; + validateCampaignFeedbackReview(campaign, review); + return review; +} + +export function scheduledCampaignFindings(campaign: RunnerE2ECampaign): string[] { + const findings: string[] = []; + if (campaign.failed || campaign.results.length !== campaign.expected.length || campaign.accounting?.missingAttempts) findings.push("product_or_missing_attempt"); + if (campaign.accounting?.attempts.some(row => row.cleanup !== "passed")) findings.push("cleanup_failure"); + const operations = campaign.operations; + if (!operations || operations.status === "unavailable") findings.push("accounting_unavailable"); + else if (operations.unresolvedRequests || operations.conflictingRequests || operations.inflight) findings.push("accounting_incomplete"); + const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); + if (feedback.incomplete || feedback.failedSubmissions || feedback.unknownSubmissions) findings.push("feedback_collection_incomplete"); + if (!campaign.feedbackReview || campaign.feedbackReview.decisions.some(row => ["actionable", "needs_investigation"].includes(row.disposition))) findings.push("feedback_needs_review"); + if (campaign.results.some(row => row.tracing?.status !== "accepted" || row.tracing.nativeConfiguration !== "configured" + || row.tracing.finalization?.status !== "accepted")) findings.push("trace_export_incomplete"); + return findings; +} + +export async function finalizeScheduledCampaign(env: NodeJS.ProcessEnv) { + if (!env.PAPERCLIP_E2E_OPERATIONS_MANIFEST || !env.GITHUB_WORKSPACE) throw new Error("Missing scheduled campaign manifest"); + const manifest = JSON.parse(await readFile(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST, "utf8")) as EvalOperationsManifest; + const directory = path.dirname(env.PAPERCLIP_E2E_OPERATIONS_MANIFEST); + const root = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results", manifest.limits.campaignId); + const campaign = JSON.parse(await readFile(path.join(root, "campaign.json"), "utf8")) as RunnerE2ECampaign; + const originals: RunnerE2EResult[] = [], evidence: Array<{ result: RunnerE2EResult; feedback: FeedbackEvidence | null }> = []; + for (const selected of campaign.results) { + if (selected.attempt < 1) continue; + if (!manifest.executionIds.includes(selected.executionId) || selected.attempt !== 1) throw new Error("Unexpected scheduled attempt"); + const resultPath = path.join(root, ...selected.executionId.split("."), "attempt-1"); + const result = JSON.parse(await readFile(path.join(resultPath, "result.json"), "utf8")) as RunnerE2EResult; + originals.push(result); + let feedback: FeedbackEvidence | null = null; + try { feedback = JSON.parse(await readFile(path.join(resultPath, "snapshots/agent-feedback.json"), "utf8")); } catch {} + evidence.push({ result, feedback }); + } + campaign.feedbackReview = reviewScheduledFeedback(campaign, evidence, new Date().toISOString()); + await writeFile(path.join(directory, "feedback-review.json"), `${JSON.stringify(campaign.feedbackReview, null, 2)}\n`, { mode: 0o600 }); + const findings = scheduledCampaignFindings(campaign); + const config = evalBraintrustConfig(env); + let experiment: BraintrustExperimentReceipt | undefined; + let ingestion: Awaited> | undefined; + try { + if (!config) throw new Error("Missing Braintrust export configuration"); + if (config.projectId !== manifest.projectId || campaign.campaignId !== manifest.limits.campaignId) throw new Error("Scheduled ingestion identity mismatch"); + ingestion = await openExperimentIngestion({ projectId: config.projectId, campaignId: manifest.limits.campaignId, + receiptPath: `${manifest.ledgerPath}.experiment-ingestion.jsonl`, deadlineAt: Date.now() + 60_000 }); + await importBraintrustExperiment(config, campaign, originals, async receipt => { + await writeFile(path.join(directory, "experiment-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + experiment = receipt; + }, ingestion.transport); + } catch { findings.push("experiment_import_incomplete"); } + finally { + try { await ingestion?.close(); } + catch { findings.push("experiment_ingestion_accounting_incomplete"); } + } + try { + if (!config) throw new Error("Missing Braintrust ingestion configuration"); + const usage = await collectScheduledIngestion({ ledgerPath: manifest.ledgerPath, campaignId: manifest.limits.campaignId, + projectId: config.projectId, executionIds: manifest.executionIds }); + await writeFile(path.join(directory, "ingestion-usage.json"), `${JSON.stringify(usage, null, 2)}\n`, { mode: 0o600 }); + if (usage.status !== "complete") findings.push("ingestion_accounting_incomplete"); + } catch { findings.push("ingestion_accounting_unavailable"); } + let indexing = "unavailable"; + try { + if (!config) throw new Error("Missing Braintrust query configuration"); + const receipt = await verifyBraintrustIndexing(config, campaign, originals, experiment); + indexing = receipt.status; + await writeFile(path.join(directory, "indexing-receipt.json"), `${JSON.stringify(receipt, null, 2)}\n`, { mode: 0o600 }); + } catch {} + if (indexing !== "verified") findings.push("trace_or_experiment_indexing_incomplete"); + await writeFile(path.join(directory, "findings.json"), `${JSON.stringify({ version: 1, findings, + nativeIndexVerification: indexing, invoiceReconciliation: "pending", + feedbackReviewMethod: "Only exact synthetic fixture bodies with all corroborating graders are automatically dismissed; all other feedback requires review" }, null, 2)}\n`, { mode: 0o600 }); + for (const finding of findings) console.error(`::error title=Eval operations::${finding}`); + return findings; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if ((await finalizeScheduledCampaign(process.env)).length) process.exitCode = 1; +} diff --git a/tests/runner-e2e/scheduled-forecast.test.ts b/tests/runner-e2e/scheduled-forecast.test.ts new file mode 100644 index 00000000000..daf5b007a11 --- /dev/null +++ b/tests/runner-e2e/scheduled-forecast.test.ts @@ -0,0 +1,91 @@ +import { expect, it } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { scheduledForecast } from "./scheduled-forecast.js"; +import { scheduledSelection } from "./scheduled-operations.js"; +import measurements from "./scheduled-measurements.json" with { type: "json" }; +import prices from "./scheduled-prices.json" with { type: "json" }; +import type { EvalOperationsManifest } from "./operations.js"; + +const models = prices.models as EvalOperationsManifest["models"]; +const selected = (cadence: "daily" | "weekly") => runnerMatrix.filter(cell => scheduledSelection(cadence).includes(cell.id)); + +it.each([ + ["daily", "0.427624800", 72757], ["weekly", "0.791539400", 127069], +] as const)("retains exact historical %s measurements without inventing hosted or tail estimates", (cadence, modelUsd, summedAttemptMs) => { + const forecast = scheduledForecast(selected(cadence), models); + expect(forecast.historicalSuccessfulSelection).toEqual({ modelUsd, summedAttemptMs }); + expect(forecast.measurementSnapshotSha256).toMatch(/^[0-9a-f]{64}$/); + expect(forecast).toMatchObject({ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, costStatus: "estimated" }); + for (const cell of forecast.cells) { + expect(cell.historicalSuccessful.samples).toBe(1); + expect(cell.historicalSuccessful.modelUsd.p95).toBeNull(); + expect(cell.historicalSuccessful.attemptMs.p95).toBeNull(); + expect(cell.samples[0]!.resultSha256).toMatch(/^[0-9a-f]{64}$/); + } +}); + +it("does not treat the older feedback definition as a current measurement", () => { + const forecast = scheduledForecast(selected("daily"), models); + const feedback = forecast.cells.find(cell => cell.executionId.startsWith("feedback-"))!; + expect(feedback.samples[0]!.mismatches).toContain("definition"); + expect(feedback.compatibleSuccessful.samples).toBe(0); + expect(forecast.compatibleSuccessfulSelection).toEqual({ modelUsd: null, summedAttemptMs: null }); +}); + +it.each(["definitionHash", "model", "environmentId", "runtimeMode", "route", "priceVersion"] as const)("excludes incompatible %s samples without dropping their evidence", field => { + const snapshot = structuredClone(measurements); + const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!; + const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!; + sample[field] = field === "definitionHash" ? "a".repeat(64) : "different"; + const forecast = scheduledForecast([cell], models, snapshot); + expect(forecast.cells[0]!.compatibleSuccessful.samples).toBe(0); + expect(forecast.cells[0]!.samples).toHaveLength(1); + expect(forecast.compatibleSuccessfulSelection.modelUsd).toBeNull(); +}); + +it("keeps missing cells and empty selections unknown", () => { + const snapshot = structuredClone(measurements); + snapshot.samples = []; + expect(scheduledForecast(selected("daily"), models, snapshot).historicalSuccessfulSelection) + .toEqual({ modelUsd: null, summedAttemptMs: null }); + expect(scheduledForecast([], models).compatibleSuccessfulSelection) + .toEqual({ modelUsd: null, summedAttemptMs: null }); +}); + +it("computes distributions within one cohort and does not sum P95s", () => { + const snapshot = structuredClone(measurements); + const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!; + const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!; + snapshot.samples = Array.from({ length: 20 }, (_, index) => ({ ...sample, attemptId: `attempt-${index}`, + knownModelUsd: String(index + 1), attemptDurationMs: (index + 1) * 1000 })); + const forecast = scheduledForecast([cell], models, snapshot); + expect(forecast.cells[0]!.compatibleSuccessful).toEqual({ samples: 20, + modelUsd: { mean: "10.500000000", median: "10.500000000", p95: "19.000000000" }, + attemptMs: { mean: 10500, median: 10500, p95: 19000 } }); + expect(forecast.selectionP95).toBeNull(); + snapshot.samples.push({ ...sample, attemptId: "other-cohort", definitionHash: "a".repeat(64) }); + const mixed = scheduledForecast([cell], models, snapshot); + expect(mixed.cells[0]!.historicalCohorts).toHaveLength(2); + expect(mixed.historicalSuccessfulSelection.modelUsd).toBeNull(); + expect(mixed.cells[0]!.compatibleSuccessful.samples).toBe(20); +}); + +it("rejects duplicate artifacts and invalid measurement provenance", () => { + const snapshot = structuredClone(measurements); + snapshot.samples.push(snapshot.samples[0]!); + expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("Duplicate forecast attempt"); + snapshot.samples.pop(); + snapshot.samples[0]!.resultSha256 = "missing"; + expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("provenance"); +}); + +it("preserves nanodollar costs and genuine measured zero", () => { + const snapshot = structuredClone(measurements); + const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!; + const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!; + snapshot.samples = [{ ...sample, knownModelUsd: "0.000000001", attemptDurationMs: 0 }]; + expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection) + .toEqual({ modelUsd: "0.000000001", summedAttemptMs: 0 }); + snapshot.samples[0]!.knownModelUsd = "0"; + expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection.modelUsd).toBe("0.000000000"); +}); diff --git a/tests/runner-e2e/scheduled-forecast.ts b/tests/runner-e2e/scheduled-forecast.ts new file mode 100644 index 00000000000..7cc1acf23f8 --- /dev/null +++ b/tests/runner-e2e/scheduled-forecast.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { centsToUsd, unitsToCents, usdToUnits } from "../../packages/shared/src/money.js"; +import measurements from "./scheduled-measurements.json" with { type: "json" }; +import type { MatrixExecution } from "./types.js"; +import type { EvalOperationsManifest } from "./operations.js"; + +type Sample = (typeof measurements.samples)[number]; +const cohortKey = (sample: Sample) => JSON.stringify([sample.definitionHash, sample.model, sample.environmentId, + sample.runtimeMode, sample.route, sample.priceVersion]); +const usd = (units: bigint) => centsToUsd(unitsToCents(units)); +const average = (values: bigint[]) => values.length + ? (values.reduce((sum, value) => sum + value, 0n) + BigInt(Math.floor(values.length / 2))) / BigInt(values.length) : null; + +function statistics(values: bigint[]) { + const sorted = [...values].sort((a, b) => a < b ? -1 : a > b ? 1 : 0); + const middle = Math.floor(sorted.length / 2); + return { + count: sorted.length, + mean: average(sorted), + median: sorted.length ? average(sorted.length % 2 ? [sorted[middle]!] : [sorted[middle - 1]!, sorted[middle]!]) : null, + p95: sorted.length >= 20 ? sorted[Math.ceil(sorted.length * 0.95) - 1]! : null, + }; +} + +function summarize(samples: Sample[]) { + const cost = statistics(samples.map(sample => usdToUnits(sample.knownModelUsd))); + const time = statistics(samples.map(sample => BigInt(sample.attemptDurationMs))); + return { + samples: cost.count, + modelUsd: { mean: cost.mean === null ? null : usd(cost.mean), median: cost.median === null ? null : usd(cost.median), + p95: cost.p95 === null ? null : usd(cost.p95) }, + attemptMs: { mean: time.mean === null ? null : Number(time.mean), median: time.median === null ? null : Number(time.median), + p95: time.p95 === null ? null : Number(time.p95) }, + }; +} + +function total(rows: ReturnType[]) { + return { + modelUsd: rows.length && rows.every(row => row.modelUsd.mean !== null) + ? usd(rows.reduce((sum, row) => sum + usdToUnits(row.modelUsd.mean!), 0n)) : null, + summedAttemptMs: rows.length && rows.every(row => row.attemptMs.mean !== null) + ? rows.reduce((sum, row) => sum + row.attemptMs.mean!, 0) : null, + }; +} + +export function scheduledForecast( + executions: MatrixExecution[], models: EvalOperationsManifest["models"], snapshot = measurements, +) { + const seen = new Set(); + for (const sample of snapshot.samples) { + if (seen.has(sample.attemptId)) throw new Error("Duplicate forecast attempt"); + seen.add(sample.attemptId); + if (!Number.isSafeInteger(sample.attemptDurationMs) || sample.attemptDurationMs < 0 || usdToUnits(sample.knownModelUsd) < 0n + || !/^[0-9a-f]{40}$/.test(sample.sourceRevision) + || [sample.definitionHash, sample.resultSha256, sample.ledgerSha256, sample.ledgerFinalHash].some(hash => !/^[0-9a-f]{64}$/.test(hash))) { + throw new Error("Invalid forecast measurement or provenance"); + } + } + if (new Set(executions.map(execution => execution.id)).size !== executions.length) throw new Error("Duplicate forecast execution"); + const cells = executions.map(execution => { + const retained = snapshot.samples.filter(sample => sample.executionId === execution.id); + const cohorts = [...new Set(retained.map(cohortKey))].map(key => ({ + key, ...summarize(retained.filter(sample => cohortKey(sample) === key)), + })); + const samples = retained.map(sample => { + const mismatches = [ + sample.definitionHash === execution.suiteDefinitionHash ? null : "definition", + sample.model === execution.profile.model ? null : "model", + sample.environmentId === execution.environment.id ? null : "environment", + sample.runtimeMode === execution.profile.expectedRuntimeMode ? null : "runtime", + sample.route === "braintrust" ? null : "route", + models.some(model => model.price.model === sample.model && model.price.version === sample.priceVersion) ? null : "pricing", + ].filter(value => value !== null); + return { ...sample, mismatches }; + }); + return { executionId: execution.id, currentDefinitionHash: execution.suiteDefinitionHash, + historicalSuccessful: summarize(cohorts.length === 1 ? retained : []), historicalCohorts: cohorts, + compatibleSuccessful: summarize(samples.filter(sample => sample.mismatches.length === 0)), samples }; + }); + return { + measurementSnapshotSha256: createHash("sha256").update(JSON.stringify(snapshot)).digest("hex"), + measuredHost: snapshot.host, costStatus: snapshot.costStatus, sampleScope: snapshot.sampleScope, + historicalSuccessfulSelection: total(cells.map(cell => cell.historicalSuccessful)), + compatibleSuccessfulSelection: total(cells.map(cell => cell.compatibleSuccessful)), + hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, + cells, limitations: [...snapshot.limitations, + "Compatibility compares definition, model, environment, runtime, route and price version; source revisions remain explicit", + "Per-cell P95 uses nearest rank only with at least 20 samples; it is descriptive, not a confidence bound", + "Summed attempt time is serial attempt work, not campaign wall time, summed agent time or billed runner time", + "Historical successful samples do not estimate failure frequency; every failure remains in campaign accounting"], + }; +} diff --git a/tests/runner-e2e/scheduled-measurements.json b/tests/runner-e2e/scheduled-measurements.json new file mode 100644 index 00000000000..afd9b43aaf1 --- /dev/null +++ b/tests/runner-e2e/scheduled-measurements.json @@ -0,0 +1,84 @@ +{ + "version": 1, + "recordedAt": "2026-10-21T18:13:01.055Z", + "costStatus": "estimated", + "sampleScope": "successful local native qualification attempts only", + "host": "linux-arm64-local-container", + "samples": [ + { + "executionId": "feedback-persistence.runner-codex.local.complaint-and-suggestion", + "attemptId": "96850eb44e6a9f9856fcf24a40fb4903", + "sourceRevision": "276b8ff966c177b8cb34af12479915d9c8f2c9a2", + "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92", + "knownModelUsd": "0.189173400", + "attemptDurationMs": 26770, + "model": "gpt-5.6-sol", + "environmentId": "local", + "runtimeMode": "native", + "route": "braintrust", + "priceVersion": "openai-standard-2026-10-20", + "measuredAt": "2026-10-21T16:32:44.767Z", + "resultSha256": "e1748f875784065f2d40ebd7cc83cb9c3f84b1ce8dd05629482a97e818608188", + "ledgerSha256": "b5f42a64e8a72c5ae23c4dbf33fda1c561a7e2833d499e86b63627166144f1ef", + "ledgerFinalHash": "f06b687f8063926b77c0378946f1d6a9bfb718e1d8b9ca7d6e2aa344187be2dc" + }, + { + "executionId": "core-compatibility.runner-acpx-claude.local.message-marker", + "attemptId": "8da10d462119f3cf6bb3a4669c43b6b5", + "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", + "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa", + "knownModelUsd": "0.123679200", + "attemptDurationMs": 34495, + "model": "claude-sonnet-5", + "environmentId": "local", + "runtimeMode": "native", + "route": "braintrust", + "priceVersion": "anthropic-standard-2026-10-20", + "measuredAt": "2026-10-21T16:54:26.516Z", + "resultSha256": "89aac65b9a703d812a5a3c83c51732031e9b6fcadc6bac3c14a86c419f169d7f", + "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", + "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + }, + { + "executionId": "core-compatibility.runner-codex.local.message-marker", + "attemptId": "03720091468148fe7a0d2b1fa68d3966", + "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", + "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa", + "knownModelUsd": "0.174741200", + "attemptDurationMs": 27542, + "model": "gpt-5.6-sol", + "environmentId": "local", + "runtimeMode": "native", + "route": "braintrust", + "priceVersion": "openai-standard-2026-10-20", + "measuredAt": "2026-10-21T16:53:20.764Z", + "resultSha256": "188a0655a7aebe37ec21c582e6b4afef1633210fa8f93e1049b9db0078657c4d", + "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", + "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + }, + { + "executionId": "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", + "attemptId": "3849bc83e59ccddfa964120053d3428c", + "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", + "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92", + "knownModelUsd": "0.303945600", + "attemptDurationMs": 38262, + "model": "claude-sonnet-5", + "environmentId": "local", + "runtimeMode": "native", + "route": "braintrust", + "priceVersion": "anthropic-standard-2026-10-20", + "measuredAt": "2026-10-21T16:52:21.240Z", + "resultSha256": "c086648b3dfc67a6d6ee04bb66a1914b3db69f9d40457198e2ce53c3a752ac24", + "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", + "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + } + ], + "limitations": [ + "One successful sample per cell; no measured slow-tail distribution or predicted failure rate", + "Gateway estimates include setup requests; historical phase labels may be inaccurate", + "Attempt time includes setup and teardown; it excludes queue, dependency installation and compilation", + "No hosted timing qualification or complete infrastructure, storage and subscription allocation", + "No change to the conservative per-request reservations or approved budgets" + ] +} diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts new file mode 100644 index 00000000000..72ebe6d4d10 --- /dev/null +++ b/tests/runner-e2e/scheduled-operations.test.ts @@ -0,0 +1,77 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { prepareScheduledCampaign, scheduledCadence, scheduledManifest, scheduledMatrix, scheduledSelection, validateScheduledBudget } from "./scheduled-operations.js"; +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { runnerMatrix } from "./catalog.js"; +import { evalRequestReservation } from "./eval-gateway.js"; +import { EVAL_OVERHEAD_COMPONENTS, type EvalOperatingOverhead } from "./operating-overhead.js"; + +const overhead: EvalOperatingOverhead = { version: 1, reference: "fixture-only", runner: "github-hosted", + verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z", + components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: `fixture-${component}` })) }; + +const input = { cadence: "daily" as const, campaignId: "scheduled-fixture", ledgerPath: "/tmp/scheduled-fixture/ledger.jsonl", + projectId: "11111111-1111-4111-8111-111111111111", now: Date.parse("2026-10-21T00:00:00Z"), runAttempt: "1", overhead }; + +it.each(["daily", "weekly"] as const)("runs the exact %s selection through one shared budget and catalog job", cadence => { + const manifest = scheduledManifest({ ...input, cadence }); + const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix); + expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort()); + expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxConcurrent: 1, maxAttemptsPerExecution: 1 }); + expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000); + expect(scheduledMatrix(cadence).include).toHaveLength(1); + expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4); + expect(manifest.models.every(model => Number(evalRequestReservation({ ...model, mode: "braintrust", apiKey: "fixture" })) <= Number(manifest.limits.maxUsd))).toBe(true); +}); + +it("does not reduce Codex's verified context ceiling to fit the daily allowance", () => { + const codex = scheduledManifest({ ...input, cadence: "weekly" }).models.find(row => row.provider === "openai")!; + expect(codex.maximumInputTokens).toBe(1_050_000); + expect(Number(evalRequestReservation({ ...codex, mode: "braintrust", apiKey: "fixture" }))).toBeGreaterThan(20); + expect(scheduledManifest(input).models.map(model => model.provider)).toEqual(["anthropic"]); +}); + +it.each(["2", "0", "", "01"])("refuses workflow attempt %s before creating a fresh allowance", runAttempt => { + expect(() => scheduledManifest({ ...input, runAttempt })).toThrow("fresh allowance"); +}); + +it.each(["2026-10-19T23:59:59Z", "2026-11-19T00:00:00Z"])("rejects pricing outside its reviewed window at %s", time => { + expect(() => scheduledManifest({ ...input, now: Date.parse(time) })).toThrow("pricing needs review"); +}); + +it("rejects unknown triggers and maps the reviewed schedule expressions", () => { + expect(scheduledCadence("schedule", "47 8 * * *")).toBe("daily"); + expect(scheduledCadence("schedule", "47 9 * * 0")).toBe("weekly"); + expect(scheduledCadence("workflow_dispatch", "", "manual")).toBeUndefined(); + expect(() => scheduledCadence("schedule", "* * * * *")).toThrow(); + expect(() => scheduledCadence("pull_request", "", "daily")).toThrow(); +}); + +it("fails closed before paid work if disabled, missing project identity, or preflight is repeated", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-schedule-")); + const env = { EVAL_EVENT: "schedule", EVAL_SCHEDULE: "47 8 * * *", EVAL_OPERATIONS_ENABLED: "true", + GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", GITHUB_WORKSPACE: directory, + EVAL_OPERATING_OVERHEAD_JSON: JSON.stringify(overhead), EVAL_TEST_RUNNER: "ubuntu-latest", + PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: input.projectId }; + try { + await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATIONS_ENABLED: "false" }, input.now)).rejects.toThrow("not enabled"); + await expect(prepareScheduledCampaign({ ...env, EVAL_OPERATING_OVERHEAD_JSON: "" }, input.now)).rejects.toThrow("Missing reviewed"); + await expect(prepareScheduledCampaign({ ...env, PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID: "" }, input.now)).rejects.toThrow("UUID"); + const prepared = await prepareScheduledCampaign(env, input.now); + expect(prepared.manifest.limits.campaignId).toBe(prepared.campaignId); + const forecast = JSON.parse(await readFile(path.join(path.dirname(prepared.manifestPath), "forecast.json"), "utf8")); + expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.427624800", summedAttemptMs: 72757 }); + expect(forecast.measurements.hostedAttemptMs).toBeNull(); + expect(forecast.maxUsd).toBe("20"); + await expect(prepareScheduledCampaign(env, input.now)).rejects.toThrow("EEXIST"); + } finally { await rm(directory, { recursive: true, force: true }); } +}); + +it("rejects an overhead allowance that cannot leave room for a full selected request before emitting the matrix", () => { + const expensive = structuredClone(overhead); + expensive.components[0]!.maxUsd = "13"; + expect(() => validateScheduledBudget({ ...input, overhead: expensive })).toThrow("insufficient allowance"); + expect(validateScheduledBudget({ ...input, overhead })).toMatchObject({ executionIds: scheduledSelection("daily") }); +}); diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts new file mode 100644 index 00000000000..84bb950e085 --- /dev/null +++ b/tests/runner-e2e/scheduled-operations.ts @@ -0,0 +1,121 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { runnerMatrix } from "./catalog.js"; +import { validateEvalOperationsManifest, type EvalOperationsManifest } from "./operations.js"; +import { evalRequestReservation } from "./eval-gateway.js"; +import prices from "./scheduled-prices.json" with { type: "json" }; +import { scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { usdToUnits } from "../../packages/shared/src/money.js"; +import { scheduledForecast } from "./scheduled-forecast.js"; + +export type EvalCadence = "daily" | "weekly"; +export const EVAL_SCHEDULES = { daily: "47 8 * * *", weekly: "47 9 * * 0" } as const; +const budget = (cadence: EvalCadence) => cadence === "daily" ? "20" : "100"; +const marker = (profile: string) => `core-compatibility.${profile}.local.message-marker`; +const feedback = (profile: string) => `feedback-persistence.${profile}.local.complaint-and-suggestion`; + +export function scheduledSelection(cadence: EvalCadence) { + if (!["daily", "weekly"].includes(cadence)) throw new Error("Unknown eval cadence"); + return [marker("runner-acpx-claude"), feedback("runner-acpx-claude"), + ...(cadence === "weekly" ? [marker("runner-codex"), feedback("runner-codex")] : [])]; +} + +export function scheduledCadence(event: string, schedule: string, requested = "manual"): EvalCadence | undefined { + if (event === "schedule") { + const found = Object.entries(EVAL_SCHEDULES).find(([, expression]) => expression === schedule); + if (!found) throw new Error("Unrecognized paid eval schedule"); + return found[0] as EvalCadence; + } + if (event !== "workflow_dispatch" || !["manual", "daily", "weekly"].includes(requested)) throw new Error("Unrecognized eval trigger"); + return requested === "manual" ? undefined : requested as EvalCadence; +} + +export function validateScheduledBudget(input: { cadence: EvalCadence; now: number; overhead: EvalOperatingOverhead }) { + if (!Number.isFinite(input.now) || input.now < Date.parse(prices.verifiedAt) || input.now >= Date.parse(prices.expiresAt)) { + throw new Error("Scheduled eval pricing needs review before paid admission"); + } + const executionIds = scheduledSelection(input.cadence); + const executions = executionIds.map(id => { + const execution = runnerMatrix.find(row => row.id === id); + if (!execution) throw new Error(`Scheduled execution is absent: ${id}`); + return execution; + }); + const models = prices.models.filter(model => executions.some(execution => execution.profile.model === model.price.model)) as EvalOperationsManifest["models"]; + if (executions.some(execution => !models.some(model => model.price.model === execution.profile.model + && model.provider === (execution.profile.credential === "ANTHROPIC_API_KEY" ? "anthropic" : "openai")))) { + throw new Error("Missing exact scheduled model pricing"); + } + const reserved = validateOperatingOverhead(input.overhead, budget(input.cadence)); + if (input.now < Date.parse(input.overhead.verifiedAt) || input.now + 30 * 60_000 >= Date.parse(input.overhead.expiresAt)) { + throw new Error("Operating overhead allowance does not cover the campaign window"); + } + if (models.some(model => reserved + usdToUnits(evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" })) > usdToUnits(budget(input.cadence)))) { + throw new Error("Operating overhead leaves insufficient allowance for the selected model"); + } + return { executionIds, executions, models }; +} + +export function scheduledManifest(input: { + cadence: EvalCadence; campaignId: string; ledgerPath: string; projectId: string; now: number; runAttempt: string; + overhead: EvalOperatingOverhead; +}): EvalOperationsManifest { + if (input.runAttempt !== "1") throw new Error("Paid campaign reruns require the retained journal; a fresh allowance is refused"); + const { executionIds, executions, models } = validateScheduledBudget(input); + const manifest: EvalOperationsManifest = { + version: 1, limits: { campaignId: input.campaignId, maxUsd: budget(input.cadence), overhead: input.overhead, + deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 60 : 120, + maxConcurrent: 1, maxAttemptsPerExecution: 1 }, + ledgerPath: input.ledgerPath, executionIds, maxAttemptsPerExecution: 1, route: "braintrust", projectId: input.projectId, + models, + }; + validateEvalOperationsManifest(manifest, executions, 0); + return manifest; +} + +export function scheduledMatrix(cadence: EvalCadence) { + return { include: [{ executionId: "scheduled-campaign", suiteId: "core-compatibility", profileId: "runner-acpx-claude", + credentialName: "", environmentId: "local", caseId: "scheduled-campaign", timeoutMinutes: 60, needsDaytona: false }], + needsDaytona: false, executionIds: scheduledSelection(cadence) }; +} + +export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Date.now()) { + const cadence = scheduledCadence(env.EVAL_EVENT ?? "", env.EVAL_SCHEDULE ?? "", env.EVAL_CADENCE); + if (!cadence || env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled"); + if (!/^[1-9][0-9]*$/.test(env.GITHUB_RUN_ID ?? "") || !env.GITHUB_WORKSPACE) throw new Error("Missing workflow campaign identity"); + const campaignId = `gha-${env.GITHUB_RUN_ID}-1-scheduled-campaign`; + const directory = path.join(env.GITHUB_WORKSPACE, "tests/runner-e2e/results/scheduled"); + const manifest = scheduledManifest({ cadence, campaignId, ledgerPath: path.join(directory, "ledger.jsonl"), + projectId: env.PAPERCLIP_E2E_BRAINTRUST_PROJECT_ID ?? "", now, runAttempt: env.GITHUB_RUN_ATTEMPT ?? "", + overhead: scheduledOperatingOverhead(env.EVAL_OPERATING_OVERHEAD_JSON, env.EVAL_TEST_RUNNER, budget(cadence), now) }); + await mkdir(directory, { recursive: true, mode: 0o700 }); + const manifestPath = path.join(directory, "manifest.json"); + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600, flag: "wx" }); + const forecast = { cadence, executionIds: manifest.executionIds, maxUsd: manifest.limits.maxUsd, + overhead: manifest.limits.overhead, budgetScope: "model requests plus declared operating overhead reservations", + measurements: scheduledForecast(runnerMatrix.filter(execution => manifest.executionIds.includes(execution.id)), manifest.models), + basis: "Retained local pilot measurements; successful samples and matching configurations are reported separately", + uncertainty: "Incomplete provider receipts and no invoice reconciliation; estimates are not a guarantee", + maximumRequestReservations: manifest.models.map(model => ({ model: model.price.model, + usd: evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" }) })), + maximumMonthlyTotalAllowanceUsd: "1120", pricingExpiresAt: prices.expiresAt }; + await writeFile(path.join(directory, "forecast.json"), `${JSON.stringify(forecast, null, 2)}\n`, { mode: 0o600, flag: "wx" }); + return { manifest, manifestPath, campaignId, cadence }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const command = process.argv[2]; + if (command === "matrix") { + const cadence = scheduledCadence(process.env.EVAL_EVENT ?? "", process.env.EVAL_SCHEDULE ?? "", process.env.EVAL_CADENCE); + if (!cadence || process.env.EVAL_OPERATIONS_ENABLED !== "true") throw new Error("Bounded eval operations are not enabled"); + const now = Date.now(); + const overhead = scheduledOperatingOverhead(process.env.EVAL_OPERATING_OVERHEAD_JSON, process.env.EVAL_TEST_RUNNER, budget(cadence), now); + validateScheduledBudget({ cadence, overhead, now }); + console.log(JSON.stringify({ ...scheduledMatrix(cadence), overhead })); + } else if (command === "prepare") { + const prepared = await prepareScheduledCampaign(process.env); + if (!process.env.GITHUB_ENV) throw new Error("Missing workflow environment file"); + const prior = await readFile(process.env.GITHUB_ENV, "utf8"); + await writeFile(process.env.GITHUB_ENV, `${prior}PAPERCLIP_E2E_OPERATIONS_MANIFEST=${prepared.manifestPath}\nPAPERCLIP_E2E_CAMPAIGN_ID=${prepared.campaignId}\nPAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID=${prepared.campaignId}\n`); + } else throw new Error("Usage: scheduled-operations.ts "); +} diff --git a/tests/runner-e2e/scheduled-prices.json b/tests/runner-e2e/scheduled-prices.json new file mode 100644 index 00000000000..c1d437be773 --- /dev/null +++ b/tests/runner-e2e/scheduled-prices.json @@ -0,0 +1,51 @@ +{ + "version": 1, + "verifiedAt": "2026-10-20T00:00:00Z", + "expiresAt": "2026-11-19T00:00:00Z", + "models": [ + { + "provider": "openai", + "maximumInputTokens": 1050000, + "maximumOutputTokens": 16384, + "timeoutMs": 180000, + "price": { + "provider": "openai", + "model": "gpt-5.6-sol", + "version": "openai-standard-2026-10-20", + "sourceUrl": "https://developers.openai.com/api/docs/pricing", + "inputUsdPerMillion": "4", + "cachedReadUsdPerMillion": "0.4", + "outputUsdPerMillion": "20", + "cacheWrite30mUsdPerMillion": "5", + "cacheWrite12hUsdPerMillion": "10", + "longContext": { + "aboveInputTokens": 272000, + "rates": { + "inputUsdPerMillion": "8", + "cachedReadUsdPerMillion": "0.8", + "outputUsdPerMillion": "30", + "cacheWrite30mUsdPerMillion": "10", + "cacheWrite12hUsdPerMillion": "20" + } + } + } + }, + { + "provider": "anthropic", + "maximumInputTokens": 1000000, + "maximumOutputTokens": 16384, + "timeoutMs": 180000, + "price": { + "provider": "anthropic", + "model": "claude-sonnet-5", + "version": "anthropic-standard-2026-10-20", + "sourceUrl": "https://platform.claude.com/docs/en/models/sonnet-5/overview", + "inputUsdPerMillion": "2", + "cachedReadUsdPerMillion": "0.2", + "outputUsdPerMillion": "10", + "cacheWrite5mUsdPerMillion": "2.5", + "cacheWrite1hUsdPerMillion": "4" + } + } + ] +} diff --git a/tests/runner-e2e/select-rerun-artifacts.ts b/tests/runner-e2e/select-rerun-artifacts.ts index 37458815323..9dab4141800 100644 --- a/tests/runner-e2e/select-rerun-artifacts.ts +++ b/tests/runner-e2e/select-rerun-artifacts.ts @@ -1,4 +1,4 @@ -import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import type { RunnerE2EResult } from "./types.js"; @@ -307,6 +307,11 @@ export async function selectRerunArtifacts(input: SelectRerunArtifactsInput) { }); selections.push({ executionId, workflowAttempt, artifactName }); } + await mkdir(input.selectedRoot, { recursive: true }); + await writeFile(path.join(input.selectedRoot, "selected-artifacts.json"), JSON.stringify({ + version: 1, workflowRunId: runId, workflowRunAttempt: currentAttempt, + sourceSha: input.sourceSha, sourceRef: input.sourceRef, workflowRunUrl: input.workflowRunUrl, selections, + })); return selections; } diff --git a/tests/runner-e2e/server.ts b/tests/runner-e2e/server.ts index 05a82316c1b..8956ac3f5aa 100644 --- a/tests/runner-e2e/server.ts +++ b/tests/runner-e2e/server.ts @@ -1,5 +1,9 @@ import { parseFinalizationCrashBinding, readFinalizationCrashBoundary, type FinalizationCrashBinding } from "./finalization-crash-boundary.js"; import { parseRestartRunnerIdentity, type RestartRunnerIdentity } from "./process-tree-owner.js"; +import { recoverInterruptedFeedback } from "./feedback-recovery.js"; +import { normalizedSecrets } from "./redaction.js"; +import { CREDENTIAL_NAMES } from "./types.js"; +import { EVAL_OBSERVABILITY_SECRET_NAMES } from "./braintrust.js"; import { usesInstalledCli, verifyInstalledCli } from "./installed-cli.js"; import { runnerMatrix } from "./catalog.js"; import { createRunnerE2EServerStopper, runnerE2EServerDetached } from "./server-stop.js"; @@ -117,6 +121,43 @@ let child: ChildProcess | null = null; let unexpectedChildFailure: Error | null = null; let shutdownSignal: NodeJS.Signals | null = null; let activeRestartRequestId: string | null = null; +let feedbackRecovery: Promise | undefined; + +function recoverFeedbackBeforeShutdown() { + return feedbackRecovery ??= (async () => { + const privateDir = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR; + const campaignId = process.env.PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID; + const attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID; + if (!privateDir || !campaignId || !attemptId || process.env.PAPERCLIP_RUNNER_E2E_PUBLIC_MCP === "1") return; + const deadlineAt = Date.now() + 5_000; + const secrets = normalizedSecrets([...CREDENTIAL_NAMES, ...EVAL_OBSERVABILITY_SECRET_NAMES, + "PAPERCLIP_E2E_GATEWAY_TOKEN", "PAPERCLIP_AGENT_JWT_SECRET", "PAPERCLIP_DECISION_SIGNING_SECRET", + "PAPERCLIP_TOOL_ACTION_SIGNING_SECRET", "BETTER_AUTH_SECRET"].map(name => process.env[name])); + for (const executionId of executionIds) { + if (Date.now() >= deadlineAt) break; + const execution = runnerMatrix.find(row => row.id === executionId); + if (!execution) continue; + try { + await recoverInterruptedFeedback({ identity: { campaignId, attemptId, executionId }, + caseDirectory: path.join(privateDir, "cases", execution.task.id), secrets, deadlineAt, + api: { async get(endpoint: string, options?: { timeout: number }): Promise { + const base = `http://127.0.0.1:${port}`; + const url = new URL(endpoint, base); + if (url.origin !== base || !url.pathname.startsWith("/api/")) throw new Error("Invalid feedback recovery endpoint"); + const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(Math.max(1, Math.min(options?.timeout ?? 1_000, deadlineAt - Date.now()))) }); + if (!response.ok) { await response.body?.cancel(); throw new Error("Feedback recovery read failed"); } + return response.json() as Promise; + } }, + }); + } catch { appendLog("Feedback recovery checkpoint unavailable; retain private recovery state.\n"); } + } + })(); +} + +async function stopWithFeedback(candidate: ChildProcess, signal: NodeJS.Signals) { + await recoverFeedbackBeforeShutdown(); + await stopServer(candidate, signal); +} function appendLog(message: string) { process.stderr.write(message); @@ -186,7 +227,7 @@ async function startServer() { // that race create an unowned replacement server. if (shutdownSignal) { // The main/error path awaits this same promise and reports any failure. - void stopServer(candidate, shutdownSignal).catch(() => {}); + void stopWithFeedback(candidate, shutdownSignal).catch(() => {}); } await stopServer.watch(candidate); return candidate; @@ -344,9 +385,7 @@ for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { if (shutdownSignal) return; shutdownSignal = signal; if (!child) return; - // Begin immediately, including during a health/restart wait. The final - // cleanup joins this promise instead of sending a second graceful signal. - void stopServer(child, signal).catch(() => {}); + void stopWithFeedback(child, signal).catch(() => {}); }); } @@ -374,6 +413,7 @@ async function supervise() { await delay(200); } + await recoverFeedbackBeforeShutdown(); await stopServer.stopAll(shutdownSignal ?? "SIGTERM"); } @@ -394,6 +434,7 @@ try { } } try { + await recoverFeedbackBeforeShutdown(); await stopServer.stopAll(); } catch (stopError) { appendLog(`Failed to stop Paperclip after supervisor failure: ${stopError instanceof Error ? stopError.message : String(stopError)}\n`); diff --git a/tests/runner-e2e/trace-recovery.test.ts b/tests/runner-e2e/trace-recovery.test.ts new file mode 100644 index 00000000000..da24f6f8243 --- /dev/null +++ b/tests/runner-e2e/trace-recovery.test.ts @@ -0,0 +1,46 @@ +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { expect, it, vi } from "vitest"; +import { EvalAttemptTrace, finalizeEvalAttemptTrace } from "./braintrust.js"; +import { readInterruptedAttemptTrace } from "./trace-recovery.js"; + +it("finalizes a lost worker's original trace while rejecting unrelated or malformed snapshots", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "eval-trace-recovery-")); + const projectId = "11111111-1111-4111-8111-111111111111"; + const companyId = "22222222-2222-4222-8222-222222222222"; + const identity = { campaignId: "campaign", attemptId: "attempt", executionId: "execution", provider: "acpx", model: "claude", environment: "local" }; + const config = { projectId, apiKey: "synthetic-canary" }; + const trace = new EvalAttemptTrace(config, identity).snapshot(); + const saved = { ...trace, companyId, nativeConfiguration: "configured", privateCanary: "must-not-copy", finalization: { status: "accepted" } }; + const fixture = { companyId, executionId: identity.executionId }; + const snapshot = path.join(root, "snapshots/braintrust-trace.json"); + try { + await mkdir(path.join(root, "snapshots")); + await writeFile(path.join(root, "snapshots/fixtures.json"), JSON.stringify(fixture)); + await writeFile(snapshot, JSON.stringify(saved)); + const recovered = await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId }); + expect(recovered).toMatchObject({ traceId: trace.traceId, spanId: trace.spanId, status: "pending", verification: "unverified" }); + expect(recovered).not.toHaveProperty("privateCanary"); + expect(recovered).not.toHaveProperty("finalization"); + const transport = vi.fn(async (_url, init) => { + const body = init!.body as Buffer; + expect(body.includes(Buffer.from(trace.traceId, "hex"))).toBe(true); + expect(body.includes(Buffer.from(trace.spanId, "hex"))).toBe(true); + expect(body.toString()).not.toContain("must-not-copy"); + return new Response(null, { status: 200 }); + }); + expect(await finalizeEvalAttemptTrace(config, { receipt: recovered!, identity, + provenance: { suiteId: "suite", suiteDefinitionHash: "a".repeat(64), profileId: "profile", caseId: "case", sourceRevision: "b".repeat(40) }, + result: { startedAt: new Date(Date.now() - 1000).toISOString(), status: "failed", cleanup: "failed", failureClass: "cleanup_failure" }, + }, transport)).toMatchObject({ status: "accepted", outcome: "failed", verification: "unverified" }); + expect(transport).toHaveBeenCalledOnce(); + expect(JSON.parse(await readFile(snapshot, "utf8"))).toEqual(saved); + expect(await readInterruptedAttemptTrace(root, { projectId: companyId, executionId: identity.executionId })).toBeUndefined(); + expect(await readInterruptedAttemptTrace(root, { projectId, executionId: "other" })).toBeUndefined(); + for (const delta of [{ companyId: projectId }, { spanId: "0".repeat(16) }, { traceId: "invalid" }, { revision: "invalid" }, { nativeConfiguration: "unknown" }]) { + await writeFile(snapshot, JSON.stringify({ ...saved, ...delta })); + expect(await readInterruptedAttemptTrace(root, { projectId, executionId: identity.executionId })).toBeUndefined(); + } + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/trace-recovery.ts b/tests/runner-e2e/trace-recovery.ts new file mode 100644 index 00000000000..2f9feef9eee --- /dev/null +++ b/tests/runner-e2e/trace-recovery.ts @@ -0,0 +1,23 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { exportBraintrustAttemptParent, type EvalTraceReceipt } from "./braintrust.js"; + +export async function readInterruptedAttemptTrace(caseDirectory: string, expected: { + projectId: string; executionId: string; +}): Promise { + try { + const snapshots = path.join(caseDirectory, "snapshots"); + const [fixture, trace] = await Promise.all(["fixtures.json", "braintrust-trace.json"].map(async name => + JSON.parse(await readFile(path.join(snapshots, name), "utf8")))); + const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + if (fixture.executionId !== expected.executionId || !uuid.test(fixture.companyId) + || trace.projectId !== expected.projectId || trace.companyId !== fixture.companyId + || trace.revision !== undefined && !uuid.test(trace.revision) + || !["pending", "accepted", "failed"].includes(trace.status) + || !["not_started", "configured", "failed"].includes(trace.nativeConfiguration)) return; + exportBraintrustAttemptParent(trace); + return { projectId: trace.projectId, traceId: trace.traceId, spanId: trace.spanId, + companyId: trace.companyId, ...(trace.revision ? { revision: trace.revision } : {}), + status: trace.status, nativeConfiguration: trace.nativeConfiguration, verification: "unverified" }; + } catch { return; } +} diff --git a/tests/runner-e2e/types.ts b/tests/runner-e2e/types.ts index df280261e58..f54348ef351 100644 --- a/tests/runner-e2e/types.ts +++ b/tests/runner-e2e/types.ts @@ -244,7 +244,7 @@ export type RunnerE2ECostStatus = export interface RunnerE2ERuntimeUsage { provider: RunnerEnvironmentId; /** Sum of the selected Paperclip heartbeat-run spans. */ - agentRunDurationMs: number; + agentRunDurationMs: number | null; /** Sum of provider lease windows when the environment exposes leases. */ leaseDurationMs: number | null; leaseCount: number; @@ -267,15 +267,17 @@ export interface RunnerE2EBillingSummary { runCount: number; runsWithTokenUsage: number; runsWithReportedCost: number; + runsWithEstimatedCost?: number; + runsWithPartialReportedCost?: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; totalTokens: number; reportedCostUsd: number; - costStatus: Exclude; + estimatedCostUsd?: number; + costStatus: Exclude; }; runtime: RunnerE2ERuntimeUsage; - /** Provider-reported model spend only; never includes unknown/unpriced runs. */ reportedCostUsd: number; /** Public-list-price estimate for metered execution infrastructure. */ estimatedRuntimeCostUsd: number; @@ -288,6 +290,7 @@ export interface RunnerE2EBillingSummary { export interface RunnerE2EResult { schema: "paperclip.runner-e2e.result/v1" | "paperclip.runner-e2e.result/v2"; + accountingIdentity?: { campaignId: string; attemptId: string }; executionId: string; suiteId?: string; suiteDefinitionHash?: string; @@ -338,9 +341,12 @@ export interface RunnerE2EResult { startedAt: string; finishedAt: string; durationMs: number; + workerTiming?: import("./worker-timing.js").WorkerTiming; usage?: Record | null; runtimeUsage?: RunnerE2ERuntimeUsage; billing?: RunnerE2EBillingSummary; + feedback?: import("./feedback.js").FeedbackSummary; + tracing?: import("./braintrust.js").EvalTraceReceipt; matcherResults?: Array<{ matcher: Matcher; passed: boolean; @@ -393,15 +399,39 @@ export interface RunnerE2EAggregateBillingSummary { assistant?: { requests: number; inputTokens: number; outputTokens: number; cachedInputTokens: number; estimatedCostUsd: number }; judge?: RunnerE2EJudgeBillingSummary; testCount: number; - agentRunDurationMs: number; - leaseDurationMs: number; + agentRunDurationMs: number | null; + leaseDurationMs: number | null; llm: RunnerE2EBillingSummary["llm"]; reportedLlmCostUsd: number; + estimatedLlmCostUsd?: number; estimatedRuntimeCostUsd: number; observedAndEstimatedCostUsd: number | null; testsWithCompleteBilling: number; } +export interface RunnerE2EAttemptReceipt { + id: string; + executionId: string; + attempt: number; + startedAt: string; + finishedAt: string; + durationMs: number; + workerTiming?: import("./worker-timing.js").WorkerTiming; + status: RunnerE2EResult["status"]; + cleanup: RunnerE2EResult["cleanup"]; + billing: RunnerE2EBillingSummary; + feedback?: import("./feedback.js").FeedbackSummary; + tracing?: import("./braintrust.js").EvalTraceReceipt; +} + +export interface RunnerE2ECampaignAccounting { + version: 1; + scope: "retained_attempts" | "selected_results"; + attempts: RunnerE2EAttemptReceipt[]; + missingAttempts: number; + excludedArtifacts: number; +} + export interface RunnerE2ECampaign { schema: "paperclip.runner-e2e.campaign/v2"; campaignId: string; @@ -427,6 +457,10 @@ export interface RunnerE2ECampaign { sourceUrl: string; }>; billing: RunnerE2EAggregateBillingSummary; + accounting?: RunnerE2ECampaignAccounting; + operations?: import("./operations-report.js").EvalOperationsReport; + feedbackReview?: import("./feedback-dispositions.js").CampaignFeedbackReview; + braintrustExperiment?: import("./braintrust-experiment.js").BraintrustExperimentReceipt; suites: RunnerE2ESuiteSummary[]; results: RunnerE2EResult[]; } diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 7d0dcdbc0fe..448f2df9235 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ name: "runner-e2e-support", include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs"], + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs"], testTimeout: 30_000, }, }); diff --git a/tests/runner-e2e/web-server-command.ts b/tests/runner-e2e/web-server-command.ts index 7735119d246..df8425695a1 100644 --- a/tests/runner-e2e/web-server-command.ts +++ b/tests/runner-e2e/web-server-command.ts @@ -1,5 +1,7 @@ import path from "node:path"; +export const runnerE2EPlaywrightShutdownSignal = "SIGINT" as const; + // Let the restart-aware supervisor reap Paperclip and finish its log streams. // Its stop path allows 30 seconds for SIGTERM and 5 seconds for SIGKILL. export const runnerE2EWebServerGracefulShutdown = { @@ -22,11 +24,6 @@ export function runnerE2ETypeScriptProcessArgs(repositoryRoot: string, entry: st return ["--import", path.join(repositoryRoot, "cli/node_modules/tsx/dist/loader.mjs"), entry, ...args]; } -// pnpm's generated playwright bin shim adds NODE_PATH, even when the caller -// rejected ambient injection. The installed lane invokes the public JS bin -// with the current Node directly, keeping the WebServer environment closed. -export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[], installed: boolean) { - return installed - ? { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] } - : { command: "pnpm", args: ["exec", "playwright", ...args] }; +export function runnerE2EPlaywrightInvocation(repositoryRoot: string, args: string[]) { + return { command: process.execPath, args: [path.join(repositoryRoot, "node_modules/@playwright/test/cli.js"), ...args] }; } diff --git a/tests/runner-e2e/worker-timing.test.ts b/tests/runner-e2e/worker-timing.test.ts new file mode 100644 index 00000000000..2a188ea5822 --- /dev/null +++ b/tests/runner-e2e/worker-timing.test.ts @@ -0,0 +1,127 @@ +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it } from "vitest"; +import { buildCampaignAccounting } from "./attempt-accounting.js"; +import { summarizeWorkerPhaseTiming, workerPhaseTimingText } from "./campaign-timing.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import { createWorkerTiming, recoverWorkerTiming, validateWorkerTiming } from "./worker-timing.js"; +import type { RunnerE2EResult } from "./types.js"; + +const temporary: string[] = []; +afterEach(async () => { await Promise.all(temporary.splice(0).map(dir => rm(dir, { recursive: true, force: true }))); }); +async function fixture() { + const dir = await mkdtemp(path.join(os.tmpdir(), "worker-timing-")); temporary.push(dir); + const result: RunnerE2EResult = { + schema: "paperclip.runner-e2e.result/v2", executionId: "timing-case", suiteId: "timing-fixture", attempt: 1, + accountingIdentity: { campaignId: "timing-campaign", attemptId: "timing-attempt" }, + status: "passed", profileId: "runner-codex", environmentId: "local", caseId: "marker", provider: "codex", + model: "fixture-model", runtimeMode: "native", startedAt: new Date(1000).toISOString(), + finishedAt: new Date(1500).toISOString(), durationMs: 500, cleanup: "passed", + }; + return { file: path.join(dir, "snapshots", "worker-timing.json"), dir, result }; +} + +async function complete(file: string, result: RunnerE2EResult) { + const recorder = await createWorkerTiming(file, result, 1000); + await recorder.enter("execution", 1100); + await recorder.enter("cleanup", 1300); + await recorder.enter("feedback", 1350); + await recorder.enter("cleanup", 1430); + await recorder.enter("finalization", 1470); + return recorder.finish(1500); +} + +describe("worker phase timing", () => { + it("retains and validates all phase boundaries, including both cleanup segments", async () => { + const { file, result } = await fixture(); result.workerTiming = await complete(file, result); + validateRetainedRunnerResult(result); + expect(await recoverWorkerTiming(file, result)).toEqual(result.workerTiming); + const accounting = buildCampaignAccounting({ results: [result], attempts: [result] }); + expect(accounting.attempts[0]?.workerTiming).toEqual(result.workerTiming); + const summary = summarizeWorkerPhaseTiming([result], accounting); + expect(summary).toMatchObject({ completeAttempts: 1, partialAttempts: 0, missingAttempts: 0 }); + expect(summary.phases.map(phase => [phase.phase, phase.totalMs])).toEqual([ + ["setup", 100], ["execution", 200], ["feedback", 80], ["cleanup", 90], ["finalization", 30], + ]); + expect(workerPhaseTimingText(summary)).toContain("Worker cleanup time: 90ms."); + }); + + it("recovers only completed segments after a real worker process is killed", async () => { + const { file, result } = await fixture(); + const loader = createRequire(new URL("../../cli/package.json", import.meta.url)).resolve("tsx"); + const module = pathToFileURL(path.join(import.meta.dirname, "worker-timing.ts")).href; + const program = `import { createWorkerTiming } from ${JSON.stringify(module)}; + const recorder = await createWorkerTiming(${JSON.stringify(file)}, { executionId: "timing-case", attempt: 1 }, 1000); + await recorder.enter("execution", 1200); process.stdout.write("checkpoint-ready"); setInterval(() => {}, 1000);`; + const child = spawn(process.execPath, ["--import", loader, "--input-type=module", "-e", program], { stdio: ["ignore", "pipe", "pipe"] }); + const closed = once(child, "close"); + try { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Worker checkpoint was not written")), 10_000); + child.once("error", error => { clearTimeout(timer); reject(error); }); + child.stdout!.once("data", chunk => { + clearTimeout(timer); String(chunk).includes("checkpoint-ready") ? resolve() : reject(new Error("Unexpected worker output")); + }); + child.once("exit", () => { clearTimeout(timer); reject(new Error("Worker exited before interruption")); }); + }); + child.kill("SIGKILL"); await closed; + result.status = "failed"; result.workerTiming = await recoverWorkerTiming(file, result); + expect(result.workerTiming).toMatchObject({ status: "partial", finishedAt: null, + segments: [{ phase: "setup", finishedAt: new Date(1200).toISOString() }, { phase: "execution", finishedAt: null }] }); + validateRetainedRunnerResult(result); + const summary = summarizeWorkerPhaseTiming([result]); + expect(summary).toMatchObject({ completeAttempts: 0, partialAttempts: 1 }); + expect(summary.phases.every(phase => phase.totalMs === null)).toBe(true); + expect(workerPhaseTimingText(summary)).toContain("Worker setup time: Unknown (200ms observed)."); + expect(workerPhaseTimingText(summary)).toContain("Worker execution time: Unknown."); + } finally { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); await closed; } + }); + + it("rejects wrong identities, overlapping segments and out-of-window timestamps before reporting", async () => { + const { file, result } = await fixture(); const receipt = await complete(file, result); + if (receipt.status === "unavailable") throw new Error("Missing fixture timing"); + const other = structuredClone(receipt); other.executionId = "another-case"; + const overlap = structuredClone(receipt); overlap.segments[1]!.startedAt = new Date(1050).toISOString(); + const future = structuredClone(receipt); future.finishedAt = new Date(1600).toISOString(); future.segments.at(-1)!.finishedAt = future.finishedAt; + for (const invalid of [other, overlap, future, { ...receipt, extra: "unapproved field" }]) { + expect(() => validateWorkerTiming(invalid, result)).toThrow("Invalid worker timing"); + expect(() => validateRetainedRunnerResult({ ...result, workerTiming: invalid })).toThrow("result.workerTiming"); + } + }); + + it("bounds recovery reads and refuses symlinks or mismatched attempt files", async () => { + const { file, dir, result } = await fixture(); await complete(file, result); + expect(await recoverWorkerTiming(file, { ...result, attempt: 2 })).toMatchObject({ status: "unavailable", reason: "invalid_evidence" }); + const link = path.join(dir, "linked.json"); await symlink(file, link); + expect(await recoverWorkerTiming(link, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" }); + await writeFile(file, " ".repeat(8193)); + expect(await recoverWorkerTiming(file, result)).toMatchObject({ status: "unavailable", reason: "invalid_evidence" }); + expect(await recoverWorkerTiming(path.join(dir, "missing.json"), result)).toBeUndefined(); + }); + + it("keeps timing failures from interrupting cleanup or overwriting earlier evidence", async () => { + const { file, result } = await fixture(); await complete(file, result); const original = await readFile(file, "utf8"); + const recorder = await createWorkerTiming(file, result, 1000); + await recorder.enter("cleanup", 1100); + expect(await recorder.finish(1500)).toMatchObject({ status: "unavailable", reason: "write_failed" }); + expect(await readFile(file, "utf8")).toBe(original); + const clockFile = file + ".clock"; const clock = await createWorkerTiming(clockFile, result, 1000); + await clock.enter("execution", 900); + expect(await clock.finish(1500)).toMatchObject({ status: "unavailable", reason: "clock_regression" }); + }); + + it("preserves a timing observation through duplicate legacy copies and rejects contradictions", async () => { + const { file, result } = await fixture(); const measured = { ...result, workerTiming: await complete(file, result) }; + for (const attempts of [[measured, result], [result, measured]]) { + expect(buildCampaignAccounting({ results: [measured], attempts }).attempts[0]?.workerTiming).toEqual(measured.workerTiming); + } + expect(() => buildCampaignAccounting({ results: [measured], attempts: [measured, { ...measured, + workerTiming: { version: 1, scope: "worker", status: "unavailable", reason: "write_failed" }, + }] })).toThrow("Conflicting timing receipts"); + }); +}); diff --git a/tests/runner-e2e/worker-timing.ts b/tests/runner-e2e/worker-timing.ts new file mode 100644 index 00000000000..8bbe9abaecd --- /dev/null +++ b/tests/runner-e2e/worker-timing.ts @@ -0,0 +1,148 @@ +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { mkdir, open, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export const WORKER_TIMING_PHASES = ["setup", "execution", "feedback", "cleanup", "finalization"] as const; +export type WorkerTimingPhase = typeof WORKER_TIMING_PHASES[number]; +type TimingContext = { executionId: string; attempt: number; startedAt: string; finishedAt: string }; +export type WorkerTiming = { + version: 1; + scope: "worker"; + status: "complete" | "partial"; + executionId: string; + attempt: number; + startedAt: string; + finishedAt: string | null; + segments: Array<{ phase: WorkerTimingPhase; startedAt: string; finishedAt: string | null }>; +} | { + version: 1; + scope: "worker"; + status: "unavailable"; + reason: "write_failed" | "invalid_evidence" | "clock_regression"; +}; + +function invalid(): never { throw new Error("Invalid worker timing receipt"); } +function shape(value: unknown, keys: string[]): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) invalid(); + const record = value as Record; + if (Object.keys(record).length !== keys.length || keys.some(key => !Object.hasOwn(record, key))) invalid(); + return record; +} +function timestamp(value: unknown) { + if (typeof value !== "string") invalid(); + const ms = Date.parse(value); + if (!Number.isFinite(ms) || new Date(ms).toISOString() !== value) invalid(); + return ms; +} + +export function validateWorkerTiming(value: unknown, context: TimingContext): asserts value is WorkerTiming { + if (!value || typeof value !== "object" || Array.isArray(value)) invalid(); + const status = (value as Record).status; + const record = shape(value, status === "unavailable" + ? ["version", "scope", "status", "reason"] + : ["version", "scope", "status", "executionId", "attempt", "startedAt", "finishedAt", "segments"]); + if (record.version !== 1 || record.scope !== "worker") invalid(); + if (status === "unavailable") { + if (typeof record.reason !== "string" || !["write_failed", "invalid_evidence", "clock_regression"].includes(record.reason)) invalid(); + return; + } + if (status !== "complete" && status !== "partial" || record.executionId !== context.executionId || record.attempt !== context.attempt + || typeof record.executionId !== "string" || record.executionId.length < 1 || record.executionId.length > 512 + || !Number.isSafeInteger(record.attempt) || Number(record.attempt) < 1) invalid(); + const start = timestamp(record.startedAt), lower = Date.parse(context.startedAt), upper = Date.parse(context.finishedAt); + if (!Number.isFinite(lower) || !Number.isFinite(upper) || start < lower || start > upper) invalid(); + if (!Array.isArray(record.segments) || record.segments.length < 1 || record.segments.length > 16) invalid(); + let cursor = start; + record.segments.forEach((raw, index) => { + const segment = shape(raw, ["phase", "startedAt", "finishedAt"]); + if (!WORKER_TIMING_PHASES.includes(segment.phase as WorkerTimingPhase) || index === 0 && segment.phase !== "setup") invalid(); + if (timestamp(segment.startedAt) !== cursor || cursor > upper) invalid(); + if (segment.finishedAt === null) { + if (status !== "partial" || index !== (record.segments as unknown[]).length - 1) invalid(); + } else { + const end = timestamp(segment.finishedAt); + if (end < cursor || end > upper || status === "partial" && index === (record.segments as unknown[]).length - 1) invalid(); + cursor = end; + } + }); + if (status === "partial" ? record.finishedAt !== null : timestamp(record.finishedAt) !== cursor) invalid(); +} + +function unavailable(reason: Extract["reason"]): WorkerTiming { + return { version: 1, scope: "worker", status: "unavailable", reason }; +} + +export async function createWorkerTiming( + file: string, + identity: Pick, + startedAtMs: number, +) { + let receipt: WorkerTiming = { + version: 1, scope: "worker", status: "partial", executionId: identity.executionId, attempt: identity.attempt, + startedAt: new Date(startedAtMs).toISOString(), finishedAt: null, + segments: [{ phase: "setup", startedAt: new Date(startedAtMs).toISOString(), finishedAt: null }], + }; + async function persist(initial = false) { + if (receipt.status === "unavailable") return; + const temporary = `${file}.${randomUUID()}.pending`; + try { + const data = JSON.stringify(receipt); + if (initial) { + await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, data, { mode: 0o600, flag: "wx" }); + } + else { + await writeFile(temporary, data, { mode: 0o600, flag: "wx" }); + await rename(temporary, file); + } + } catch { receipt = unavailable("write_failed"); } + finally { await rm(temporary, { force: true }).catch(() => undefined); } + } + function closeCurrent(at: number) { + if (receipt.status === "unavailable" || receipt.status === "complete") return false; + const active = receipt.segments.at(-1)!; + if (!Number.isSafeInteger(at) || !Number.isFinite(new Date(at).getTime()) || at < Date.parse(active.startedAt)) { + receipt = unavailable("clock_regression"); + return false; + } + active.finishedAt = new Date(at).toISOString(); + return true; + } + await persist(true); + return { + async enter(phase: WorkerTimingPhase, at = Date.now()) { + if (!closeCurrent(at) || receipt.status === "unavailable") return; + if (receipt.segments.length >= 16) { receipt = unavailable("invalid_evidence"); return; } + receipt.segments.push({ phase, startedAt: new Date(at).toISOString(), finishedAt: null }); + await persist(); + }, + async finish(at = Date.now()): Promise { + if (closeCurrent(at) && receipt.status !== "unavailable") { + receipt.status = "complete"; + receipt.finishedAt = new Date(at).toISOString(); + await persist(); + } + return structuredClone(receipt); + }, + }; +} + +export async function recoverWorkerTiming(file: string, context: TimingContext): Promise { + try { + if (typeof constants.O_NOFOLLOW !== "number" || typeof constants.O_NONBLOCK !== "number") return unavailable("invalid_evidence"); + const handle = await open(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.size > 8192) return unavailable("invalid_evidence"); + const buffer = Buffer.alloc(8193); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > 8192) return unavailable("invalid_evidence"); + const receipt: unknown = JSON.parse(buffer.subarray(0, bytesRead).toString("utf8")); + validateWorkerTiming(receipt, context); + return receipt; + } finally { await handle.close(); } + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ENOENT" ? undefined : unavailable("invalid_evidence"); + } +} diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index b89cc7c3b19..e6c8b1c16e4 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -93,7 +93,7 @@ describe("public repository paid workflow security", () => { const workflows = [ { name: "runner-full-stack-e2e.yml", - expectedCachedSetupNodeSteps: 4, + expectedCachedSetupNodeSteps: 3, }, { name: "pr-trusted.yml", @@ -311,7 +311,7 @@ describe("public repository paid workflow security", () => { const everydayOraclePreparation = paidJob.indexOf( "- name: Prepare pinned Python artifact oracle image", ); - const paidExecution = paidJob.indexOf("- name: Run paid cell"); + const paidExecution = paidJob.indexOf("- name: Run bounded campaign and retain feedback review"); expect(paidInstall).toBeGreaterThan(0); expect(daytonaPluginPreparation).toBeGreaterThan(paidInstall); expect(awsFfmpegInstall).toBeGreaterThan(daytonaPluginPreparation); @@ -588,7 +588,7 @@ describe("public repository paid workflow security", () => { } }); - it("runs paid scheduled campaigns only on Sundays", async () => { + it("uses bounded daily and weekly Product schedules while preserving Runner schedules", async () => { const workflows = await Promise.all( [ "runner-full-stack-e2e.yml", @@ -598,12 +598,20 @@ describe("public repository paid workflow security", () => { readFile(path.join(repositoryRoot, ".github/workflows", name), "utf8"), ), ); - for (const workflow of workflows) { - const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map( - (match) => match[1]!, - ); - expect(crons).toHaveLength(1); - expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/); + for (const [index, workflow] of workflows.entries()) { + const crons = [...workflow.matchAll(/cron:\s*"([^"]+)"/g)].map(match => match[1]!); + if (index === 0) { + expect(crons).toEqual(["47 8 * * *", "47 9 * * 0"]); + expect(workflow).toContain("vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'"); + expect(workflow).toContain('test "$TARGET_SHA" = "$GITHUB_SHA"'); + expect(workflow).toContain('echo "max_parallel=1"'); + expect(workflow).toContain("--max-automatic-retries 0"); + const paid = workflow.split("- name: Run bounded campaign and retain feedback review")[1]!.split("- name: Run paid cell")[0]!; + expect([...paid.matchAll(/secrets\.([A-Z_]+)/g)].map(match => match[1])).toEqual(["BRAINTRUST_API_KEY"]); + } else { + expect(crons).toHaveLength(1); + expect(crons[0]).toMatch(/^\d{1,2} \d{1,2} \* \* 0$/); + } expect(workflow).toContain("workflow_dispatch:"); } }); @@ -720,18 +728,18 @@ describe("public repository paid workflow security", () => { ); }); - it("resolves each trusted reporting lockfile before frozen install and AWS credentials", async () => { + it("keeps reporting jobs on the committed lockfile without registry re-resolution", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8", ); for (const jobName of ["report", "publish_history"]) { const job = workflow.split(`\n ${jobName}:`)[1]!.split(/\n [a-z_]+:/u)[0]!; - const resolve = job.indexOf("pnpm install --lockfile-only --ignore-scripts --no-frozen-lockfile"); - const install = job.indexOf("pnpm install --frozen-lockfile"); + const install = job.indexOf("pnpm install --frozen-lockfile --ignore-scripts"); expect(job).toContain("ref: ${{ github.sha }}"); expect(job).not.toContain("resolved-target-lockfile"); - expect(resolve, jobName).toBeGreaterThan(-1); - expect(install, jobName).toBeGreaterThan(resolve); + expect(job, jobName).not.toContain("--no-frozen-lockfile"); + expect(job, jobName).not.toContain("--lockfile-only"); + expect(install, jobName).toBeGreaterThan(job.indexOf("ref: ${{ github.sha }}")); const credentials = job.indexOf("aws-actions/configure-aws-credentials@"); if (credentials >= 0) expect(install).toBeLessThan(credentials); } @@ -767,11 +775,12 @@ describe("public repository paid workflow security", () => { "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", ); expect(report).toContain( - "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }}", - ); - expect(report).toContain( - "PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}", + "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}", ); + const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}"; + expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); + expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); + expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'"); expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports")); @@ -789,6 +798,64 @@ describe("public repository paid workflow security", () => { ); }); + it("keeps recurring reports private while preserving a content-free liveness receipt", async () => { + const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); + const paid = workflow.split("\n test:")[1]!.split("\n report:")[0]!; + const manual = paid.split("\n test_recurring:")[0]!; + const recurring = paid.split("\n test_recurring:")[1]!; + expect(manual).not.toContain("id-token: write"); + expect(manual).toContain("if: needs.catalog.outputs.operations_cadence == ''"); + expect(recurring).toContain("id-token: write"); + expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); + expect(recurring).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); + expect(recurring).toContain("steps: *paid_test_steps"); + expect(manual).toContain("steps: &paid_test_steps"); + const report = workflow.split("\n report:")[1]!.split("\n publish_history:")[0]!; + const step = (source: string, name: string) => source.split(` - name: ${name}\n`)[1]!.split(/\n - /u)[0]!; + for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) { + expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''"); + } + expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''"); + expect(step(paid, "Run bounded campaign and retain feedback review")).not.toContain("AWS_"); + for (const job of [paid, report]) { + const credentials = step(job, "Obtain scoped private history credentials"); + expect(credentials).toContain("output-env-credentials: false"); + expect(credentials).toContain("disable-retry: true"); + expect(credentials).toContain("RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN"); + } + expect(paid.indexOf("Verify private evidence retention before paid execution")).toBeLessThan(paid.indexOf("Run bounded campaign and retain feedback review")); + expect(report).toContain("report > runner-e2e-merged-report/report.log 2>&1"); + const marker = step(report, "Retain report availability for the schedule monitor"); + expect(marker).toContain("steps.private_report.outcome == 'success'"); + expect(marker).toContain("path: private-report-status.json"); + expect(report).toContain('echo \'{"version":1,"privateReportRetained":true}\' > private-report-status.json'); + expect(marker).not.toMatch(/BUCKET|ROLE_ARN|private-report-reference/); + }); + + it("isolates private archive trust from workflows that execute target-branch code", async () => { + const workflows = path.join(repositoryRoot, ".github/workflows"); + const source = await readFile(path.join(workflows, "runner-full-stack-e2e.yml"), "utf8"); + const trust = await readFile(path.join(repositoryRoot, ".github/infra/runner-e2e-private-history.yml"), "utf8"); + const environments = ["runner-e2e-private-paid", "runner-e2e-private-history"]; + const subjects = [...trust.matchAll(/!Sub 'repo:\$\{GitHubRepository\}:environment:([^']+)'/gu)].map(match => match[1]); + expect(subjects).toEqual(environments); + const recurring = source.split("\n test_recurring:")[1]!.split("\n report:")[0]!; + const report = source.split("\n report:")[1]!.split("\n publish_history:")[0]!; + for (const [index, job] of [recurring, report].entries()) { + expect(job).toContain(`name: ${environments[index]}`); + expect(job).toContain("github.ref == format('refs/heads/{0}', github.event.repository.default_branch)"); + } + expect(recurring).toContain("needs.authorize.outputs.target_sha == github.sha"); + for (const filename of await readdir(workflows)) { + if (!/\.ya?ml$/u.test(filename) || filename === "runner-full-stack-e2e.yml") continue; + const other = await readFile(path.join(workflows, filename), "utf8"); + for (const environment of environments) expect(other, filename).not.toContain(environment); + } + const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); + expect(protocol).toContain("name: runner-e2e-paid"); + expect(protocol).toContain("needs.authorize.outputs.target_sha"); + }); + it("uses environment-scoped OIDC for a no-delete history publisher", async () => { const workflow = await readFile( path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), @@ -827,4 +894,26 @@ describe("public repository paid workflow security", () => { workflow.indexOf("pnpm test:e2e:runner:history:publish"), ).toBeLessThan(workflow.indexOf("actions/upload-pages-artifact@")); }); + + it("does not restore dependency caches or run install scripts in OIDC-authorized jobs", async () => { + const workflow = await readFile(path.join(repositoryRoot, ".github/workflows/runner-full-stack-e2e.yml"), "utf8"); + const jobs = workflow.split(/\n(?= [a-z_]+:)/u); + const paidSteps = workflow.split("steps: &paid_test_steps")[1]!.split("\n test_recurring:")[0]!; + const oidcJobs = jobs.filter(job => job.includes("id-token: write")); + expect(oidcJobs.length).toBeGreaterThanOrEqual(3); + for (const job of oidcJobs) { + const steps = job + (job.includes("steps: *paid_test_steps") ? paidSteps : ""); + expect(steps, job.split("\n")[0]).not.toMatch(/cache:\s*(?:pnpm|npm|yarn)\b/u); + expect(steps).not.toMatch(/actions\/cache(?:\/restore)?@/u); + for (const install of steps.matchAll(/\bpnpm install[^\n]*/gu)) { + expect(install[0]).toContain("--ignore-scripts"); + } + } + for (const name of ["report", "publish_history"]) { + const job = jobs.find(value => value.startsWith(` ${name}:`))!; + const installs = [...job.matchAll(/\bpnpm install[^\n]*/gu)]; + expect(installs).toHaveLength(1); + for (const install of installs) expect(install[0]).toContain('--store-dir "${{ runner.temp }}/eval-'); + } + }); }); diff --git a/tests/runner-e2e/workflow-target.test.ts b/tests/runner-e2e/workflow-target.test.ts new file mode 100644 index 00000000000..88aa5b6cb5c --- /dev/null +++ b/tests/runner-e2e/workflow-target.test.ts @@ -0,0 +1,49 @@ +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const queuedSha = "a".repeat(40); +const advancedSha = "b".repeat(40); + +async function resolveTarget(event: string, cadence: string, branch = "master") { + const workflow = await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml", import.meta.url), "utf8"); + const step = workflow.split(" - name: Resolve requested repository branch to an immutable commit\n")[1]!.split("\n - name:")[0]!; + const script = step.split(" run: |\n")[1]!.replace(/^ {10}/gm, ""); + const directory = await mkdtemp(path.join(os.tmpdir(), "eval-workflow-target-")); + try { + const output = path.join(directory, "output"); + const result = spawnSync("bash", ["-c", `gh() { printf '%s\\n' "$MOCK_BRANCH_SHA"; }\n${script}`], { + encoding: "utf8", env: { ...process.env, EVENT_NAME: event, EVAL_CADENCE: cadence, + TARGET_BRANCH: branch, DEFAULT_BRANCH: "master", GITHUB_SHA: queuedSha, + MOCK_BRANCH_SHA: advancedSha, GITHUB_OUTPUT: output, REPOSITORY: "paperclipai/paperclip" }, + }); + return { status: result.status, stderr: result.stderr, + output: await readFile(output, "utf8").catch(() => "") }; + } finally { await rm(directory, { recursive: true, force: true }); } +} + +describe("paid workflow target authorization", () => { + it.each([["schedule", "manual"], ["workflow_dispatch", "daily"], ["workflow_dispatch", "weekly"]])( + "pins %s/%s to the queued revision when the branch advances", async (event, cadence) => { + const result = await resolveTarget(event, cadence); + expect(result.status, result.stderr).toBe(0); + expect(result.output).toContain(`sha=${queuedSha}\n`); + expect(result.output).not.toContain(advancedSha); + }); + + it("rejects custom bounded targets before emitting a commit", async () => { + const result = await resolveTarget("workflow_dispatch", "daily", "development"); + expect(result.status).toBe(1); + expect(result.output).toBe(""); + expect(result.stderr).toContain("default-branch commit"); + }); + + it("still resolves explicit manual targets at authorization time", async () => { + const result = await resolveTarget("workflow_dispatch", "manual", "development"); + expect(result.status, result.stderr).toBe(0); + expect(result.output).toContain(`sha=${advancedSha}\n`); + expect(result.output).toContain("ref=refs/heads/development\n"); + }); +}); diff --git a/ui/src/pages/Cases.test.tsx b/ui/src/pages/Cases.test.tsx index c952f558d11..613314a8cda 100644 --- a/ui/src/pages/Cases.test.tsx +++ b/ui/src/pages/Cases.test.tsx @@ -278,6 +278,7 @@ describe("Read-only cases list", () => { }))); expect(container.querySelector('[data-testid="location"]')?.textContent).toBe("/PAP/cases?sort=title&dir=asc"); await escape(); + await waitFor(() => expect(document.activeElement).toBe(labelledButton("Sort"))); await click(labelledButton("Group")); const type = [...document.body.querySelectorAll('[aria-label="Group cases"] button')] .find(element => element.textContent === "Type")!;