diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index fcac6ebf260..5f469f96c8a 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -74,6 +74,8 @@ jobs: playwright_channel: ${{ steps.runner.outputs.playwright_channel }} target_sha: ${{ steps.target.outputs.sha }} target_ref: ${{ steps.target.outputs.ref }} + evidence_retention: ${{ steps.evidence_route.outputs.evidence_retention }} + manual_private_ids: ${{ steps.evidence_route.outputs.manual_private_ids }} steps: - name: Require default branch and allowlisted numeric actor IDs env: @@ -211,6 +213,22 @@ jobs: - name: Validate report dependency manifests offline run: pnpm install --lockfile-only --frozen-lockfile --ignore-scripts --offline + - name: Classify manual evidence before any protected credential job + id: evidence_route + env: + EVENT_NAME: ${{ github.event_name }} + EVAL_CADENCE: ${{ inputs.cadence || 'manual' }} + SELECT_ID: ${{ inputs.id }} + SELECT_SUITE: ${{ inputs.suite }} + SELECT_GROUP: ${{ inputs.group }} + SELECT_PROFILE: ${{ inputs.profile }} + SELECT_ENVIRONMENT: ${{ inputs.environment }} + SELECT_CASE: ${{ inputs.case }} + TEST_RUNNER: ${{ steps.runner.outputs.runner }} + NODE_OPTIONS: '' + NODE_PATH: '' + run: node tests/runner-e2e/manual-private-policy.mjs + target_lock: name: Resolve target pnpm lockfile needs: authorize @@ -864,7 +882,7 @@ jobs: test: name: ${{ matrix.executionId }} - if: needs.catalog.outputs.operations_cadence == '' + if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' needs: [ authorize, @@ -916,6 +934,13 @@ jobs: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: needs.catalog.outputs.operations_cadence == '' + with: + ref: ${{ github.sha }} + path: .manual-private-trusted + persist-credentials: false + - name: Download resolved target lockfile uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -948,6 +973,79 @@ jobs: with: version: 9.15.4 + - name: Validate actual manual cell against trusted evidence routing + id: manual_cell_route + if: needs.catalog.outputs.operations_cadence == '' + env: + EVIDENCE_RETENTION: ${{ needs.authorize.outputs.evidence_retention }} + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + NODE_OPTIONS: '' + NODE_PATH: '' + run: node "$GITHUB_WORKSPACE/.manual-private-trusted/tests/runner-e2e/manual-private-policy.mjs" cell + + - name: Prepare workflow-pinned private transfer runtime before archive credentials + if: steps.manual_cell_route.outputs.route == 'private' + working-directory: .manual-private-trusted + env: + NODE_OPTIONS: '' + NODE_PATH: '' + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + node tests/runner-e2e/prepare-report-lock.mjs + pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/manual-private-publication-store" + + - name: Bind protected original run and compatible target evidence protocol + id: manual_private + if: steps.manual_cell_route.outputs.route == 'private' + working-directory: .manual-private-trusted + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + TARGET_ROOT: ${{ github.workspace }} + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + run: node "$GITHUB_WORKSPACE/.manual-private-trusted/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/.manual-private-trusted/tests/runner-e2e/manual-private-transfer.ts" prepare + + - name: Obtain exact-key manual archive session + id: manual_private_credentials + if: steps.manual_private.outcome == 'success' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_private.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Verify restricted complete evidence storage before model or fixture secrets + id: manual_private_verified + if: steps.manual_private.outcome == 'success' + working-directory: .manual-private-trusted + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + TARGET_ROOT: ${{ github.workspace }} + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_EVIDENCE_ROOT: ${{ steps.manual_private.outputs.root }} + AWS_ACCESS_KEY_ID: ${{ steps.manual_private_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_private_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_private_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/.manual-private-trusted/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/.manual-private-trusted/tests/runner-e2e/manual-private-transfer.ts" verify + # This job receives provider credentials only in the final paid-test # step. Keep target-selected dependency lifecycle code from running in # the protected environment during setup. @@ -1201,8 +1299,10 @@ jobs: test "$run_status" = 0 && test "$finalization_status" = 0 - name: Run paid cell - if: needs.catalog.outputs.operations_cadence == '' + if: needs.catalog.outputs.operations_cadence == '' && (steps.manual_cell_route.outputs.route == 'actions' || steps.manual_private_verified.outcome == 'success') env: + PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.manual_private_verified.outcome == 'success' && format('{0}/preflight.json', steps.manual_private.outputs.root) || '' }} + PRIVATE_EVIDENCE_ROOT: ${{ steps.manual_private.outputs.root }} OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }} @@ -1217,15 +1317,20 @@ jobs: PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} PAPERCLIP_PLAYWRIGHT_CHANNEL: ${{ needs.authorize.outputs.playwright_channel }} - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" + run: | + if [ -n "$PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT" ]; then + pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 + else + pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" + fi - name: Stage bounded campaign evidence id: stage_evidence if: always() && needs.catalog.outputs.operations_cadence != '' run: node tests/runner-e2e/artifact-budget.mjs stage evidence tests/runner-e2e/results bounded-runner-e2e-evidence - - name: Upload access-controlled packaged cell evidence - if: always() && needs.catalog.outputs.operations_cadence == '' + - name: Upload nonprivate packaged cell evidence + if: always() && needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && steps.manual_cell_route.outputs.route == 'actions' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.executionId }} @@ -1247,6 +1352,26 @@ jobs: if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_evidence.outputs.complete != 'true' run: exit 1 + - name: Retain complete approved manual originals with no public fallback + if: always() && steps.manual_private_verified.outcome == 'success' + working-directory: .manual-private-trusted + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + TARGET_ROOT: ${{ github.workspace }} + SELECTED_EXECUTION_ID: ${{ matrix.executionId }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_EVIDENCE_ROOT: ${{ steps.manual_private.outputs.root }} + AWS_ACCESS_KEY_ID: ${{ steps.manual_private_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_private_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_private_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/.manual-private-trusted/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/.manual-private-trusted/tests/runner-e2e/manual-private-transfer.ts" upload-cell + test_recurring: name: ${{ matrix.executionId }} if: needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) @@ -1264,10 +1389,27 @@ jobs: matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }} steps: *paid_test_steps + test_manual_private: + name: ${{ matrix.executionId }} + if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'private' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack] + runs-on: ${{ needs.authorize.outputs.test_runner }} + timeout-minutes: ${{ matrix.timeoutMinutes }} + permissions: + contents: read + id-token: write + environment: + name: runner-e2e-private-paid + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel) }} + matrix: ${{ fromJSON(needs.catalog.outputs.matrix) }} + steps: *paid_test_steps + report: name: Merge and enforce campaign result if: always() && !cancelled() && needs.catalog.result == 'success' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) - needs: [authorize, catalog, daytona_image, test, test_recurring] + needs: [authorize, catalog, daytona_image, test, test_recurring, test_manual_private] outputs: history_source_ready: ${{ steps.history_source_ready.outputs.ready }} runs-on: ubuntu-latest @@ -1303,6 +1445,292 @@ jobs: - name: Install pinned reporting dependencies run: pnpm install --frozen-lockfile --ignore-scripts --store-dir "${{ runner.temp }}/eval-report-store" + - name: Prepare exact manual report storage identity + id: manual_report + if: needs.authorize.outputs.evidence_retention == 'private' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" report-session + + - name: Obtain exact current manual report archive session + id: manual_report_credentials + if: steps.manual_report.outcome == 'success' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Obtain exact private cell 0 read session + id: manual_read_credentials_0 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 0 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_0 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 0 + if: steps.manual_read_credentials_0.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '0' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_0.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_0.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_0.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 1 read session + id: manual_read_credentials_1 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 1 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_1 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 1 + if: steps.manual_read_credentials_1.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '1' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_1.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_1.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_1.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 2 read session + id: manual_read_credentials_2 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 2 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_2 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 2 + if: steps.manual_read_credentials_2.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '2' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_2.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_2.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_2.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 3 read session + id: manual_read_credentials_3 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 3 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_3 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 3 + if: steps.manual_read_credentials_3.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '3' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_3.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_3.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_3.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 4 read session + id: manual_read_credentials_4 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 4 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_4 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 4 + if: steps.manual_read_credentials_4.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '4' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_4.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_4.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_4.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 5 read session + id: manual_read_credentials_5 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 5 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_5 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 5 + if: steps.manual_read_credentials_5.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '5' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_5.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_5.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_5.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 6 read session + id: manual_read_credentials_6 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 6 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_6 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 6 + if: steps.manual_read_credentials_6.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '6' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_6.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_6.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_6.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Obtain exact private cell 7 read session + id: manual_read_credentials_7 + if: steps.manual_report.outcome == 'success' && fromJSON(steps.manual_report.outputs.cell_count) > 7 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.manual_report.outputs.read_policy_7 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + + - name: Download restricted manual cell 7 + if: steps.manual_read_credentials_7.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '7' + AWS_ACCESS_KEY_ID: ${{ steps.manual_read_credentials_7.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_read_credentials_7.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_read_credentials_7.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" download-cell + + - name: Verify all approved private cell transfers before merging + id: manual_cells_downloaded + if: steps.manual_report.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + NODE_OPTIONS: '' + NODE_PATH: '' + run: node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" verify-downloads + - name: Resolve workflow job attempts env: GH_TOKEN: ${{ github.token }} @@ -1324,7 +1752,7 @@ jobs: - name: Download cell evidence id: download_evidence - if: needs.catalog.outputs.operations_cadence == '' + if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -1333,7 +1761,7 @@ jobs: merge-multiple: false - name: Retry cell evidence download after transport failure - if: needs.catalog.outputs.operations_cadence == '' && steps.download_evidence.outcome == 'failure' + if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && steps.download_evidence.outcome == 'failure' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: runner-e2e-${{ github.run_id }}-*-* @@ -1363,7 +1791,7 @@ jobs: run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/private-history.ts download evidence "$GITHUB_RUN_ID" "downloaded-runner-e2e/runner-e2e-$GITHUB_RUN_ID-1-scheduled-campaign" - name: Select latest workflow attempt per cell - if: always() && needs.catalog.outputs.operations_cadence == '' + if: always() && needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' env: PAPERCLIP_RUNNER_E2E_ARTIFACT_ROOT: ${{ github.workspace }}/downloaded-runner-e2e PAPERCLIP_RUNNER_E2E_SELECTED_ROOT: ${{ github.workspace }}/selected-runner-e2e @@ -1408,7 +1836,13 @@ jobs: if: always() env: PAPERCLIP_RUNNER_E2E_MERGED_REPORT_DIR: ${{ github.workspace }}/runner-e2e-merged-report - run: pnpm exec playwright merge-reports --config tests/runner-e2e/merge.config.ts merged-blob-reports + run: | + if [ "${{ needs.authorize.outputs.evidence_retention }}" = private ]; then + mkdir -p runner-e2e-merged-report + pnpm exec playwright merge-reports --config tests/runner-e2e/merge.config.ts merged-blob-reports > runner-e2e-merged-report/merge.log 2>&1 + else + pnpm exec playwright merge-reports --config tests/runner-e2e/merge.config.ts merged-blob-reports + fi - name: Aggregate normalized campaign results if: always() @@ -1419,12 +1853,13 @@ jobs: PAPERCLIP_E2E_CAMPAIGN_ID: ${{ needs.catalog.outputs.operations_cadence != '' && format('gha-{0}-1-scheduled-campaign', github.run_id) || format('gha-{0}-{1}', github.run_id, github.run_attempt) }} PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} - PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }} + PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }} PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }} + EVIDENCE_RETENTION: ${{ needs.authorize.outputs.evidence_retention }} EVAL_OPERATIONS_CADENCE: ${{ needs.catalog.outputs.operations_cadence }} run: | set +e - if [ -n "$EVAL_OPERATIONS_CADENCE" ]; then + if [ -n "$EVAL_OPERATIONS_CADENCE" ] || [ "$EVIDENCE_RETENTION" = private ]; then mkdir -p runner-e2e-merged-report pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 else @@ -1432,7 +1867,7 @@ jobs: fi report_status=$? set -e - if [ -z "$EVAL_OPERATIONS_CADENCE" ]; then + if [ -z "$EVAL_OPERATIONS_CADENCE" ] && [ "$EVIDENCE_RETENTION" = actions ]; then cat runner-e2e-merged-report/normalized/summary.md >> "$GITHUB_STEP_SUMMARY" fi exit "$report_status" @@ -1442,8 +1877,8 @@ jobs: if: always() && needs.catalog.outputs.operations_cadence != '' run: node tests/runner-e2e/artifact-budget.mjs stage report runner-e2e-merged-report bounded-runner-e2e-report - - name: Upload access-controlled merged report - if: always() && needs.catalog.outputs.operations_cadence == '' + - name: Upload nonprivate merged report + if: always() && needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} @@ -1478,9 +1913,30 @@ jobs: if: always() && needs.catalog.outputs.operations_cadence != '' && steps.stage_report.outputs.complete != 'true' run: exit 1 + - name: Retain complete manual merged evidence only in restricted storage + id: manual_private_report + if: always() && steps.manual_report_credentials.outcome == 'success' && steps.manual_cells_downloaded.outcome == 'success' + env: + MANUAL_PRIVATE_IDS: ${{ needs.authorize.outputs.manual_private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ vars.RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON }} + TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report + AWS_ACCESS_KEY_ID: ${{ steps.manual_report_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.manual_report_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.manual_report_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: | + node "$GITHUB_WORKSPACE/cli/node_modules/tsx/dist/cli.mjs" "$GITHUB_WORKSPACE/tests/runner-e2e/manual-private-transfer.ts" upload-report + echo 'The private report archive is retained. Transfer does not certify feature PASS or required evidence completeness; see its typed receipt. No raw Actions artifact or public page was uploaded.' >> "$GITHUB_STEP_SUMMARY" + - name: Verify normalized history source report id: history_source_ready - if: always() + if: always() && needs.authorize.outputs.evidence_retention == 'actions' run: | set -euo pipefail dashboard_root="runner-e2e-merged-report/normalized" @@ -1493,7 +1949,7 @@ jobs: publish_history: name: Publish S3 history and Pages bundle with declared screenshots needs: [authorize, catalog, report] - if: always() && needs.catalog.result == 'success' && needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true' + if: always() && needs.catalog.result == 'success' && needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && needs.report.outputs.history_source_ready == 'true' runs-on: ubuntu-latest timeout-minutes: 15 outputs: diff --git a/doc/manual-e2e-private-evidence.md b/doc/manual-e2e-private-evidence.md new file mode 100644 index 00000000000..844d3f3cd10 --- /dev/null +++ b/doc/manual-e2e-private-evidence.md @@ -0,0 +1,153 @@ +# Restricted evidence for manual AWS E2E runs + +The manual AWS workflow has a separate restricted evidence route. It uses the +existing private-history bucket, protected environments, OIDC role and bounded +archive transfer. It does not change the scheduled private-history lane. + +GitHub Actions artifacts in a public repository are readable by repository +readers. They are not an organization-private evidence store. The restricted +route never uploads raw cell evidence or a merged report to Actions, S3 public +history, or Pages. This stays true on setup failure, launcher failure and report +failure. Previously staged files do not enable a public fallback. + +## Supported first selections + +The first supported selection is an exact subset of these execution IDs: + +- `chat-features.runner-opencode.local.activity.agent-feedback` +- `chat-features.runner-opencode.local.activity.work-timeline` +- `chat-features.runner-opencode.local.companies.import-preview` +- `chat-features.runner-opencode.local.navigation-preferences.dismiss-announcement` +- `chat-features.runner-opencode.local.tasks.completion-with-gate` +- `chat-features.runner-opencode.local.tasks.private-task` +- `chat-features.runner-opencode.local.secrets.personal-secret` +- `chat-features.runner-opencode.local.agents.choose-primary` + +These IDs require the reviewed AWS Fleet route. Broad chat suites, mixed +unsupported IDs, reruns and missing approvals fail closed. All chat-feature IDs +are classified as restricted by trusted code. A target descriptor cannot opt out. +Other manual selections keep the existing Actions route only after a trusted +per-cell check confirms that the actual selected execution is not restricted. +Selector whitespace and exclusivity follow the catalog producer. + +A case that needs invitation-capability handling or other new producer support +is not supported merely because restricted storage exists. It needs its own +reviewed source, evidence contract and exact operation approval. + +## Deployment and operator gates + +This source change alone does not configure a role, bucket, environment, target, +or run. Before a supported run, the operator must: + +1. Deploy this workflow and its helpers together on the protected default branch. +2. Keep the existing numeric actor allowlist and protected default-branch checks. +3. Keep `runner-e2e-private-paid` and `runner-e2e-private-history` restricted to + the default branch. Establish administrator-controlled approval for the exact + source and original run. A reviewer UI may supply this control but is not + mandatory. Prove the actual control and runtime contract before admission. +4. Provide the existing private-history role ARN, AWS region and bucket variables + in those environments. The existing role must support the existing bounded + prefix and bucket-policy, ownership, public-access and retention reads. +5. Provide `RUNNER_E2E_MANUAL_PRIVATE_APPROVALS_JSON` through administrator-controlled + configuration available to both protected jobs. Approve the exact resolved + workflow SHA, target SHA, original run ID, + attempt `1`, selected execution IDs and launcher SHA-256. +6. Use a reviewed compatible target. Its private handoff and policy modules must + match the workflow implementation byte for byte. Its launcher digest must + match the protected approval. Earlier target branches do not gain this + protocol automatically. + +Classification happens before the protected job. It grants no archive authority. +The administrator-controlled record must bind the actual original run ID before +target dependency setup or archive credentials. Prove who can change that record +and that each job consumes the exact approved record. An environment reviewer +gate is one way to govern authorization; its UI is not required. A missing or +stale row still fails closed before target dependency setup, archive credentials, +model credentials or fixture secrets. There is no wildcard approval. + +The variable has this closed shape (all values below are placeholders): + +```json +{ + "version": 1, + "approvals": [{ + "workflowSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "targetSha": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "runId": "123456789", + "attempt": "1", + "executionIds": ["chat-features.runner-opencode.local.activity.agent-feedback"], + "protocol": "paperclip.manual-private-evidence/v1", + "launchSha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + }] +} +``` + +## Trust and credential lifetime + +The workflow checks out its own immutable `github.sha` into a separate directory. +It prepares pinned publication dependencies with `--ignore-scripts` and a +separate store before archive credentials. Transfers use that checkout's +absolute helper, tsx and archive-script paths. They do not load the target's +node_modules or an ambient Node loader. The target executes only after the +protected approval and private-storage preflight pass. + +The AWS action returns session credentials as step outputs, not job-wide +environment exports. Only trusted preflight and transfer steps receive those +outputs. Transfers require an explicit complete session and disable EC2 metadata +fallback. A failed or expired session fails retention. It never enables an +Actions upload. Exact-key session policies limit the intended transfer calls. Each current cell +uses its own two-object read session; the report writer uses a separate session. +Every generated policy is checked against the 2,048-character STS limit before +the credential action. No wildcard object access is added. + +This is **operator-approved target execution**, not a sandbox for malicious +source. The protected job has OIDC permission. Its target launcher can inherit +the GitHub OIDC request environment, and the existing harness copies its process +environment into server/native children. A target that can request a new OIDC +session may re-assume the role. An inline session policy does not prevent that. +Reviewers must approve the complete exact target, not just its evidence flag. +This change does not alter IAM, OIDC trust, host roles or credential duration. + +## Evidence and failure rules + +The launcher keeps its ordinary standard evidence in an owned temporary root. +It preserves the original framework result before launcher result updates. It +also seals final packaged results after cleanup and campaign finalization. +The transfer validates exact file inventories and hashes before archiving. +It excludes instance databases, workspaces, credential registries, auth files, +raw capture directories and arbitrary files outside the standard evidence scope. +Free-form launcher console capture is not an approved retained carrier. + +The existing credential scanners and PASS PNG, HTML, blob and snapshot checks +still apply. Original text, ZIP and media bytes receive those scans before +retention. Raw and encoded registered roots and provider-shaped names are also +rejected from original paths and inventories. Unknown extensions and SVG files +that the ordinary packager does not inspect cannot become complete originals. +Files cannot be silently dropped to meet a bound. Credential contamination, +unsupported carriers or a changed inventory stop the transfer. Missing required +PASS files still convert the attempt to infrastructure FAIL. The private inventory +retains all safe available originals and records `requiredEvidenceComplete: false` +and the exact missing list. Its `complete` field means the available file inventory +is complete; it does not certify PASS evidence completeness. Failed framework +attempts do not claim that PASS evidence checks ran. No file is synthesized and +no secret-bearing original is uploaded just because the bucket is private. + +The existing archive bounds remain unchanged: at most 256 MiB of source files, +4,097 archive entries and 264 MiB transferred archive bytes. Duplicated original +and packaged files count toward those bounds. A run that exceeds a bound does +not have complete retained evidence and needs an explicit disposition. + +Private objects use immutable conditional puts and version receipts under +`runner-e2e-private/bounded-v1/campaigns/gha-RUN_ID-1/`. Each cell has a hash-derived +object name. The report reads only the approved current cells and verifies each +archive against its retained transfer digest. Fresh merged HTML, blob, normalized output and report logs receive a final +standard-file scan and sealed inventory with the actual current report-step +credential roots, including its explicit AWS session. The report uses the same +raw, encoded-root and provider-shape checks before transfer. It retains the +scanned report in the same restricted store. A report error stays a report +error; retention does not change its grade or manufacture a public page. + +Authenticated retrieval uses the existing private-history operator machinery +and approved bucket access. This PR does not grant a reader role or perform an +upload. Actual deployment, environment approval, compatible target fingerprints, +private storage access and one exact run approval remain operational gates. diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 259ff499454..6ba750220c6 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -57,6 +57,12 @@ const REQUIRED_PASS_FILES = [ path.join("snapshots", "api-state.json"), ] as const; +export function isStandardEvidenceFile(relative: string) { + const extension = path.extname(relative).toLowerCase(); + return isStandardEvidencePath(relative) && (TEXT_EXTENSIONS.has(extension) || BINARY_EXTENSIONS.has(extension) + || extension === ".zip"); +} + export interface EvidencePackageResult { files: string[]; leaks: Array<{ file: string; reason: string }>; @@ -77,7 +83,7 @@ async function walk(root: string, relative = ""): Promise { return files; } -function isAllowed(relative: string) { +export function isStandardEvidencePath(relative: string) { const segments = relative.split(path.sep); if (segments.length === 1) return ( @@ -143,7 +149,7 @@ export async function packageEvidence(input: { const leaks: EvidencePackageResult["leaks"] = []; const available = await walk(input.privateDir); - for (const relative of available.filter(isAllowed)) { + for (const relative of available.filter(isStandardEvidencePath)) { const source = path.join(input.privateDir, relative); const extension = path.extname(relative).toLowerCase(); const destination = path.join(input.uploadDir, relative); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 7694feb483a..e9605563aba 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -33,6 +33,8 @@ import { import { isImmutableDaytonaImage, runnerMatrix } from "./catalog.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { packageEvidence } from "./evidence.js"; +import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { requiresPrivateManualEvidence } from "./manual-private-retention.js"; import { copySharedEvidence } from "./shared-evidence.js"; import { classifyFailure } from "./failure-classifier.js"; import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js"; @@ -89,7 +91,9 @@ import { const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const localEnvPath = path.join(repositoryRoot, ".env.runner-e2e.local"); -const resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results"); +let resultsRoot = path.join(repositoryRoot, "tests/runner-e2e/results"); +const manualEvidenceSecrets = new Set(); +let manualPrivateHandoff: ManualPrivateHandoff | undefined; const activeProcessGroups = new Set(); const activeProcessCleanup = new Map>(); const activeProcessTerminators = new Map void>(); @@ -478,6 +482,7 @@ async function runAttempt(input: { "One isolated runner E2E harness cannot mix profiles or environments", ); } + if (process.env.GITHUB_ACTIONS === "true" && !input.operations && executions.some(candidate => requiresPrivateManualEvidence(candidate.id)) && !manualPrivateHandoff) throw new Error("Sensitive manual evidence requires verified restricted retention before setup"); const upgrades = await readUpgradeArtifacts(process.env[upgradeArtifactKey]); if (upgrades) { // All attempts and the browser use A's installed fixture admission. The @@ -494,6 +499,7 @@ async function runAttempt(input: { const sharedMemoryBaseline = snapshotDarwinSharedMemory(); const temporaryParent = await realpath(os.tmpdir()); const temporaryRoot = await createRunnerE2ETemporaryRoot(temporaryParent); + const privateDir = manualPrivateHandoff ? path.join(manualPrivateHandoff.root, "capture") : path.join(temporaryRoot, "artifacts-private"); const publishedResults: RunnerE2EResult[] = []; const publishedResultPaths = new Map(); let attemptSecrets: string[] = []; @@ -504,7 +510,6 @@ async function runAttempt(input: { try { const paperclipHome = path.join(temporaryRoot, "paperclip-home"); const workspace = path.join(temporaryRoot, "workspace"); - const privateDir = path.join(temporaryRoot, "artifacts-private"); const instanceId = `runner-e2e-${randomBytes(8).toString("hex")}`; const configPath = path.join( paperclipHome, @@ -541,6 +546,7 @@ async function runAttempt(input: { }; const credentials = normalizedSecrets(Object.values(credentialSources)); attemptSecrets = credentials; + if (manualPrivateHandoff) for (const credential of credentials) manualEvidenceSecrets.add(credential); if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 }); const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( @@ -820,6 +826,7 @@ async function runAttempt(input: { originalResult: { status: original.status, failureClass: original.failureClass, error: original.error }, }, credentials), null, 2)}\n`, { mode: 0o600 }); } + if (manualPrivateHandoff) await preserveOriginalFrameworkResult(casePrivateDir); await writeFile( resultPath, `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`, @@ -831,6 +838,7 @@ async function runAttempt(input: { secrets: credentials, expectPassScreenshot: result.status === "passed", }); + const originalEvidence = evidence, originalFeaturePassed = result.status === "passed"; if (evidence.leaks.length > 0 || evidence.missing.length > 0) { result = { ...result, @@ -856,6 +864,8 @@ async function runAttempt(input: { expectPassScreenshot: false, }); } + if (manualPrivateHandoff) await sealStandardOriginalEvidence({ context: manualPrivateHandoff, + executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed }); console.log( `${result.status === "passed" ? "PASS" : "FAIL"} ${candidate.id} attempt ${attempt} -> ${uploadDir}`, ); @@ -871,7 +881,7 @@ async function runAttempt(input: { try { await gateway?.close(); } catch (error) { gatewayCleanupError = error; } if (!processCleanupFailed) reapNewDetachedDarwinSharedMemory(sharedMemoryBaseline); let cleanupError: unknown; - const resourceAdmissionStarted = await access(path.join(temporaryRoot, "artifacts-private", "resource-admission-started")) + const resourceAdmissionStarted = await access(path.join(privateDir, "resource-admission-started")) .then(() => true).catch((error: NodeJS.ErrnoException) => error.code !== "ENOENT"); if (mustPreserveRecoveryState({ processCleanupFailed, resourceAdmissionStarted, results: rawCleanupResults })) { // Remote allocation cleanup is journaled in this instance's database. @@ -1046,6 +1056,8 @@ async function main() { throw error; } const executions = selectRunnerExecutions(options, runnerMatrix); + manualPrivateHandoff = await readManualPrivateHandoff(process.env, executions.map(execution => execution.id)); + if (manualPrivateHandoff) resultsRoot = path.join(manualPrivateHandoff.root, "results"); if (options.installedStartupOnly) { assertInstalledStartupOnly(process.env, executions, options); assertRunnerE2EPrerequisites(executions); @@ -1226,4 +1238,6 @@ async function main() { await main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 1; -}); +}).finally(async () => { + if (manualPrivateHandoff) await sealManualResultEvidence(manualPrivateHandoff, [...manualEvidenceSecrets]); +}).catch(() => { process.stderr.write("Private evidence finalization failed; no public fallback\n"); process.exitCode = 1; }); diff --git a/tests/runner-e2e/manual-private-handoff.test.ts b/tests/runner-e2e/manual-private-handoff.test.ts new file mode 100644 index 00000000000..ff7fd0b0d9e --- /dev/null +++ b/tests/runner-e2e/manual-private-handoff.test.ts @@ -0,0 +1,73 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdtemp, mkdir, realpath, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { readManualPrivateHandoff, sealStandardOriginalEvidence, preserveOriginalFrameworkResult, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +const roots: string[]=[]; +afterEach(async()=>{await Promise.all(roots.splice(0).map(p=>rm(p,{recursive:true,force:true})));}); +async function fixture() { + const root=await mkdtemp(path.join(os.tmpdir(),"manual-private-handoff-test-"));roots.push(root); + const source=path.join(root,"source"),out=path.join(root,"private");await mkdir(source,{mode:0o700});await mkdir(out,{mode:0o700}); + const context={root:out,protocol:MANUAL_PRIVATE_PROTOCOL,private:true,workflowSha:"a".repeat(40),targetSha:"b".repeat(40),runId:"123",attempt:"1",executionIds:[MANUAL_PRIVATE_EXECUTIONS[0]!]}as ManualPrivateHandoff; + await mkdir(path.join(source,"snapshots"));await writeFile(path.join(source,"snapshots","state.json"),' { "legitimatePrivateContent": "Keep this exact original formatting" }\n'); + await writeFile(path.join(source,"failure.png"),Buffer.from([137,80,78,71,13,10,26,10])); + return{root,source,out,context,input:{context,executionId:context.executionIds[0]!,source,evidence:{files:[],leaks:[],missing:[]},secrets:[]as string[],expectPassScreenshot:false}}; +} +it("retains exact approved original bytes and hashes without expanding evidence scope",async()=>{ + const f=await fixture();await writeFile(path.join(f.source,"unapproved-home.json"),'not an evidence root'); + const receipt=await sealStandardOriginalEvidence(f.input);expect(receipt.complete).toBe(true);expect(receipt.files.map(r=>r.path)).toEqual(["failure.png","snapshots/state.json"]); + const directory=path.join(f.out,"originals");const {readdir}=await import("node:fs/promises");const [cell]=await readdir(directory); + expect(await readFile(path.join(directory,cell!,"snapshots/state.json"),"utf8")).toBe(await readFile(path.join(f.source,"snapshots/state.json"),"utf8")); +}); +it("preserves the original framework result before later launcher rewrites",async()=>{ + const f=await fixture();await writeFile(path.join(f.source,"result.json"),' {"original":true}\n');await preserveOriginalFrameworkResult(f.source);await writeFile(path.join(f.source,"result.json"),'{"later":true}'); + expect(await readFile(path.join(f.source,"snapshots/manual-original-framework-result.json"),"utf8")).toBe(' {"original":true}\n'); +}); +it("a genuine credential match remains fatal and no contaminated original is sealed",async()=>{ + const f=await fixture();f.input.secrets=["synthetic-provider-credential-1234567890"];await writeFile(path.join(f.source,"snapshots/secret.json"),JSON.stringify({value:f.input.secrets[0]})); + await expect(sealStandardOriginalEvidence(f.input)).rejects.toThrow(/privacy/);expect(JSON.parse(await readFile(path.join(f.out,"contamination.json"),"utf8"))).toMatchObject({complete:false,status:"withheld_credential_contamination"}); +}); +it("retains all safe available originals while required pass evidence remains explicitly incomplete",async()=>{ + const f=await fixture();f.input.expectPassScreenshot=true;const receipt=await sealStandardOriginalEvidence(f.input); + expect(receipt.complete).toBe(true);expect(receipt.requiredEvidenceComplete).toBe(false); + expect(receipt.missingRequiredEvidence).toContain("final-state.png");expect(receipt.missingRequiredEvidence).toContain("blob-report/*.zip"); + expect(receipt.files.map(row=>row.path)).toContain("failure.png"); +}); +it("does not copy an escaped or symlinked original",async()=>{ + const f=await fixture();await symlink(path.join(f.source,"snapshots/state.json"),path.join(f.source,"snapshots/link.json"));await expect(sealStandardOriginalEvidence(f.input)).rejects.toThrow(/unsupported/); +}); +it("requires the complete preflight's exact attempt, owner-only file and target before selecting a private root",async()=>{ + const f=await fixture();const file=path.join(f.out,"preflight.json");await writeFile(file,JSON.stringify({...f.context,status:"verified"}),{mode:0o600}); + const env={GITHUB_SHA:f.context.workflowSha,RUNNER_TEMP:f.root,GITHUB_RUN_ID:"123",GITHUB_RUN_ATTEMPT:"1",PAPERCLIP_RUNNER_E2E_SOURCE_SHA:f.context.targetSha,PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT:file}; + expect((await readManualPrivateHandoff(env,f.context.executionIds))?.root).toBe(await realpath(f.out)); + await expect(readManualPrivateHandoff({...env,GITHUB_RUN_ID:"124"},f.context.executionIds)).rejects.toThrow(/exact attempt/); + await expect(readManualPrivateHandoff(env,[MANUAL_PRIVATE_EXECUTIONS[1]!])).rejects.toThrow(/exact attempt/); +}); +it.each(["unknown.bin", "private.db"])("rejects unscanned %s under a standard directory without retaining it",async name=>{ + const f=await fixture();f.input.secrets=["synthetic-provider-credential-1234567890"]; + await writeFile(path.join(f.source,"snapshots",name),f.input.secrets[0]!); + await expect(sealStandardOriginalEvidence(f.input)).rejects.toThrow(/Unsupported/); +}); +it("an active SVG omitted by the existing package cannot be called a complete retained original",async()=>{ + const f=await fixture();await writeFile(path.join(f.source,"snapshots/active.svg"),''); + await expect(sealStandardOriginalEvidence(f.input)).rejects.toThrow(/Unsupported/); +}); + +it.each(["raw", "url", "base64", "shape"])("rejects %s credential-bearing original names rather than sanitizing a complete inventory", async kind => { + const f=await fixture();const secret="synthetic-credential/+1234567890";f.input.secrets=[secret]; + const name=kind==="raw"?"synthetic-credential-1234567890":kind==="url"?encodeURIComponent(secret):kind==="base64"?Buffer.from(secret).toString("base64url"):"sk-proj-synthetic123456789012345"; + if(kind==="raw")f.input.secrets=[name]; + await writeFile(path.join(f.source,"snapshots",`${name}.json`),'{}'); + await expect(sealStandardOriginalEvidence(f.input)).rejects.toThrow(/Secret leak/); +}); +it("retains original framework PASS and actual missing-evidence infrastructure FAIL as separate bytes",async()=>{ + const f=await fixture();await writeFile(path.join(f.source,"result.json"),'{"status":"passed"}'); + await preserveOriginalFrameworkResult(f.source); + await writeFile(path.join(f.source,"result.json"),'{"status":"failed","failureClass":"permanent_infrastructure","error":"Required evidence missing: final-state.png"}'); + const receipt=await sealStandardOriginalEvidence({...f.input,expectPassScreenshot:true}); + expect(receipt.requiredEvidenceComplete).toBe(false); + const {manualCellKind}=await import("./manual-private-retention.js"),dir=path.join(f.out,"originals",manualCellKind(f.input.executionId)); + expect(JSON.parse(await readFile(path.join(dir,"snapshots/manual-original-framework-result.json"),"utf8")).status).toBe("passed"); + expect(JSON.parse(await readFile(path.join(dir,"result.json"),"utf8"))).toMatchObject({status:"failed",failureClass:"permanent_infrastructure"}); +}); diff --git a/tests/runner-e2e/manual-private-handoff.ts b/tests/runner-e2e/manual-private-handoff.ts new file mode 100644 index 00000000000..92143c47852 --- /dev/null +++ b/tests/runner-e2e/manual-private-handoff.ts @@ -0,0 +1,174 @@ +import { constants } from "node:fs"; +import { createHash } from "node:crypto"; +import { copyFile, lstat, mkdir, open, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { packageEvidence, isStandardEvidencePath, isStandardEvidenceFile, type EvidencePackageResult } from "./evidence.js"; +import { canonicalGeneratedReportAsset, isTrustedGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { assertSecretFree, normalizedSecrets } from "./redaction.js"; +import { MANUAL_PRIVATE_PROTOCOL, manualCellKind, type ManualRetentionIdentity } from "./manual-private-retention.js"; + +export interface ManualPrivateHandoff extends ManualRetentionIdentity { + protocol: typeof MANUAL_PRIVATE_PROTOCOL; private: true; root: string; +} +export async function readManualPrivateHandoff(env: NodeJS.ProcessEnv, ids: string[]): Promise { + const file = env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT; + if (!file) return undefined; + const metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > 65536 || metadata.mode & 0o077 + || typeof process.getuid === "function" && metadata.uid !== process.getuid()) throw new Error("Invalid private evidence preflight custody"); + const receipt = JSON.parse(await readFile(file, "utf8")) as ManualPrivateHandoff & { status: string }; + if (receipt.protocol !== MANUAL_PRIVATE_PROTOCOL || receipt.status !== "verified" || receipt.private !== true + || receipt.runId !== env.GITHUB_RUN_ID || receipt.attempt !== env.GITHUB_RUN_ATTEMPT || receipt.attempt !== "1" + || receipt.workflowSha !== env.GITHUB_SHA || receipt.targetSha !== env.PAPERCLIP_RUNNER_E2E_SOURCE_SHA || receipt.executionIds.length !== ids.length + || [...receipt.executionIds].sort().join() !== [...ids].sort().join()) throw new Error("Private evidence preflight does not match this exact attempt"); + const root = await realpath(receipt.root), temporary = await realpath(env.RUNNER_TEMP ?? ""); + const directory = await lstat(root); + if (!root.startsWith(`${temporary}${path.sep}`) || root !== path.dirname(await realpath(file)) + || !directory.isDirectory() || directory.isSymbolicLink() || directory.mode & 0o077 + || typeof process.getuid === "function" && directory.uid !== process.getuid()) throw new Error("Invalid private evidence staging ownership"); + return Object.freeze({ ...receipt, root }); +} +export async function preserveOriginalFrameworkResult(directory: string) { + const source = path.join(directory, "result.json"); + const info = await lstat(source).catch(error => { if (error.code === "ENOENT") return undefined; throw error; }); + if (!info) return; + if (!info.isFile() || info.isSymbolicLink()) throw new Error("Invalid original framework result"); + const destination = path.join(directory, "snapshots", "manual-original-framework-result.json"); + await mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); + await copyFile(source, destination, constants.COPYFILE_EXCL); +} +export async function sealStandardOriginalEvidence(input: { + context: ManualPrivateHandoff; executionId: string; source: string; evidence: EvidencePackageResult; secrets: readonly string[]; expectPassScreenshot: boolean; +}) { + if (!input.context.executionIds.includes(input.executionId)) throw new Error("Private evidence execution is outside preflight"); + const secrets = normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]); + const out = path.join(input.context.root, "originals", manualCellKind(input.executionId)); + const root = await realpath(input.source), info = await lstat(input.source); + if (!info.isDirectory() || info.isSymbolicLink()) throw new Error("Invalid original standard evidence root"); + if (input.evidence.leaks.length) { + await writeFile(path.join(input.context.root, "contamination.json"), JSON.stringify({ protocol: MANUAL_PRIVATE_PROTOCOL, + status: "withheld_credential_contamination", complete: false }), { mode: 0o600, flag: "wx" }); + throw new Error("Credential-contaminated original evidence cannot be retained"); + } + const parent = path.dirname(out); + await mkdir(parent, { recursive: true, mode: 0o700 }); + if (!(await realpath(parent)).startsWith(`${await realpath(input.context.root)}${path.sep}`)) throw new Error("Private original handoff escaped its preflight root"); + await mkdir(out, { mode: 0o700 }); + const records: Array<{ path: string; bytes: number; sha256: string }> = []; + let visited = 0, bytes = 0; + let missingRequiredEvidence: string[] = []; + async function visit(relative: string) { + for (const entry of await readdir(path.join(root, relative), { withFileTypes: true })) { + if (++visited > 4096) throw new Error("Complete original evidence exceeds entry allowance"); + const name = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isSymbolicLink() || !entry.isDirectory() && !entry.isFile()) throw new Error("Original evidence contains an unsupported entry"); + if (entry.isDirectory()) { + if (name.split("/").length > 24) throw new Error("Original evidence exceeds depth allowance"); + if (isStandardEvidencePath(`${name}/placeholder`)) await visit(name); + continue; + } + if (!isStandardEvidencePath(name)) continue; + if (!isStandardEvidenceFile(name)) throw new Error("Unsupported original evidence carrier cannot be retained"); + assertSecretFree(name, secrets, "private evidence path"); + const source = path.join(root, name), target = path.join(out, name); + if (!(await realpath(source)).startsWith(`${root}${path.sep}`)) throw new Error("Original evidence escaped its owned root"); + const handle = await open(source, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const before = await handle.stat(); bytes += before.size; + if (!before.isFile() || !Number.isSafeInteger(bytes) || bytes > 256 * 1024 * 1024) throw new Error("Complete original evidence exceeds byte allowance"); + await mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); + const output = await open(target, "wx", 0o600), hash = createHash("sha256"), buffer = Buffer.alloc(65536); + try { + let copied = 0; + while (copied < before.size) { + const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, before.size - copied), copied); + if (!bytesRead) throw new Error("Original evidence changed during handoff"); + hash.update(buffer.subarray(0, bytesRead)); let written = 0; + while (written < bytesRead) { const part = await output.write(buffer, written, bytesRead - written, copied + written); if (!part.bytesWritten) throw new Error("Original evidence copy stopped"); written += part.bytesWritten; } + copied += bytesRead; + } + const after = await handle.stat(); + if (after.size !== before.size || after.mtimeMs !== before.mtimeMs || after.ctimeMs !== before.ctimeMs) throw new Error("Original evidence changed during handoff"); + await output.sync(); records.push({ path: name, bytes: copied, sha256: hash.digest("hex") }); + } finally { await output.close(); } + } finally { await handle.close(); } + } + } + try { + await visit(""); + const validation = path.join(input.context.root, `validation-${manualCellKind(input.executionId)}`); + await mkdir(validation, { mode: 0o700 }); + try { + const scan = await packageEvidence({ privateDir: out, uploadDir: validation, secrets, expectPassScreenshot: input.expectPassScreenshot }); + missingRequiredEvidence = scan.missing; + if (scan.leaks.length) { + await writeFile(path.join(input.context.root, "contamination.json"), JSON.stringify({ protocol: MANUAL_PRIVATE_PROTOCOL, status: "withheld_credential_contamination", complete: false }), { mode: 0o600, flag: "wx" }); + throw new Error("Original evidence failed unchanged privacy or required-file checks"); + } + } finally { await rm(validation, { recursive: true, force: true }); } + records.sort((a, b) => a.path.localeCompare(b.path)); + const receipt = { protocol: MANUAL_PRIVATE_PROTOCOL, executionId: input.executionId, targetSha: input.context.targetSha, + complete: true, requiredEvidenceComplete: input.expectPassScreenshot ? missingRequiredEvidence.length === 0 : null, + missingRequiredEvidence, sourceBytes: bytes, files: records, inventorySha256: createHash("sha256").update(JSON.stringify(records)).digest("hex") }; + const serialized = `${JSON.stringify(receipt, null, 2)}\n`; + assertSecretFree(serialized, secrets, "private evidence inventory"); + await writeFile(path.join(out, "original-evidence-inventory.json"), serialized, { mode: 0o600, flag: "wx" }); + return receipt; + } catch (error) { await rm(out, { recursive: true, force: true }); throw error; } +} + +export async function sealManualResultEvidence(context: ManualPrivateHandoff, secrets: readonly string[], source = path.join(context.root, "results"), treeKind: "standard" | "generated-report" = "standard") { + const root = await realpath(source), owned = await realpath(context.root); + if (!root.startsWith(`${owned}${path.sep}`)) throw new Error("Packaged evidence escaped its owned root"); + const wrapper = path.join(context.root, "result-validation"); + const { cp, rename } = await import("node:fs/promises"); + await mkdir(wrapper, { mode: 0o700 }); + const records: Array<{ path: string; bytes: number; sha256: string }> = []; + let bytes = 0; + const scanSecrets = normalizedSecrets(secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]); + async function visit(relative: string) { + for (const entry of await readdir(path.join(root, relative), { withFileTypes: true })) { + const name = relative ? `${relative}/${entry.name}` : entry.name; + assertSecretFree(name, scanSecrets, "private packaged evidence path"); + if (entry.isSymbolicLink() || !entry.isFile() && !entry.isDirectory() || name.split("/").length > 24) throw new Error("Invalid packaged evidence entry"); + if (entry.isDirectory()) { await visit(name); continue; } + const standard = isStandardEvidenceFile(`html-report/${name}`); + const canonical = treeKind === "generated-report" ? canonicalGeneratedReportAsset(name) : undefined; + const staticReportOwner = treeKind === "generated-report" && (name.startsWith("html/trace/") || canonical !== undefined); + if (staticReportOwner ? !canonical : !standard) throw new Error("Unsupported packaged evidence carrier"); + const file = path.join(root, name), metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink()) throw new Error("Invalid packaged evidence entry"); + if (metadata.size > 256 * 1024 * 1024 - bytes || records.length >= 4096) throw new Error("Complete packaged evidence exceeds allowance"); + const data = await readFile(file); + if (staticReportOwner) { + if (!await isTrustedGeneratedReportAsset(name, data)) throw new Error("Unsupported packaged evidence carrier"); + assertSecretFree(data, scanSecrets, "authenticated static report asset"); + } + bytes += data.length; + if (bytes > 256 * 1024 * 1024) throw new Error("Complete packaged evidence exceeds allowance"); + records.push({ path: name, bytes: data.length, sha256: createHash("sha256").update(data).digest("hex") }); + } + } + try { + await visit(""); + if (!records.length) throw new Error("No genuine packaged evidence was produced"); + await cp(root, path.join(wrapper, "html-report"), { recursive: true, errorOnExist: true, force: false }); + const validation = path.join(context.root, "packaged-scan"); + try { + const scan = await packageEvidence({ privateDir: wrapper, uploadDir: validation, secrets: scanSecrets, expectPassScreenshot: false }); + if (scan.leaks.length) throw new Error("Packaged evidence contains credential contamination"); + for (const record of records) { + if (createHash("sha256").update(await readFile(path.join(wrapper, "html-report", record.path))).digest("hex") !== record.sha256) + throw new Error("Packaged evidence changed during handoff"); + } + } finally { await rm(validation, { recursive: true, force: true }); } + records.sort((a, b) => a.path.localeCompare(b.path)); + const receipt = { protocol: MANUAL_PRIVATE_PROTOCOL, treeKind, complete: true, sourceBytes: bytes, files: records, + inventorySha256: createHash("sha256").update(JSON.stringify(records)).digest("hex") }; + const serialized = `${JSON.stringify(receipt, null, 2)}\n`; + assertSecretFree(serialized, scanSecrets, "private packaged inventory"); + await writeFile(path.join(wrapper, "result-evidence-inventory.json"), serialized, { mode: 0o600, flag: "wx" }); + await rename(wrapper, path.join(context.root, "sealed-results")); + return receipt; + } catch (error) { await rm(wrapper, { recursive: true, force: true }); throw error; } +} diff --git a/tests/runner-e2e/manual-private-policy.mjs b/tests/runner-e2e/manual-private-policy.mjs new file mode 100644 index 00000000000..57bf2862dd4 --- /dev/null +++ b/tests/runner-e2e/manual-private-policy.mjs @@ -0,0 +1,39 @@ +import { appendFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; +import { canonicalManualIds, MANUAL_PRIVATE_EXECUTIONS, requiresPrivateManualEvidence } from './manual-private-retention.ts'; + +const values = value => (value ?? '').split(',').map(item => item.trim()).filter(Boolean); +export function manualEvidenceRouting(env) { + if (env.EVENT_NAME !== 'workflow_dispatch' || env.EVAL_CADENCE !== 'manual') return { retention: 'recurring', executionIds: [] }; + const ids = values(env.SELECT_ID), selectors = ['SELECT_SUITE', 'SELECT_GROUP', 'SELECT_PROFILE', 'SELECT_ENVIRONMENT', 'SELECT_CASE']; + if ((env.SELECT_ID ?? '') && selectors.some(key => env[key])) throw new Error('Exact IDs are exclusive with other selectors'); + if (!ids.length) { + if (values(env.SELECT_SUITE).includes('chat-features')) throw new Error('Explicit reviewed IDs required for private manual execution'); + return { retention: 'actions', executionIds: [] }; + } + if (!ids.some(requiresPrivateManualEvidence)) return { retention: 'actions', executionIds: ids }; + const selected = canonicalManualIds(ids); + if (selected.some(id => !MANUAL_PRIVATE_EXECUTIONS.includes(id))) throw new Error('Unsupported private manual selection'); + if (env.TEST_RUNNER !== 'runs-on/fleet=paperclip-public-pr-x64/env=public-ci') throw new Error('Private manual evidence currently requires the reviewed AWS route'); + return { retention: 'private', executionIds: selected }; +} +export function authorizeManualCellRoute(retention, executionId, privateIds) { + const ids = canonicalManualIds([executionId]); + if (retention === 'private') { + if (!MANUAL_PRIVATE_EXECUTIONS.includes(ids[0]) || !canonicalManualIds(JSON.parse(privateIds)).includes(ids[0])) throw new Error('Private cell differs from trusted selection'); + return 'private'; + } + if (retention !== 'actions' || requiresPrivateManualEvidence(ids[0])) throw new Error('Sensitive cell cannot enter public manual artifacts'); + return 'actions'; +} +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + if (process.argv[2] === 'cell') { + const route = authorizeManualCellRoute(process.env.EVIDENCE_RETENTION, process.env.SELECTED_EXECUTION_ID, process.env.MANUAL_PRIVATE_IDS); + appendFileSync(process.env.GITHUB_OUTPUT, `route=${route}\n`); + } else { + const result = manualEvidenceRouting(process.env); + appendFileSync(process.env.GITHUB_OUTPUT, `evidence_retention=${result.retention}\nmanual_private_ids=${JSON.stringify(result.executionIds)}\n`); + } + } catch { process.stderr.write('Manual evidence authorization failed before paid access; details withheld\n'); process.exitCode = 1; } +} diff --git a/tests/runner-e2e/manual-private-policy.test.ts b/tests/runner-e2e/manual-private-policy.test.ts new file mode 100644 index 00000000000..f2d53c28ead --- /dev/null +++ b/tests/runner-e2e/manual-private-policy.test.ts @@ -0,0 +1,30 @@ +import { expect, it, vi } from "vitest"; +import { manualEvidenceRouting, authorizeManualCellRoute } from "./manual-private-policy.mjs"; +import { MANUAL_PRIVATE_EXECUTIONS } from "./manual-private-retention.js"; +vi.mock("./catalog.js",()=>({runnerMatrix:[]})); +import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import type { MatrixExecution } from "./types.js"; +const env={EVENT_NAME:"workflow_dispatch",EVAL_CADENCE:"manual",TEST_RUNNER:"runs-on/fleet=paperclip-public-pr-x64/env=public-ci"}; +it.each(["chat-features"," chat-features ","first-task, chat-features "])("rejects broad sensitive suite %s before credentials",suite=>expect(()=>manualEvidenceRouting({...env,SELECT_SUITE:suite})).toThrow()); +it("classifies exact private IDs without granting run approval",()=>expect(manualEvidenceRouting({...env,SELECT_ID:` ${MANUAL_PRIVATE_EXECUTIONS[0]} `})).toEqual({retention:"private",executionIds:[MANUAL_PRIVATE_EXECUTIONS[0]]})); +it.each(["chat-features","chat-features.runner-opencode.local.*","chat-features.runner-opencode.local.unknown"])("rejects partial or unsupported sensitive ID %s",id=>expect(()=>manualEvidenceRouting({...env,SELECT_ID:id})).toThrow()); +it("keeps selector exclusivity equal to the actual catalog producer",()=>expect(()=>manualEvidenceRouting({...env,SELECT_ID:MANUAL_PRIVATE_EXECUTIONS[0],SELECT_SUITE:" "})).toThrow()); +it("retains nonprivate suite and exact-cell routes but requires a successful trusted cell guard",()=>{ + expect(manualEvidenceRouting({...env,SELECT_SUITE:"first-task"})).toMatchObject({retention:"actions"}); + expect(authorizeManualCellRoute("actions","first-task.runner-opencode.local.create","[]")).toBe("actions"); + expect(()=>authorizeManualCellRoute("actions",MANUAL_PRIVATE_EXECUTIONS[0],"[]")).toThrow(); + expect(()=>authorizeManualCellRoute(undefined,"first-task.runner-opencode.local.create","[]")).toThrow(); +}); +it("rejects target-selected foreign private cells and unsupported compute",()=>{ + expect(()=>authorizeManualCellRoute("private",MANUAL_PRIVATE_EXECUTIONS[1],JSON.stringify([MANUAL_PRIVATE_EXECUTIONS[0]]))).toThrow(); + expect(()=>manualEvidenceRouting({...env,TEST_RUNNER:"ubuntu-latest",SELECT_ID:MANUAL_PRIVATE_EXECUTIONS[0]})).toThrow(); +}); + +it.each(["33-known-ids","duplicate-known-id"])("preserves actual nonprivate catalog semantics for %s through classifier and cell guard",kind=>{ + const matrix=Array.from({length:33},(_,i)=>({id:`first-task.runner-opencode.local.fixture-${i}`,groups:[],suite:{id:"first-task"},profile:{id:"runner-opencode"},environment:{id:"local"},task:{id:`fixture-${i}`}} as unknown as MatrixExecution)); + const ids=kind==="duplicate-known-id"?[matrix[0]!.id,matrix[0]!.id]:matrix.map(row=>row.id); + const options=parseRunnerSelectors(["--matrix-json","--max-parallel","1",...ids.flatMap(id=>["--id",id])]); + const selected=selectRunnerExecutions(options,matrix);expect(selected.length).toBe(kind==="duplicate-known-id"?1:33); + const route=manualEvidenceRouting({...env,SELECT_ID:ids.join(",")});expect(route).toEqual({retention:"actions",executionIds:ids}); + for(const row of selected)expect(authorizeManualCellRoute(route.retention,row.id,"[]")).toBe("actions"); +}); diff --git a/tests/runner-e2e/manual-private-report-assets.test.ts b/tests/runner-e2e/manual-private-report-assets.test.ts new file mode 100644 index 00000000000..c6afba82121 --- /dev/null +++ b/tests/runner-e2e/manual-private-report-assets.test.ts @@ -0,0 +1,58 @@ +import { afterEach, expect, it } from "vitest"; +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { canonicalGeneratedReportAsset, isTrustedGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { sealManualResultEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +import { validateSealedInventory } from "./manual-private-transfer.js"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function fixture(name = "normalized/assets/InterVariable.woff2") { + const root = await mkdtemp(path.join(os.tmpdir(), "synthetic-report-asset-")); roots.push(root); + const source = path.join(root, "source"), bytes = await readFile(canonicalGeneratedReportAsset(name)!); + await mkdir(path.dirname(path.join(source, name)), { recursive: true }); await writeFile(path.join(source, name), bytes); + const context = { root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff; + return { root, source, name, bytes, context }; +} +it.each(["normalized/assets/InterVariable.woff2", "html/trace/playwright-logo.svg"])("default cell/result admission still rejects %s", async name => { + const f = await fixture(name); await expect(sealManualResultEvidence(f.context, [], f.source)).rejects.toThrow("Unsupported packaged evidence carrier"); +}); +it.each(["wrong-path", "tampered", "unknown", "unowned-svg"])("refuses report asset %s without omitting the original", async kind => { + const f = await fixture(); + if (kind === "tampered") await writeFile(path.join(f.source, f.name), Buffer.concat([f.bytes, Buffer.from("changed")])); + else { const name = kind === "wrong-path" ? "normalized/assets/other.woff2" : kind === "unknown" ? "html/trace/unknown.bin" : "normalized/assets/other.svg"; await mkdir(path.dirname(path.join(f.source, name)), { recursive: true }); await writeFile(path.join(f.source, name), f.bytes); } + await expect(sealManualResultEvidence(f.context, [], f.source, "generated-report")).rejects.toThrow("Unsupported packaged evidence carrier"); + expect(await readFile(path.join(f.source, f.name))).toBeInstanceOf(Buffer); +}); +it("checks registered roots even in an authenticated canonical binary", async () => { + const f = await fixture(); await expect(sealManualResultEvidence(f.context, ["wOF2"], f.source, "generated-report")).rejects.toThrow("Secret leak"); +}); +it("revalidates canonical content identity after an attacker recomputes the inventory hash", async () => { + const f = await fixture(); await sealManualResultEvidence(f.context, [], f.source, "generated-report"); + const directory = path.join(f.root, "sealed-results/html-report"), inventoryPath = path.join(f.root, "sealed-results/result-evidence-inventory.json"); + const inventory = JSON.parse(await readFile(inventoryPath, "utf8")), changed = Buffer.from(f.bytes); changed[changed.length - 1] ^= 1; + await writeFile(path.join(directory, f.name), changed); inventory.files[0].sha256 = createHash("sha256").update(changed).digest("hex"); + inventory.inventorySha256 = createHash("sha256").update(JSON.stringify(inventory.files)).digest("hex"); await writeFile(inventoryPath, JSON.stringify(inventory)); + await expect(validateSealedInventory(directory, "../result-evidence-inventory.json", undefined, "generated-report")).rejects.toThrow("Invalid original evidence inventory row"); +}); +it("rejects map/watch/traversal and foreign asset owners", async () => { + for (const name of ["html/trace/../InterVariable.woff2", "html/trace/assets/xtermModule.js", "html/trace/index.js.map", "html/trace/watch.js", "normalized/assets/other.woff2", "snapshots/private.db"]) + expect(await isTrustedGeneratedReportAsset(name, Buffer.from("synthetic"))).toBe(false); +}); +it.each(["tampered-index", "watch", "xterm"])("sealer rejects standard-extension trace owner bypass %s", async kind => { + const f = await fixture("html/trace/index.html"); + if (kind === "tampered-index") await writeFile(path.join(f.source, f.name), Buffer.concat([f.bytes, Buffer.from("")])); + else { const name = kind === "watch" ? "html/trace/watch.js" : "html/trace/assets/xtermModule.synthetic.js"; await mkdir(path.dirname(path.join(f.source, name)), { recursive: true }); await writeFile(path.join(f.source, name), "synthetic unapproved script"); } + await expect(sealManualResultEvidence(f.context, [], f.source, "generated-report")).rejects.toThrow("Unsupported packaged evidence carrier"); +}); +it.each(["tampered-index", "watch", "xterm"])("validator rejects a rehashed standard-extension trace owner bypass %s", async kind => { + const root = await mkdtemp(path.join(os.tmpdir(), "synthetic-trace-owner-")); roots.push(root); + const name = kind === "tampered-index" ? "html/trace/index.html" : kind === "watch" ? "html/trace/watch.js" : "html/trace/assets/xtermModule.synthetic.js"; + const data = kind === "tampered-index" ? Buffer.concat([await readFile(canonicalGeneratedReportAsset(name)!), Buffer.from("")]) : Buffer.from("synthetic unapproved script"); + await mkdir(path.dirname(path.join(root, name)), { recursive: true }); await writeFile(path.join(root, name), data); + const files = [{ path: name, bytes: data.length, sha256: createHash("sha256").update(data).digest("hex") }]; + await writeFile(path.join(root, "inventory.json"), JSON.stringify({ protocol: MANUAL_PRIVATE_PROTOCOL, treeKind: "generated-report", complete: true, sourceBytes: data.length, files, inventorySha256: createHash("sha256").update(JSON.stringify(files)).digest("hex") })); + await expect(validateSealedInventory(root, "inventory.json", undefined, "generated-report")).rejects.toThrow("Invalid original evidence inventory row"); +}); diff --git a/tests/runner-e2e/manual-private-report-assets.ts b/tests/runner-e2e/manual-private-report-assets.ts new file mode 100644 index 00000000000..73fc61fb3fa --- /dev/null +++ b/tests/runner-e2e/manual-private-report-assets.ts @@ -0,0 +1,25 @@ +import { createRequire } from "node:module"; +import { lstat, readFile } from "node:fs/promises"; +import path from "node:path"; + +const require = createRequire(import.meta.url); +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +export function canonicalGeneratedReportAsset(relative: string): string | undefined { + if (relative === "normalized/assets/InterVariable.woff2") return path.join(repositoryRoot, "ui/public/fonts/InterVariable.woff2"); + if (!relative.startsWith("html/trace/")) return; + const parts = relative.slice("html/trace/".length).split("/"); + if (parts.some(part => !part || part === "." || part === ".." || /[\\:\x00-\x1f]/.test(part))) return; + if (parts.length === 1) { + if (parts[0]!.endsWith(".map") || parts[0]!.includes("watch") || parts[0]!.includes("assets")) return; + } else if (parts.length !== 2 || parts[0] !== "assets" || parts[1]!.endsWith(".map") || parts[1]!.includes("xtermModule")) return; + const test = require.resolve("@playwright/test/package.json"); + const playwright = require.resolve("playwright/package.json", { paths: [path.dirname(test)] }); + const core = require.resolve("playwright-core/package.json", { paths: [path.dirname(playwright)] }); + return path.join(path.dirname(core), "lib/vite/traceViewer", ...parts); +} +export async function isTrustedGeneratedReportAsset(relative: string, bytes: Buffer): Promise { + const source = canonicalGeneratedReportAsset(relative); + if (!source) return false; + const metadata = await lstat(source).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return undefined; throw error; }); + return !!metadata && metadata.isFile() && !metadata.isSymbolicLink() && metadata.size === bytes.length && bytes.equals(await readFile(source)); +} diff --git a/tests/runner-e2e/manual-private-report-integration.test.ts b/tests/runner-e2e/manual-private-report-integration.test.ts new file mode 100644 index 00000000000..c85f0a01e7e --- /dev/null +++ b/tests/runner-e2e/manual-private-report-integration.test.ts @@ -0,0 +1,83 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const transport = vi.hoisted(() => ({ objects: new Map() })); +vi.mock("./history-budget.js", async original => { + const actual = await original(); + return { ...actual, BoundedHistoryPublisher: class extends actual.BoundedHistoryPublisher { + constructor(receipt: string) { super(receipt, async args => { + const operation = args[1]; + if (operation === "put-object") { + expect(args.slice(args.indexOf("--if-none-match"), args.indexOf("--if-none-match") + 2)).toEqual(["--if-none-match", "*"]); + const key = args[args.indexOf("--key") + 1]!; + expect(transport.objects.has(key)).toBe(false); + transport.objects.set(key, await readFile(args[args.indexOf("--body") + 1]!)); + return { stdout: JSON.stringify({ VersionId: `synthetic-version-${transport.objects.size}` }) }; + } + if (operation === "get-object-lock-configuration" || operation === "get-bucket-replication") + throw new Error(`(${operation === "get-object-lock-configuration" ? "ObjectLockConfigurationNotFoundError" : "ReplicationConfigurationNotFoundError"})`); + const configurations: Record = { + "get-bucket-versioning": { Status: "Enabled" }, + "get-bucket-lifecycle-configuration": { Rules: [ + { Status: "Enabled", Filter: { Prefix: "runner-e2e-private/bounded-v1/" }, Expiration: { Days: 30 }, NoncurrentVersionExpiration: { NoncurrentDays: 1 }, AbortIncompleteMultipartUpload: { DaysAfterInitiation: 1 } }, + { Status: "Enabled", Filter: { Prefix: "runner-e2e-private/bounded-v1/" }, Expiration: { ExpiredObjectDeleteMarker: true } }, + ] }, + "get-public-access-block": { PublicAccessBlockConfiguration: { BlockPublicAcls: true, IgnorePublicAcls: true, BlockPublicPolicy: true, RestrictPublicBuckets: true } }, + "get-bucket-ownership-controls": { OwnershipControls: { Rules: [{ ObjectOwnership: "BucketOwnerEnforced" }] } }, + "get-bucket-policy": { Policy: JSON.stringify({ Statement: [{ Effect: "Deny" }] }) }, + }; + if (!(operation! in configurations)) throw new Error("Unexpected synthetic transport operation"); + return { stdout: JSON.stringify(configurations[operation!]) }; + }); } + } }; +}); +import { stageDashboardBrandAssets } from "./report-assets.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; +import { archivePrivateEvidence, privateArtifactKey } from "./private-history.js"; +import { canonicalGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { uploadReport } from "./manual-private-transfer.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +const roots: string[] = []; +afterEach(async () => { transport.objects.clear(); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); + +it("archives the actual synthetic Node-only Playwright and dashboard producers with immutable byte-bound receipts", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "synthetic-private-report-")); roots.push(root); + const source = path.join(root, "report"), tests = path.join(root, "node-tests"), trace = path.join(root, "synthetic-trace"); + await Promise.all([mkdir(source), mkdir(tests), mkdir(trace)]); + await writeFile(path.join(trace, "trace.trace"), '{"synthetic":true}\n'); + const traceZip = path.join(root, "synthetic-trace.zip"); await archivePrivateEvidence("pack", trace, traceZip); + const require = createRequire(import.meta.url), testPackage = require.resolve("@playwright/test/package.json"), testEntry = require.resolve("@playwright/test"); + expect(JSON.parse(await readFile(testPackage, "utf8")).version).toBe("1.62.1"); + await writeFile(path.join(tests, "synthetic-node-report.spec.cjs"), `const {test,expect}=require(${JSON.stringify(testEntry)});test('synthetic Node metadata only',async({},info)=>{expect(1).toBe(1);await info.attach('trace',{path:${JSON.stringify(traceZip)},contentType:'application/zip'});});`); + const config = path.join(tests, "synthetic.config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "synthetic-node-report.spec.cjs", workers: 1, use: { trace: "off" }, reporter: [["html", { outputFolder: path.join(source, "html"), open: "never" }], ["junit", { outputFile: path.join(source, "playwright-junit.xml") }], ["blob", { outputDir: path.join(source, "blob-report") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(testPackage), "cli.js"), "test", "--config", config], { + cwd: tests, timeout: 30_000, maxBuffer: 1024 * 1024, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "", PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" }, + }); + const normalized = path.join(source, "normalized"); await mkdir(normalized); await stageDashboardBrandAssets(normalized); + const dashboard = renderRunnerE2EDashboard({ title: "Synthetic private-retention metadata", generatedAt: "2026-10-29T00:00:00Z", expected: [], catalog: [], entries: [] }); + await writeFile(path.join(normalized, "index.html"), dashboard); + const workflowSha = "a".repeat(40), targetSha = "b".repeat(40), id = MANUAL_PRIVATE_EXECUTIONS[0]!; + await writeFile(path.join(root, "manual-report-input-completeness.json"), JSON.stringify({ meaning: "available-files-and-transfer-only", doesNotAttestFeatureOutcome: true, cells: [{ executionId: id, availableFileInventoryComplete: true, requiredPassEvidenceComplete: null, missingRequiredEvidence: [] }] })); + await uploadReport({ MANUAL_PRIVATE_IDS: JSON.stringify([id]), MANUAL_PRIVATE_APPROVALS_JSON: JSON.stringify({ version: 1, approvals: [{ workflowSha, targetSha, runId: "123", attempt: "1", executionIds: [id], protocol: MANUAL_PRIVATE_PROTOCOL, launchSha256: "c".repeat(64) }] }), GITHUB_SHA: workflowSha, TARGET_SHA: targetSha, GITHUB_RUN_ID: "123", GITHUB_RUN_ATTEMPT: "1", RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: "private-bucket", PRIVATE_REPORT_ROOT: root, PRIVATE_REPORT_SOURCE: source, AWS_ACCESS_KEY_ID: "synthetic-access-123456", AWS_SECRET_ACCESS_KEY: "synthetic-secret-123456", AWS_SESSION_TOKEN: "synthetic-session-123456" }); + const key = privateArtifactKey("manual-report", "123"), archive = transport.objects.get(key)!; + expect(transport.objects.size).toBe(2); expect(archive).toBeInstanceOf(Buffer); + const receipt = JSON.parse(transport.objects.get(`${key}.upload-receipt.json`)!.toString()); + expect(receipt.transfer).toMatchObject({ status: "complete", sha256: createHash("sha256").update(archive).digest("hex"), versionId: "synthetic-version-1" }); + expect(receipt.evidenceCompleteness.doesNotAttestFeatureOutcome).toBe(true); + const archivePath = path.join(root, "retained.zip"), restored = path.join(root, "restored"); await writeFile(archivePath, archive); await archivePrivateEvidence("unpack", archivePath, restored); + const inventory = JSON.parse(await readFile(path.join(restored, "result-evidence-inventory.json"), "utf8")); + for (const name of ["normalized/assets/InterVariable.woff2", "html/trace/playwright-logo.svg", "html/trace/manifest.webmanifest"]) { + const actual = await readFile(path.join(restored, "html-report", name)); + expect(actual.equals(await readFile(canonicalGeneratedReportAsset(name)!))).toBe(true); + expect(inventory.files).toContainEqual({ path: name, bytes: actual.length, sha256: createHash("sha256").update(actual).digest("hex") }); + } + expect(inventory.files.some((row: { path: string }) => row.path.endsWith(".ttf"))).toBe(true); + expect(await readFile(path.join(restored, "html-report/normalized/index.html"), "utf8")).toBe(dashboard); +}, 60_000); diff --git a/tests/runner-e2e/manual-private-report-read-order.test.ts b/tests/runner-e2e/manual-private-report-read-order.test.ts new file mode 100644 index 00000000000..e07722d0b06 --- /dev/null +++ b/tests/runner-e2e/manual-private-report-read-order.test.ts @@ -0,0 +1,45 @@ +import { createHash } from "node:crypto"; +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, realpath, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +const state = vi.hoisted(() => ({ target: "", reads: [] as string[], oversized: false })); +vi.mock("node:fs/promises", async original => { + const actual = await original(); + return { ...actual, + readFile: async (...args: Parameters) => { + if (String(args[0]) === state.target) { state.reads.push(state.target); throw new Error("Forbidden body read before admission"); } + return actual.readFile(...args); + }, + lstat: async (...args: Parameters) => { + const result = await actual.lstat(...args); + return state.oversized && String(args[0]) === state.target ? new Proxy(result, { get(object, key) { return key === "size" ? 256 * 1024 * 1024 + 1 : Reflect.get(object, key); } }) : result; + }, + }; +}); +import { validateSealedInventory } from "./manual-private-transfer.js"; +import { sealManualResultEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +const roots: string[] = []; +afterEach(async () => { state.target = ""; state.reads = []; state.oversized = false; await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +it.each(["unknown", "oversized-canonical", "oversized-standard"])("rejects %s before materializing its body", async kind => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "synthetic-report-read-order-"))); roots.push(root); + const source = path.join(root, "source"), name = kind === "unknown" ? "unknown.bin" : kind === "oversized-canonical" ? "normalized/assets/InterVariable.woff2" : "report.log"; + state.target = path.join(source, name); state.oversized = kind !== "unknown"; + await mkdir(path.dirname(state.target), { recursive: true }); await writeFile(state.target, "synthetic small bytes; size only mocked"); + const context = { root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff; + await expect(sealManualResultEvidence(context, [], source, "generated-report")).rejects.toThrow(kind === "unknown" ? "Unsupported packaged evidence carrier" : "Complete packaged evidence exceeds allowance"); + expect(state.reads).toEqual([]); +}); + +it.each(["unknown", "oversized-canonical", "oversized-standard"])("validator rejects %s before materializing its body", async kind => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "synthetic-validator-read-order-"))); roots.push(root); + const name = kind === "unknown" ? "unknown.bin" : kind === "oversized-canonical" ? "normalized/assets/InterVariable.woff2" : "report.log"; + state.target = path.join(root, name); state.oversized = kind !== "unknown"; + const data = Buffer.from("synthetic small bytes; size only mocked"), size = state.oversized ? 256 * 1024 * 1024 + 1 : data.length; + await mkdir(path.dirname(state.target), { recursive: true }); await writeFile(state.target, data); + const files = [{ path: name, bytes: size, sha256: createHash("sha256").update(data).digest("hex") }]; + await writeFile(path.join(root, "inventory.json"), JSON.stringify({ protocol: MANUAL_PRIVATE_PROTOCOL, treeKind: "generated-report", complete: true, sourceBytes: size, files, inventorySha256: createHash("sha256").update(JSON.stringify(files)).digest("hex") })); + await expect(validateSealedInventory(root, "inventory.json", undefined, "generated-report")).rejects.toThrow(kind === "unknown" ? "Invalid original evidence inventory row" : "Original evidence byte accounting changed"); + expect(state.reads).toEqual([]); +}); diff --git a/tests/runner-e2e/manual-private-report.test.ts b/tests/runner-e2e/manual-private-report.test.ts new file mode 100644 index 00000000000..f09ca69fc0b --- /dev/null +++ b/tests/runner-e2e/manual-private-report.test.ts @@ -0,0 +1,34 @@ +import { afterEach, expect, it, vi } from "vitest"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +vi.mock("./private-history.js",async original=>({...await original(),retainPrivateTransfer:vi.fn().mockResolvedValue({private:true})})); +import { retainPrivateTransfer } from "./private-history.js"; +import { stageDashboardBrandAssets } from "./report-assets.js"; +import { uploadReport } from "./manual-private-transfer.js"; +const roots:string[]=[]; +afterEach(async()=>{vi.clearAllMocks();await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); +async function fixture(){ + const root=await mkdtemp(path.join(os.tmpdir(),"manual-report-scan-test-"));roots.push(root);const source=path.join(root,"report");await mkdir(source); + const workflowSha="a".repeat(40),targetSha="b".repeat(40),id=MANUAL_PRIVATE_EXECUTIONS[0]!; + await writeFile(path.join(root,"manual-report-input-completeness.json"),JSON.stringify({meaning:"available-files-and-transfer-only",doesNotAttestFeatureOutcome:true,cells:[{executionId:id,availableFileInventoryComplete:true,requiredPassEvidenceComplete:null,missingRequiredEvidence:[]}]})); + const env={MANUAL_PRIVATE_IDS:JSON.stringify([id]),MANUAL_PRIVATE_APPROVALS_JSON:JSON.stringify({version:1,approvals:[{workflowSha,targetSha,runId:"123",attempt:"1",executionIds:[id],protocol:MANUAL_PRIVATE_PROTOCOL,launchSha256:"c".repeat(64)}]}),GITHUB_SHA:workflowSha,TARGET_SHA:targetSha,GITHUB_RUN_ID:"123",GITHUB_RUN_ATTEMPT:"1",RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET:"private-bucket",PRIVATE_REPORT_ROOT:root,PRIVATE_REPORT_SOURCE:source,AWS_ACCESS_KEY_ID:"synthetic-report-access-123456",AWS_SECRET_ACCESS_KEY:"synthetic-report-secret-123456",AWS_SESSION_TOKEN:"synthetic-report-session-123456"}; + return{root,source,env}; +} +it.each(["root","encoded","shape","unsupported"])("rejects fresh %s report output before any archive upload",async kind=>{ + const f=await fixture();const value=kind==="root"?f.env.AWS_SESSION_TOKEN:kind==="encoded"?Buffer.from(f.env.AWS_SECRET_ACCESS_KEY).toString("base64"):kind==="shape"?"sk-proj-synthetic123456789012345":"opaque"; + await writeFile(path.join(f.source,kind==="unsupported"?"new.db":"report.log"),value); + await expect(uploadReport(f.env)).rejects.toThrow();expect(retainPrivateTransfer).not.toHaveBeenCalled(); +}); +it("scans and seals genuine generated report bytes before transfer",async()=>{ + const f=await fixture();await writeFile(path.join(f.source,"report.log"),"The original feature failed; raw grade unchanged.\n");await uploadReport(f.env); + expect(retainPrivateTransfer).toHaveBeenCalledOnce();const options=vi.mocked(retainPrivateTransfer).mock.calls[0]![0]; + expect(options.directory).not.toBe(f.source);expect(options.evidenceCompleteness).toMatchObject({doesNotAttestFeatureOutcome:true}); +}); + +it("retains the actual brand asset producer tree",async()=>{ + const f=await fixture();await mkdir(path.join(f.source,"normalized"));await stageDashboardBrandAssets(path.join(f.source,"normalized")); + await mkdir(path.join(f.source,"html"));await writeFile(path.join(f.source,"html/index.html"),"Synthetic report metadata"); + await uploadReport(f.env);expect(retainPrivateTransfer).toHaveBeenCalledOnce(); +}); diff --git a/tests/runner-e2e/manual-private-retention.test.ts b/tests/runner-e2e/manual-private-retention.test.ts new file mode 100644 index 00000000000..58812848cbd --- /dev/null +++ b/tests/runner-e2e/manual-private-retention.test.ts @@ -0,0 +1,59 @@ +import { expect, it } from "vitest"; +import { authorizeManualPrivateEvidence, MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL, manualCellKind, manualPrivateSessionPolicy, requiresPrivateManualEvidence } from "./manual-private-retention.js"; +const workflowSha = "a".repeat(40), targetSha = "b".repeat(40); +const identity = { workflowSha, targetSha, runId: "123", attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] }; +const approval = () => ({ version: 1, approvals: [{ workflowSha, targetSha, runId: "123", attempt: "1", executionIds: identity.executionIds, protocol: MANUAL_PRIVATE_PROTOCOL, launchSha256: "c".repeat(64) }] }); +it("requires exact protected workflow, target and selected IDs for the first supported manual scope", () => { + expect(authorizeManualPrivateEvidence(identity, JSON.stringify(approval()))).toMatchObject({ private: true, protocol: MANUAL_PRIVATE_PROTOCOL }); +}); +it.each(["chat-features.runner-opencode.local.activity.agent-feedback", "chat-features.runner-codex.local.unknown", "chat-features"])("classification of %s cannot be downgraded by a target flag", id => expect(requiresPrivateManualEvidence(id)).toBe(true)); +it.each(["workflow", "target", "ids", "protocol", "extra", "duplicate", "empty", "rerun", "wrong-run"])("rejects %s approval mismatch without granting a role", kind => { + const p = approval(); const i = { ...identity }; + if (kind === "workflow") p.approvals[0]!.workflowSha = "c".repeat(40); + if (kind === "target") p.approvals[0]!.targetSha = "c".repeat(40); + if (kind === "ids") p.approvals[0]!.executionIds = [MANUAL_PRIVATE_EXECUTIONS[1]!]; + if (kind === "protocol") p.approvals[0]!.protocol = "unknown"; + if (kind === "extra") Object.assign(p.approvals[0]!, { private: true }); + if (kind === "duplicate") p.approvals.push(p.approvals[0]!); + if (kind === "empty") p.approvals = []; + if (kind === "rerun") i.attempt = "2"; + if (kind === "wrong-run") p.approvals[0]!.runId = "124"; + expect(() => authorizeManualPrivateEvidence(i, JSON.stringify(p))).toThrow(); +}); +it("unknown and mixed manual cases cannot acquire private role authority", () => { + expect(() => authorizeManualPrivateEvidence({ ...identity, executionIds: ["chat-features.runner-opencode.local.access.invite-teammate"] }, JSON.stringify(approval()))).toThrow(); + expect(() => authorizeManualPrivateEvidence({ ...identity, executionIds: [...identity.executionIds, "runner.native.local.other"] }, JSON.stringify(approval()))).toThrow(); +}); +it("different cells have distinct immutable private object identities", () => { + expect(manualCellKind(MANUAL_PRIVATE_EXECUTIONS[0]!)).not.toBe(manualCellKind(MANUAL_PRIVATE_EXECUTIONS[1]!)); + expect(() => manualCellKind("../escape")).toThrow(); +}); +it("vetted paid transfer session is put-only and the trusted report reads only selected current cells", () => { + const paid = manualPrivateSessionPolicy({ bucket: "private-evidence", runId: "123", executionIds: identity.executionIds, mode: "cell-write" }); + expect(paid.Statement.flatMap(s => s.Action)).not.toContain("s3:GetObject"); + const report = manualPrivateSessionPolicy({ bucket: "private-evidence", runId: "123", executionIds: identity.executionIds, mode: "cell-read" }); + const reads = report.Statement.find(s => s.Action.includes("s3:GetObject"))!; + expect(reads.Resource).toEqual([`arn:aws:s3:::private-evidence/runner-e2e-private/bounded-v1/campaigns/gha-123-1/${manualCellKind(identity.executionIds[0]!)}.zip`, `arn:aws:s3:::private-evidence/runner-e2e-private/bounded-v1/campaigns/gha-123-1/${manualCellKind(identity.executionIds[0]!)}.zip.upload-receipt.json`]); + expect(JSON.stringify(report)).not.toContain("gha-124"); +}); +it("the supported private archive kind is collision-free and rejects arbitrary manual paths", async () => { + const { privateArtifactKey } = await import("./private-history.js"); + expect(privateArtifactKey(manualCellKind(identity.executionIds[0]!), "123")).toContain("/manual-cell-"); + expect(privateArtifactKey("manual-report", "123")).toMatch(/manual-report\.zip$/); + expect(() => privateArtifactKey("manual-cell-../escape", "123")).toThrow(); +}); + +it("keeps the full-eight report inline session policy within STS's 2048-character limit",()=>{ + const policy=manualPrivateSessionPolicy({bucket:"b".repeat(63),runId:"12345678901234567890",executionIds:[...MANUAL_PRIVATE_EXECUTIONS],mode:"report-write"}); + expect(JSON.stringify(policy).length).toBeLessThanOrEqual(2048); +}); + +it("all eight exact read/write cell scopes fit STS and never broaden to other cells or wildcards",()=>{ + for(const id of MANUAL_PRIVATE_EXECUTIONS)for(const mode of ["cell-read","cell-write"]as const){ + const p=manualPrivateSessionPolicy({bucket:"b".repeat(63),runId:"12345678901234567890",executionIds:[id],mode}); + expect(JSON.stringify(p).length).toBeLessThanOrEqual(2048);expect(JSON.stringify(p)).not.toContain("*"); + const objects=p.Statement.filter(s=>Array.isArray(s.Resource)).flatMap(s=>s.Resource);expect(objects).toHaveLength(2); + expect(objects.every(r=>r.includes(manualCellKind(id)))).toBe(true); + } + expect(()=>manualPrivateSessionPolicy({bucket:"private-bucket",runId:"123",executionIds:[...MANUAL_PRIVATE_EXECUTIONS],mode:"cell-read"})).toThrow(); +}); diff --git a/tests/runner-e2e/manual-private-retention.ts b/tests/runner-e2e/manual-private-retention.ts new file mode 100644 index 00000000000..82af9ed1ef2 --- /dev/null +++ b/tests/runner-e2e/manual-private-retention.ts @@ -0,0 +1,67 @@ +import { createHash } from "node:crypto"; + +export const MANUAL_PRIVATE_PROTOCOL = "paperclip.manual-private-evidence/v1"; +export const MANUAL_PRIVATE_EXECUTIONS = Object.freeze([ + "activity.agent-feedback", "activity.work-timeline", "companies.import-preview", + "navigation-preferences.dismiss-announcement", "tasks.completion-with-gate", + "tasks.private-task", "secrets.personal-secret", "agents.choose-primary", +].map(id => `chat-features.runner-opencode.local.${id}`)); +const sha = /^[0-9a-f]{40}$/; +const execution = /^[A-Za-z0-9][A-Za-z0-9._-]{0,249}$/; +function keys(value: unknown, expected: string[]): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) + && Object.keys(value).sort().join() === [...expected].sort().join(); +} +export function canonicalManualIds(value: unknown): string[] { + if (!Array.isArray(value) || !value.length || value.length > 32 + || value.some(id => typeof id !== "string" || !execution.test(id)) || new Set(value).size !== value.length) + throw new Error("Invalid exact manual evidence selection"); + return [...value].sort(); +} +export function requiresPrivateManualEvidence(id: string) { + return id === "chat-features" || id.startsWith("chat-features."); +} +export interface ManualRetentionIdentity { + workflowSha: string; targetSha: string; runId: string; attempt: string; executionIds: string[]; +} +export function authorizeManualPrivateEvidence(identity: ManualRetentionIdentity, approvalJson: string) { + if (!sha.test(identity.workflowSha) || !sha.test(identity.targetSha) || !/^[1-9][0-9]{0,19}$/.test(identity.runId) || identity.attempt !== "1") + throw new Error("Invalid manual private evidence identity"); + const ids = canonicalManualIds(identity.executionIds); + if (ids.some(id => !MANUAL_PRIVATE_EXECUTIONS.includes(id))) throw new Error("Unsupported private manual evidence selection"); + if (Buffer.byteLength(approvalJson) > 65536) throw new Error("Invalid protected manual evidence approval"); + let policy: unknown; + try { policy = JSON.parse(approvalJson); } catch { throw new Error("Invalid protected manual evidence approval"); } + if (!keys(policy, ["version", "approvals"]) || policy.version !== 1 || !Array.isArray(policy.approvals) || policy.approvals.length > 32) + throw new Error("Invalid protected manual evidence approval"); + const matches = policy.approvals.filter(row => { + if (!keys(row, ["workflowSha", "targetSha", "runId", "attempt", "executionIds", "protocol", "launchSha256"]) || !sha.test(String(row.workflowSha)) + || !sha.test(String(row.targetSha)) || !/^[1-9][0-9]{0,19}$/.test(String(row.runId)) || row.attempt !== "1" || row.protocol !== MANUAL_PRIVATE_PROTOCOL || !/^[0-9a-f]{64}$/.test(String(row.launchSha256))) + throw new Error("Invalid protected manual evidence approval"); + const selected = canonicalManualIds(row.executionIds); + return row.workflowSha === identity.workflowSha && row.targetSha === identity.targetSha && row.runId === identity.runId && row.attempt === identity.attempt && selected.join() === ids.join(); + }); + if (matches.length !== 1) throw new Error("Exact protected manual evidence approval missing or ambiguous"); + return Object.freeze({ ...identity, executionIds: Object.freeze(ids), protocol: MANUAL_PRIVATE_PROTOCOL, + launchSha256: String(matches[0]!.launchSha256), approvalSha256: createHash("sha256").update(approvalJson).digest("hex"), private: true as const }); +} +export function manualCellKind(id: string) { + if (!execution.test(id)) throw new Error("Invalid manual evidence execution identity"); + return `manual-cell-${createHash("sha256").update(id).digest("hex")}`; +} +export function manualPrivateSessionPolicy(input: { bucket: string; runId: string; executionIds: string[]; mode: "cell-write" | "cell-read" | "report-write" }) { + if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket) || !/^[1-9][0-9]{0,19}$/.test(input.runId)) + throw new Error("Invalid manual private storage identity"); + const ids = canonicalManualIds(input.executionIds); + if (ids.some(id => !MANUAL_PRIVATE_EXECUTIONS.includes(id)) || input.mode !== "report-write" && ids.length !== 1) + throw new Error("Private archive sessions require one exact cell or the report writer"); + const prefix = `arn:aws:s3:::${input.bucket}/runner-e2e-private/bounded-v1/campaigns/gha-${input.runId}-1/`; + const cells = ids.map(id => `${prefix}${manualCellKind(id)}.zip`); + const policy = { Version: "2012-10-17", Statement: [ + { Effect: "Allow", Action: ["s3:GetBucketVersioning", "s3:GetLifecycleConfiguration", "s3:GetBucketObjectLockConfiguration", "s3:GetReplicationConfiguration", "s3:GetBucketPublicAccessBlock", "s3:GetBucketOwnershipControls", "s3:GetBucketPolicy"], Resource: `arn:aws:s3:::${input.bucket}` }, + ...(input.mode === "cell-read" ? [{ Effect: "Allow", Action: ["s3:GetObject"], Resource: cells.flatMap(key => [key, `${key}.upload-receipt.json`]) }] : []), + ...(input.mode === "cell-read" ? [] : [{ Effect: "Allow", Action: ["s3:PutObject"], Resource: (input.mode === "report-write" ? [`${prefix}manual-report.zip`] : cells).flatMap(key => [key, `${key}.upload-receipt.json`]) }]), + ] }; + if (JSON.stringify(policy).length > 2048) throw new Error("Private archive session policy exceeds the STS limit"); + return policy; +} diff --git a/tests/runner-e2e/manual-private-transfer.test.ts b/tests/runner-e2e/manual-private-transfer.test.ts new file mode 100644 index 00000000000..c163340bd9e --- /dev/null +++ b/tests/runner-e2e/manual-private-transfer.test.ts @@ -0,0 +1,49 @@ +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { afterEach, expect, it } from "vitest"; +import { mkdtemp, mkdir, readFile, rm, writeFile, copyFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { createHash } from "node:crypto"; +import { validateSealedInventory } from "./manual-private-transfer.js"; +import { sealManualResultEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +const roots:string[]=[]; +afterEach(async()=>{await Promise.all(roots.splice(0).map(root=>rm(root,{recursive:true,force:true})));}); +async function fixture(){const root=await mkdtemp(path.join(os.tmpdir(),"manual-transfer-test-"));roots.push(root);await mkdir(path.join(root,"results"));const context={root,protocol:MANUAL_PRIVATE_PROTOCOL,private:true,workflowSha:"a".repeat(40),targetSha:"b".repeat(40),runId:"123",attempt:"1",executionIds:["chat-features.runner-opencode.local.activity.agent-feedback"]} as ManualPrivateHandoff;return{root,context};} +it("seals genuine final result bytes and rejects an unscanned staged addition before transfer",async()=>{ + const f=await fixture();await writeFile(path.join(f.root,"results","campaign.json"),'{"cleanup":"passed"}'); + await sealManualResultEvidence(f.context,[]);const dir=path.join(f.root,"sealed-results","html-report"); + expect((await validateSealedInventory(dir,"../result-evidence-inventory.json")).complete).toBe(true); + await writeFile(path.join(dir,"staged.db"),"unscanned");await expect(validateSealedInventory(dir,"../result-evidence-inventory.json")).rejects.toThrow(/Unscanned/); +}); +it("rejects modification of final cleanup/result bytes after the scan",async()=>{ + const f=await fixture();await writeFile(path.join(f.root,"results","campaign.json"),'{}');await sealManualResultEvidence(f.context,[]); + const dir=path.join(f.root,"sealed-results","html-report");await writeFile(path.join(dir,"campaign.json"),'{"later":true}');await expect(validateSealedInventory(dir,"../result-evidence-inventory.json")).rejects.toThrow(/changed/); +}); +it("an abnormal fallback cannot retain credential-bearing final data or unsupported staged bytes",async()=>{ + const f=await fixture();await writeFile(path.join(f.root,"results","result.json"),'{"originalResult":"synthetic-final-credential-123456"}'); + await expect(sealManualResultEvidence(f.context,["synthetic-final-credential-123456"])).rejects.toThrow(/contamination/); + await writeFile(path.join(f.root,"results","result.json"),'{}');await writeFile(path.join(f.root,"results","unknown.bin"),"opaque"); + await expect(sealManualResultEvidence(f.context,[])).rejects.toThrow(/Unsupported/); +}); +it("rejects a forged inventory that lists an unscannable original carrier",async()=>{ + const f=await fixture(),dir=path.join(f.root,"results");await mkdir(path.join(dir,"snapshots"));await writeFile(path.join(dir,"snapshots","unknown.bin"),"secret"); + const files=[{path:"snapshots/unknown.bin",bytes:6,sha256:createHash("sha256").update("secret").digest("hex")}]; + await writeFile(path.join(dir,"inventory.json"),JSON.stringify({protocol:MANUAL_PRIVATE_PROTOCOL,complete:true,executionId:"id",sourceBytes:6,files,inventorySha256:createHash("sha256").update(JSON.stringify(files)).digest("hex")})); + await expect(validateSealedInventory(dir,"inventory.json","id")).rejects.toThrow(/row/); +}); + +it.each([false, true])("actual prepare checks new report dependency before credentials (tampered=%s)", async tampered => { + const root=await mkdtemp(path.join(os.tmpdir(),"synthetic-prepare-dependency-"));roots.push(root); + const target=path.join(root,"target"),modules=path.join(target,"tests/runner-e2e");await mkdir(modules,{recursive:true}); + const trusted=path.resolve(import.meta.dirname,"../.."),launch=Buffer.from("synthetic approved launcher; never executed");await writeFile(path.join(modules,"launch.ts"),launch); + for(const name of ["manual-private-handoff.ts","manual-private-retention.ts","manual-private-report-assets.ts"])await copyFile(path.join(import.meta.dirname,name),path.join(modules,name)); + if(tampered)await writeFile(path.join(modules,"manual-private-report-assets.ts"),"synthetic incompatible dependency; never executed"); + const workflowSha="a".repeat(40),targetSha="b".repeat(40),id="chat-features.runner-opencode.local.activity.agent-feedback",output=path.join(root,"outputs.txt"); + const require=createRequire(import.meta.url),tsx=require.resolve("tsx",{paths:[path.join(trusted,"server")]}); + const result=spawnSync(process.execPath,["--import",tsx,path.join(import.meta.dirname,"manual-private-transfer.ts"),"prepare"],{encoding:"utf8",timeout:10000,env:{PATH:process.env.PATH,NODE_OPTIONS:"",NODE_PATH:"",SELECTED_EXECUTION_ID:id,MANUAL_PRIVATE_IDS:JSON.stringify([id]),MANUAL_PRIVATE_APPROVALS_JSON:JSON.stringify({version:1,approvals:[{workflowSha,targetSha,runId:"123",attempt:"1",executionIds:[id],protocol:MANUAL_PRIVATE_PROTOCOL,launchSha256:createHash("sha256").update(launch).digest("hex")}]}),GITHUB_SHA:workflowSha,TARGET_SHA:targetSha,GITHUB_RUN_ID:"123",GITHUB_RUN_ATTEMPT:"1",RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET:"private-bucket",TARGET_ROOT:target,RUNNER_TEMP:root,GITHUB_OUTPUT:output}}); + expect(result.error).toBeUndefined();expect(result.status).toBe(tampered?1:0); + if(tampered){expect(result.stderr).toContain("Restricted manual evidence operation failed");await expect(readFile(output)).rejects.toMatchObject({code:"ENOENT"});} + else expect(await readFile(output,"utf8")).toContain("session_policy="); +}); diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts new file mode 100644 index 00000000000..0dd8271c69b --- /dev/null +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -0,0 +1,197 @@ +import { createHash } from "node:crypto"; +import { appendFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { canonicalGeneratedReportAsset, isTrustedGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { sealManualResultEvidence } from "./manual-private-handoff.js"; +import { normalizedSecrets } from "./redaction.js"; +import { CREDENTIAL_NAMES } from "./types.js"; +import { isStandardEvidenceFile } from "./evidence.js"; +import { BoundedHistoryPublisher } from "./history-budget.js"; +import { privateArtifactKey, retainPrivateTransfer, transferPrivateArtifact, verifyPrivateHistory, type PrivateEvidenceCompleteness } from "./private-history.js"; +import { authorizeManualPrivateEvidence, MANUAL_PRIVATE_PROTOCOL, manualCellKind, manualPrivateSessionPolicy } from "./manual-private-retention.js"; + +const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); +function request(env: NodeJS.ProcessEnv) { + const ids = JSON.parse(env.MANUAL_PRIVATE_IDS ?? "null") as string[]; + const approval = authorizeManualPrivateEvidence({ workflowSha: env.GITHUB_SHA ?? "", targetSha: env.TARGET_SHA ?? "", + runId: env.GITHUB_RUN_ID ?? "", attempt: env.GITHUB_RUN_ATTEMPT ?? "", executionIds: ids }, env.MANUAL_PRIVATE_APPROVALS_JSON ?? ""); + return { ids, approval, bucket: env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? "" }; +} +async function output(name: string, value: string) { + if (/[\r\n]/.test(value)) throw new Error("Invalid private evidence step output"); + await appendFile(process.env.GITHUB_OUTPUT!, `${name}=${value}\n`); +} +async function prepare(env: NodeJS.ProcessEnv) { + const { ids, approval, bucket } = request(env), id = env.SELECTED_EXECUTION_ID; + if (!id || !ids.includes(id)) throw new Error("Private evidence cell is not approved"); + const target = await realpath(env.TARGET_ROOT ?? ""), trusted = path.resolve(import.meta.dirname, "../.."); + if (target === trusted) throw new Error("Trusted transfer code must be separate from target checkout"); + const launch = await readFile(path.join(target, "tests/runner-e2e/launch.ts")); + if (hash(launch) !== approval.launchSha256) throw new Error("Target evidence launcher differs from protected approval"); + for (const file of ["manual-private-handoff.ts", "manual-private-retention.ts", "manual-private-report-assets.ts"]) { + if (!(await readFile(path.join(target, "tests/runner-e2e", file))).equals(await readFile(path.join(trusted, "tests/runner-e2e", file)))) + throw new Error("Target private evidence protocol differs from trusted implementation"); + } + const temporary = await realpath(env.RUNNER_TEMP ?? ""), root = await mkdtemp(path.join(temporary, "manual-private-evidence-")); + await writeFile(path.join(root, "identity.json"), JSON.stringify({ ...approval, executionIds: [id], root, status: "pending" }), { mode: 0o600, flag: "wx" }); + await output("root", root); await output("session_policy", JSON.stringify(manualPrivateSessionPolicy({ bucket, runId: approval.runId, executionIds: [id], mode: "cell-write" }))); +} +function requireSession(env: NodeJS.ProcessEnv) { + if (![env.AWS_ACCESS_KEY_ID, env.AWS_SECRET_ACCESS_KEY, env.AWS_SESSION_TOKEN].every(value => typeof value === "string" && value.length > 0)) + throw new Error("Explicit private archive session is required; no credential fallback"); +} +async function verify(env: NodeJS.ProcessEnv) { + requireSession(env); const { approval, bucket } = request(env), root = await realpath(env.PRIVATE_EVIDENCE_ROOT ?? ""); + const identity = JSON.parse(await readFile(path.join(root, "identity.json"), "utf8")); + if (identity.approvalSha256 !== approval.approvalSha256 || identity.targetSha !== approval.targetSha || identity.workflowSha !== approval.workflowSha + || identity.runId !== approval.runId || identity.attempt !== "1" || identity.root !== root) throw new Error("Private preflight identity changed"); + const publisher = new BoundedHistoryPublisher(path.join(root, "preflight-requests.json")); + await verifyPrivateHistory(publisher, bucket); + await writeFile(path.join(root, "preflight.json"), JSON.stringify({ ...identity, status: "verified" }), { mode: 0o600, flag: "wx" }); +} +export async function validateSealedInventory(directory: string, inventory: string, id?: string, treeKind: "standard" | "generated-report" = "standard") { + const receipt = JSON.parse(await readFile(path.join(directory, inventory), "utf8")); + if (receipt.protocol !== MANUAL_PRIVATE_PROTOCOL || id !== undefined && receipt.executionId !== id || receipt.complete !== true + || !Array.isArray(receipt.files) || !receipt.files.length || receipt.files.length > 4096 + || hash(JSON.stringify(receipt.files)) !== receipt.inventorySha256) throw new Error("Original evidence inventory is incomplete"); + if (id === undefined && receipt.treeKind !== treeKind) throw new Error("Private evidence tree kind differs from its inventory"); + const expected = new Set(inventory.startsWith("../") ? [] : [inventory]); let bytes = 0; + for (const row of receipt.files) { + if (!row || Object.keys(row).sort().join() !== "bytes,path,sha256" || typeof row.path !== "string" + || row.path.split("/").some((part: string) => !part || part === "." || part === "..") || row.path.includes("\\") + || expected.has(row.path) || !Number.isSafeInteger(row.bytes) || row.bytes < 0 || !/^[0-9a-f]{64}$/.test(row.sha256)) throw new Error("Invalid original evidence inventory row"); + const standard = isStandardEvidenceFile(id === undefined ? `html-report/${row.path}` : row.path); + const canonical = id === undefined && treeKind === "generated-report" ? canonicalGeneratedReportAsset(row.path) : undefined; + const staticReportOwner = id === undefined && treeKind === "generated-report" && (row.path.startsWith("html/trace/") || canonical !== undefined); + if (staticReportOwner ? !canonical : !standard) throw new Error("Invalid original evidence inventory row"); + if (row.bytes > 256 * 1024 * 1024 - bytes) throw new Error("Original evidence byte accounting changed"); + expected.add(row.path); bytes += row.bytes; + const file = path.join(directory, row.path), metadata = await lstat(file); + if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size !== row.bytes) throw new Error("Original evidence changed after its scan"); + const data = await readFile(file); + if (hash(data) !== row.sha256) throw new Error("Original evidence changed after its scan"); + if (staticReportOwner && !await isTrustedGeneratedReportAsset(row.path, data)) throw new Error("Invalid original evidence inventory row"); + } + if (bytes !== receipt.sourceBytes || bytes > 256 * 1024 * 1024) throw new Error("Original evidence byte accounting changed"); + const actual = new Set(); + async function visit(relative: string) { + for (const entry of await readdir(path.join(directory, relative), { withFileTypes: true })) { + const name = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isSymbolicLink() || !entry.isFile() && !entry.isDirectory() || name.split("/").length > 24) throw new Error("Invalid sealed evidence entry"); + if (entry.isDirectory()) { await visit(name); continue; } + if (!expected.has(name)) throw new Error("Unscanned file added to sealed evidence"); + actual.add(name); + } + } + await visit(""); + if (actual.size !== expected.size) throw new Error("Sealed evidence is incomplete"); + return receipt; +} +export async function validateSealedOriginals(root: string, id: string) { + const receipt = await validateSealedInventory(path.join(root, "originals", manualCellKind(id)), "original-evidence-inventory.json", id); + const missing = receipt.missingRequiredEvidence; + const allowed = new Set(["final-state.png", "result.json", "junit.xml", "html-report/index.html", "snapshots/fixtures.json", "snapshots/api-state.json", "blob-report/*.zip"]); + if (![true, false, null].includes(receipt.requiredEvidenceComplete) || !Array.isArray(missing) || new Set(missing).size !== missing.length + || missing.some((name: unknown) => typeof name !== "string" || !allowed.has(name)) + || (receipt.requiredEvidenceComplete === false) !== (missing.length > 0)) throw new Error("Required evidence completeness is unknown"); + return receipt; +} +function cellCompleteness(id: string, receipt: { requiredEvidenceComplete: boolean | null; missingRequiredEvidence: string[] }): PrivateEvidenceCompleteness { + return { meaning: "available-files-and-transfer-only", doesNotAttestFeatureOutcome: true, + cells: [{ executionId: id, availableFileInventoryComplete: true, requiredPassEvidenceComplete: receipt.requiredEvidenceComplete, missingRequiredEvidence: receipt.missingRequiredEvidence }] }; +} +async function uploadCell(env: NodeJS.ProcessEnv) { + requireSession(env); const { approval, bucket } = request(env), id = env.SELECTED_EXECUTION_ID!; + if (!approval.executionIds.includes(id)) throw new Error("Private evidence cell is not approved"); + const root = await realpath(env.PRIVATE_EVIDENCE_ROOT ?? ""), context = JSON.parse(await readFile(path.join(root, "preflight.json"), "utf8")); + if (context.status !== "verified" || context.approvalSha256 !== approval.approvalSha256 || context.targetSha !== approval.targetSha + || context.runId !== approval.runId || context.executionIds.join() !== id || context.root !== root) throw new Error("Private evidence preflight identity changed"); + if (await lstat(path.join(root, "contamination.json")).then(() => true, error => { if (error.code === "ENOENT") return false; throw error; })) + throw new Error("Contaminated private evidence cannot be uploaded"); + const originals = await validateSealedOriginals(root, id); + const evidenceCompleteness = cellCompleteness(id, originals); + const bundle = path.join(root, "sealed-bundle"); await mkdir(bundle, { mode: 0o700 }); + const results = path.join(root, "sealed-results"); + await validateSealedInventory(path.join(results, "html-report"), "../result-evidence-inventory.json"); + await mkdir(path.join(bundle, "originals"), { mode: 0o700 }); + await cp(path.join(root, "originals", manualCellKind(id)), path.join(bundle, "originals", manualCellKind(id)), { recursive: true, errorOnExist: true, force: false }); + await cp(results, path.join(bundle, "sealed-results"), { recursive: true, errorOnExist: true, force: false }); + await writeFile(path.join(bundle, "identity.json"), JSON.stringify({ ...context, evidenceCompleteness }), { mode: 0o600, flag: "wx" }); + const receipt = path.join(root, "cell-transfer.json"), publisher = new BoundedHistoryPublisher(receipt); + await retainPrivateTransfer({ operation: "upload", kind: manualCellKind(id), runId: approval.runId, directory: bundle, bucket, publisher, evidenceCompleteness }, receipt, + () => new BoundedHistoryPublisher(`${receipt}.retention.json`)); +} +async function reportSession(env: NodeJS.ProcessEnv) { + const { approval, bucket } = request(env); + await output("session_policy", JSON.stringify(manualPrivateSessionPolicy({ bucket, runId: approval.runId, executionIds: [...approval.executionIds], mode: "report-write" }))); + await output("cell_count", String(approval.executionIds.length)); + for (const [index, id] of approval.executionIds.entries()) { + await output(`read_policy_${index}`, JSON.stringify(manualPrivateSessionPolicy({ bucket, runId: approval.runId, executionIds: [id], mode: "cell-read" }))); + } +} +async function downloadCell(env: NodeJS.ProcessEnv) { + requireSession(env); const { approval, bucket } = request(env), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); + const index = Number(env.SELECTED_CELL_INDEX); + if (!/^[0-7]$/.test(env.SELECTED_CELL_INDEX ?? "") || !approval.executionIds[index]) throw new Error("Invalid exact private cell index"); + const selected = path.join(root, "selected-runner-e2e"); await mkdir(selected, { recursive: true, mode: 0o700 }); + const id = approval.executionIds[index]!; + { + const directory = path.join(root, manualCellKind(id)), publisher = new BoundedHistoryPublisher(path.join(root, `${manualCellKind(id)}-download.json`)); + const receipt = await publisher.downloadJson<{ evidenceCompleteness?: PrivateEvidenceCompleteness; transfer?: { status?: string; sha256?: string } }>(bucket, + `${privateArtifactKey(manualCellKind(id), approval.runId)}.upload-receipt.json`, path.join(root, `${manualCellKind(id)}-receipt.json`)); + if (receipt?.transfer?.status !== "complete" || !/^[0-9a-f]{64}$/.test(receipt.transfer.sha256 ?? "")) throw new Error("Private cell transfer receipt is unavailable"); + const downloaded = await transferPrivateArtifact({ operation: "download", kind: manualCellKind(id), runId: approval.runId, directory, bucket, publisher }); + if (downloaded.sha256 !== receipt.transfer.sha256) throw new Error("Private cell archive differs from retained transfer receipt"); + const identity = JSON.parse(await readFile(path.join(directory, "identity.json"), "utf8")); + if (identity.targetSha !== approval.targetSha || identity.workflowSha !== approval.workflowSha || identity.runId !== approval.runId + || identity.executionIds.join() !== id || identity.protocol !== MANUAL_PRIVATE_PROTOCOL) throw new Error("Private cell archive has a foreign identity"); + const originals = await validateSealedOriginals(directory, id), summary = cellCompleteness(id, originals); + if (hash(JSON.stringify(identity.evidenceCompleteness)) !== hash(JSON.stringify(summary)) + || hash(JSON.stringify(receipt.evidenceCompleteness)) !== hash(JSON.stringify(summary))) throw new Error("Private evidence completeness receipt differs from archive"); + await validateSealedInventory(path.join(directory, "sealed-results", "html-report"), "../result-evidence-inventory.json"); + await cp(path.join(directory, "sealed-results", "html-report"), path.join(selected, manualCellKind(id)), { recursive: true, errorOnExist: true, force: false }); + await writeFile(path.join(root, `manual-cell-completeness-${index}.json`), JSON.stringify(summary), { mode: 0o600, flag: "wx" }); + } +} +async function verifyDownloads(env: NodeJS.ProcessEnv) { + const { approval } = request(env), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); + const cells: PrivateEvidenceCompleteness["cells"] = []; + for (const [index, id] of approval.executionIds.entries()) { + const summary = JSON.parse(await readFile(path.join(root, `manual-cell-completeness-${index}.json`), "utf8")) as PrivateEvidenceCompleteness; + if (summary.meaning !== "available-files-and-transfer-only" || summary.doesNotAttestFeatureOutcome !== true + || summary.cells.length !== 1 || summary.cells[0]!.executionId !== id) throw new Error("Private report lacks an exact cell transfer"); + cells.push(...summary.cells); + } + await writeFile(path.join(root, "manual-report-input-completeness.json"), JSON.stringify({ meaning: "available-files-and-transfer-only", doesNotAttestFeatureOutcome: true, cells }), { mode: 0o600, flag: "wx" }); +} +export async function uploadReport(env: NodeJS.ProcessEnv) { + requireSession(env); const { approval, bucket } = request(env), source = await realpath(env.PRIVATE_REPORT_SOURCE ?? ""), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); + if (!source.startsWith(`${root}${path.sep}`)) throw new Error("Private report escaped owned staging"); + const evidenceCompleteness = JSON.parse(await readFile(path.join(root, "manual-report-input-completeness.json"), "utf8")) as PrivateEvidenceCompleteness; + if (evidenceCompleteness.meaning !== "available-files-and-transfer-only" || evidenceCompleteness.doesNotAttestFeatureOutcome !== true + || evidenceCompleteness.cells.map(cell => cell.executionId).sort().join() !== [...approval.executionIds].sort().join()) throw new Error("Private report lacks exact cell completeness receipts"); + const secrets = normalizedSecrets([...CREDENTIAL_NAMES.map(name => env[name]), env.AWS_ACCESS_KEY_ID, env.AWS_SECRET_ACCESS_KEY, env.AWS_SESSION_TOKEN]); + await sealManualResultEvidence({ root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: approval.workflowSha, + targetSha: approval.targetSha, runId: approval.runId, attempt: approval.attempt, executionIds: [...approval.executionIds] }, secrets, source, "generated-report"); + const sealed = path.join(root, "sealed-results"); + await validateSealedInventory(path.join(sealed, "html-report"), "../result-evidence-inventory.json", undefined, "generated-report"); + const receipt = path.join(root, "report-transfer.json"), publisher = new BoundedHistoryPublisher(receipt); + await retainPrivateTransfer({ operation: "upload", kind: "manual-report", runId: approval.runId, directory: sealed, bucket, publisher, evidenceCompleteness }, receipt, + () => new BoundedHistoryPublisher(`${receipt}.retention.json`)); +} +async function main() { + const action = process.argv[2]; + if (process.argv.length !== 3) throw new Error("Invalid private evidence command"); + if (action === "prepare") await prepare(process.env); + else if (action === "verify") await verify(process.env); + else if (action === "upload-cell") await uploadCell(process.env); + else if (action === "report-session") await reportSession(process.env); + else if (action === "download-cell") await downloadCell(process.env); + else if (action === "verify-downloads") await verifyDownloads(process.env); + else if (action === "upload-report") await uploadReport(process.env); + else throw new Error("Invalid private evidence command"); +} +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + main().catch(() => { process.stderr.write("Restricted manual evidence operation failed; no public fallback\n"); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/manual-private-workflow.test.ts b/tests/runner-e2e/manual-private-workflow.test.ts new file mode 100644 index 00000000000..dbb92264970 --- /dev/null +++ b/tests/runner-e2e/manual-private-workflow.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vitest"; +import { readFile, mkdtemp, rm } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { spawnSync } from "node:child_process"; +import YAML from "yaml"; +const source=await readFile(new URL("../../.github/workflows/runner-full-stack-e2e.yml",import.meta.url),"utf8"); +const jobs=YAML.parse(source).jobs; +const step=(job:string,name:string)=>jobs[job].steps.find((row:{name?:string})=>row.name===name); +function condition(expression:string, state:unknown){return Function("always","needs","steps",`return (${expression})`)(()=>true,(state as {needs:unknown}).needs,(state as {steps:unknown}).steps);} +it("never uploads staged manual cell or merged evidence to public Actions on private failure",()=>{ + const state={needs:{catalog:{outputs:{operations_cadence:""}},authorize:{outputs:{evidence_retention:"private"}}},steps:{manual_cell_route:{outputs:{route:"private"}},manual_private_verified:{outcome:"failure"}}}; + for(const [job,name]of[["test","Upload nonprivate packaged cell evidence"],["report","Upload nonprivate merged report"]]){ + expect(condition(step(job!,name!).if,state)).toBe(false); + const publicState={...state,needs:{...state.needs,authorize:{outputs:{evidence_retention:"actions"}}},steps:{...state.steps,manual_cell_route:{outputs:{route:"actions"}}}}; + expect(condition(step(job!,name!).if,publicState)).toBe(true); + } + const noGuard={...state,needs:{...state.needs,authorize:{outputs:{evidence_retention:"actions"}}},steps:{...state.steps,manual_cell_route:{outputs:{}}}}; + expect(condition(step("test","Upload nonprivate packaged cell evidence").if,noGuard)).toBe(false); + expect(jobs.publish_history.if).toContain("evidence_retention == 'actions'"); +}); +it("keeps archive authority in explicit protected jobs and the workflow-pinned transfer runtime",()=>{ + expect(jobs.test.permissions["id-token"]).toBeUndefined(); + expect(jobs.test_manual_private.environment.name).toBe("runner-e2e-private-paid"); + expect(jobs.test_manual_private.if).toContain("evidence_retention == 'private'"); + const steps=jobs.test_manual_private.steps; + const checkout=steps.find((row:any)=>row.with?.path===".manual-private-trusted");expect(checkout.with.ref).toBe("${{ github.sha }}"); + const runtime=step("test_manual_private","Prepare workflow-pinned private transfer runtime before archive credentials");expect(runtime.run).toContain("--ignore-scripts --store-dir"); + expect(steps.indexOf(runtime)).toBeLessThan(steps.indexOf(step("test_manual_private","Obtain exact-key manual archive session"))); + const verified=steps.indexOf(step("test_manual_private","Verify restricted complete evidence storage before model or fixture secrets")); + expect(verified).toBeLessThan(steps.findIndex((row:any)=>row.run==="pnpm install --frozen-lockfile --ignore-scripts")); + for(const name of["Bind protected original run and compatible target evidence protocol","Verify restricted complete evidence storage before model or fixture secrets","Retain complete approved manual originals with no public fallback"]){ + const row=step("test_manual_private",name);expect(row.workingDirectory??row["working-directory"]).toBe(".manual-private-trusted");expect(row.run).toContain('$GITHUB_WORKSPACE/.manual-private-trusted/cli/node_modules/tsx/dist/cli.mjs');expect(row.env.NODE_OPTIONS).toBe("");expect(row.env.NODE_PATH).toBe(""); + } + expect(step("test_manual_private","Run paid cell").env.AWS_ACCESS_KEY_ID).toBeUndefined(); + expect(step("test_manual_private","Run paid cell").if).toContain("manual_private_verified.outcome == 'success'"); +}); +it("the actual private paid shell withholds launcher diagnostics on abnormal exit",async()=>{ + const root=await mkdtemp(path.join(os.tmpdir(),"manual-shell-test-")); + try{const shell=step("test","Run paid cell").run.replaceAll('${{ matrix.executionId }}',"test"); + const result=spawnSync("bash",["-c",`pnpm(){ printf '%s\\n' 'synthetic-private-diagnostic'; return 7; };\n${shell}`],{encoding:"utf8",env:{PATH:process.env.PATH,PRIVATE_EVIDENCE_ROOT:root,PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT:"bound"}}); + expect(result.status).toBe(7);expect(result.stdout+result.stderr).not.toContain("synthetic-private-diagnostic");expect(await readFile(path.join(root,"launcher-output.log"),"utf8")).toContain("synthetic-private-diagnostic"); + }finally{await rm(root,{recursive:true,force:true});} +}); +it("requires downloaded complete private cell receipts before retaining merged report",()=>{ + expect(step("report","Retain complete manual merged evidence only in restricted storage").if).toContain("manual_cells_downloaded.outcome == 'success'"); + expect(step("report","Aggregate normalized campaign results").run).toContain('"$EVIDENCE_RETENTION" = private'); + expect(step("report","Merge Playwright HTML and JUnit").run).toContain('> runner-e2e-merged-report/merge.log 2>&1'); +}); + +it("uses one exact read session per approved cell and a distinct write-only report session",()=>{ + const report=jobs.report.steps; + for(let index=0;index<8;index++){ + const credentials=report.find((row:any)=>row.id===`manual_read_credentials_${index}`); + expect(credentials.with["inline-session-policy"]).toBe("${{ steps.manual_report.outputs.read_policy_"+index+" }}"); + expect(credentials.if).toContain(`cell_count) > ${index}`); + const read=step("report",`Download restricted manual cell ${index}`); + expect(read.env.SELECTED_CELL_INDEX).toBe(String(index));expect(read.env.AWS_ACCESS_KEY_ID).toBe("${{ steps.manual_read_credentials_"+index+".outputs.aws-access-key-id }}"); + } + expect(step("report","Obtain exact current manual report archive session").with["inline-session-policy"]).toBe("${{ steps.manual_report.outputs.session_policy }}"); + expect(step("report","Verify all approved private cell transfers before merging").run).toContain("verify-downloads"); +}); diff --git a/tests/runner-e2e/private-history.test.ts b/tests/runner-e2e/private-history.test.ts index 39422cfdd2a..69db0311238 100644 --- a/tests/runner-e2e/private-history.test.ts +++ b/tests/runner-e2e/private-history.test.ts @@ -209,3 +209,21 @@ it("gives independent shared-manifest downloaders separate immutable receipts", receipt, () => new BoundedHistoryPublisher(`${receipt}.retention.json`, runner))).rejects.toThrow("receipt retention failed"); expect(receipts.size).toBe(3); }); + +it("separates a complete private transfer from missing required PASS evidence in the immutable receipt", async () => { + const root = await temporary(), source = path.join(root, "source"), receipt = path.join(root, "transfer.json"); + await mkdir(source); await writeFile(path.join(source, "result.json"), '{"status":"failed","failureClass":"permanent_infrastructure"}'); + const config = configurationRunner(configurations()); + const runner = async (args: string[]) => args[1] === "put-object" ? { stdout: '{"VersionId":"archive-version"}' } : config(args); + const evidenceCompleteness = { meaning: "available-files-and-transfer-only" as const, doesNotAttestFeatureOutcome: true as const, + cells: [{ executionId: "chat-features.runner-opencode.local.activity.agent-feedback", availableFileInventoryComplete: true as const, + requiredPassEvidenceComplete: false, missingRequiredEvidence: ["final-state.png"] }] }; + let saved: unknown; + await retainPrivateTransfer({ operation: "upload", kind: "manual-report", runId: "12345", directory: source, + bucket: "private-bucket", evidenceCompleteness, publisher: new BoundedHistoryPublisher(receipt, runner) }, receipt, + () => new BoundedHistoryPublisher(`${receipt}.retention.json`, async args => { + saved = JSON.parse(await readFile(args[args.indexOf("--body") + 1]!, "utf8")); + return { stdout: '{"VersionId":"receipt-version"}' }; + })); + expect(saved).toMatchObject({ evidenceCompleteness, transfer: { status: "complete" } }); +}); diff --git a/tests/runner-e2e/private-history.ts b/tests/runner-e2e/private-history.ts index d02e8172d09..cd058a23198 100644 --- a/tests/runner-e2e/private-history.ts +++ b/tests/runner-e2e/private-history.ts @@ -14,7 +14,8 @@ export const PRIVATE_ARCHIVE_BYTES = 264 * 1024 * 1024; export function privateArtifactKey(kind: string, runId: string) { if (!(["evidence", "report", "protocol-control", "protocol-report"].includes(kind) - || /^protocol-cell-[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(kind)) || !/^[1-9][0-9]{0,19}$/.test(runId)) { + || /^protocol-cell-[A-Za-z0-9][A-Za-z0-9._-]{0,199}$/.test(kind) + || /^manual-cell-[0-9a-f]{64}$/.test(kind) || kind === "manual-report") || !/^[1-9][0-9]{0,19}$/.test(runId)) { throw new Error("Private history needs a supported artifact kind and numeric workflow run ID"); } return `${PRIVATE_HISTORY_PREFIX}/campaigns/gha-${runId}-1/${kind}.zip`; @@ -45,6 +46,12 @@ export async function verifyPrivateHistory(publisher: BoundedHistoryPublisher, b } } +export interface PrivateEvidenceCompleteness { + meaning: "available-files-and-transfer-only"; + doesNotAttestFeatureOutcome: true; + cells: Array<{ executionId: string; availableFileInventoryComplete: true; requiredPassEvidenceComplete: boolean | null; missingRequiredEvidence: string[] }>; +} + type TransferOptions = { operation: "upload" | "download"; kind: string; @@ -53,6 +60,7 @@ type TransferOptions = { bucket: string; publisher: BoundedHistoryPublisher; receiptIdentity?: string; + evidenceCompleteness?: PrivateEvidenceCompleteness; }; function privateTransferReceiptKey(options: TransferOptions) { @@ -88,7 +96,9 @@ async function transferVerifiedArtifact(options: TransferOptions) { "--range", `bytes=0-${PRIVATE_ARCHIVE_BYTES}`, archive]); if ((await lstat(archive)).size > PRIVATE_ARCHIVE_BYTES) throw new Error("Private archive exceeds transfer allowance"); const metadata = await archivePrivateEvidence("unpack", archive, options.directory); - return { ...metadata, uri: `s3://${options.bucket}/${key}` }; + const hash = createHash("sha256"); + for await (const chunk of createReadStream(archive)) hash.update(chunk); + return { ...metadata, uri: `s3://${options.bucket}/${key}`, sha256: hash.digest("hex") }; } finally { await rm(temporary, { recursive: true, force: true }); } @@ -108,6 +118,7 @@ export async function retainPrivateTransfer(options: TransferOptions, receipt: s await options.publisher.save(); const snapshot = JSON.stringify({ ...JSON.parse(await readFile(receipt, "utf8")), ...(options.receiptIdentity === undefined ? {} : { receiptIdentity: options.receiptIdentity }), + ...(options.evidenceCompleteness === undefined ? {} : { evidenceCompleteness: options.evidenceCompleteness }), transfer: result ? { status: "complete", ...result } : { status: "failed_or_unknown" }, receiptUpload: { status: "pending_at_snapshot", maxBytes: 1024 * 1024 } }); if (Buffer.byteLength(snapshot) > 1024 * 1024) throw new Error("Private transfer receipt exceeds allowance"); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index d4a577b9f2c..7d18453e59c 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -541,8 +541,8 @@ describe("public repository paid workflow security", () => { /ref: \$\{\{ needs\.authorize\.outputs\.target_sha \}\}/g, ), ).toHaveLength(6); - expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(3); - expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(9); + expect(fullStack.match(/ref: \$\{\{ github\.sha \}\}/g)).toHaveLength(4); + expect(fullStack.match(/persist-credentials: false/g)).toHaveLength(10); expect(fullStack).not.toContain("ref: ${{ inputs.target_branch }}"); expect(fullStack).toContain( "PAPERCLIP_RUNNER_SOURCE_REVISION=${TARGET_SHA}", @@ -817,12 +817,12 @@ describe("public repository paid workflow security", () => { "PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e", ); expect(report).toContain( - "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}", + "PAPERCLIP_RUNNER_E2E_HISTORY_PUBLIC_BASE_URL: ${{ needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL || '' }}", ); const historyPrefix = "${{ vars.RUNNER_E2E_HISTORY_PREFIX || 'runner-e2e' }}"; expect(report).toContain(`PAPERCLIP_RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); expect(publisher).toContain(`RUNNER_E2E_HISTORY_PREFIX: ${historyPrefix}`); - expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.report.outputs.history_source_ready == 'true'"); + expect(publisher).toContain("needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' && needs.report.outputs.history_source_ready == 'true'"); expect( report.indexOf("Select latest workflow attempt per cell"), ).toBeLessThan(report.indexOf("Collect blob reports")); @@ -854,7 +854,7 @@ describe("public repository paid workflow security", () => { expect(manual).toContain("steps: &paid_test_steps"); const report = workflow.split("\n report:")[1]!.split("\n publish_history:")[0]!; const step = (source: string, name: string) => source.split(` - name: ${name}\n`)[1]!.split(/\n - /u)[0]!; - for (const [job, name] of [[paid, "Upload access-controlled packaged cell evidence"], [report, "Upload access-controlled merged report"]]) { + for (const [job, name] of [[paid, "Upload nonprivate packaged cell evidence"], [report, "Upload nonprivate merged report"]]) { expect(step(job!, name!)).toContain("if: always() && needs.catalog.outputs.operations_cadence == ''"); } expect(step(report, "Download cell evidence")).toContain("if: needs.catalog.outputs.operations_cadence == ''");