diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml new file mode 100644 index 00000000000..232408401d0 --- /dev/null +++ b/.github/workflows/chat-feature-testops.yml @@ -0,0 +1,448 @@ +name: Chat feature TestOps +run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} + +on: + workflow_dispatch: + inputs: + operation: + type: choice + options: [evaluate, bootstrap, launch-plan] + default: evaluate + description: Fixed evaluation, zero-result registration, or one TestOps plan launch + ALLURE_JOB_RUN_ID: + type: string + required: false + description: TestOps supplies its job-run identity; no test/source selector is accepted + ALLURE_USERNAME: + type: string + required: false + description: Reserved TestOps service metadata; never used as authorization + +permissions: + contents: read + +concurrency: + group: chat-feature-testops + cancel-in-progress: false + +jobs: + authorize: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + sha: ${{ steps.guard.outputs.sha }} + operation: ${{ steps.guard.outputs.operation }} + triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} + runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci + steps: + - &checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - &node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - &guard + name: Verify fixed default-source dispatch and both actor identities + id: guard + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + NODE_OPTIONS: '' + NODE_PATH: '' + run: node tests/runner-e2e/testops-chat-dispatch.ts + + select: + name: Validate TestOps selection or register zero-result setup + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: runner-e2e-testops-reporting + permissions: + contents: read + actions: read + outputs: + artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} + sha256: ${{ steps.selection.outputs.sha256 }} + steps: + - *checkout + - *node + - *guard + - &allure + if: needs.authorize.outputs.operation != 'launch-plan' + uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 + with: + allurectl-version: 2.23.1 + allure-endpoint: https://paperclip.testops.cloud + allure-project-id: '1' + allure-token: ${{ secrets.ALLURE_TOKEN }} + - name: Register the public job without an evaluation or a synthetic result + if: needs.authorize.outputs.operation == 'bootstrap' + env: + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results + run: | + mkdir -m 700 "$ALLURE_RESULTS" + allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap + - name: Validate the one real TestOps selection before any build or provider access + id: selection + if: needs.authorize.outputs.operation == 'evaluate' + env: + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} + CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} + CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results + run: | + mkdir -m 700 "$ALLURE_RESULTS" + allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection + - name: Start exactly the validated TestOps plan and exit without waiting for its child + if: needs.authorize.outputs.operation == 'launch-plan' + env: + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} + CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} + CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} + CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} + CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} + CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} + CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} + TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + GH_TOKEN: ${{ github.token }} + run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan + - name: Retain only closed metadata from zero-result setup + if: always() && needs.authorize.outputs.operation == 'bootstrap' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-testops-setup-${{ github.run_id }}-1 + path: | + ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json + ${{ runner.temp }}/chat-testops-bootstrap-state/once.json + ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json + ${{ runner.temp }}/chat-testops-bootstrap-state/job.json + ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json + retention-days: 30 + if-no-files-found: error + - name: Retain the one plan-launch receipt and stop state + if: always() && needs.authorize.outputs.operation == 'launch-plan' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-testops-plan-launch-${{ github.run_id }}-1 + path: | + ${{ runner.temp }}/chat-testops-launch-plan-state/once.json + ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json + ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json + retention-days: 30 + if-no-files-found: error + - name: Retain only the token-free exact selection receipt + id: selection_upload + if: steps.selection.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-testops-selection-${{ github.run_id }}-1 + path: testops-selection.json + retention-days: 30 + if-no-files-found: error + + build: + name: Build the immutable shipped native candidate without model credentials + needs: [authorize, select] + if: needs.authorize.outputs.operation == 'evaluate' + runs-on: ${{ needs.authorize.outputs.runner }} + timeout-minutes: 45 + outputs: + artifact_id: ${{ steps.bundle.outputs.artifact-id }} + steps: + - *checkout + - *node + - *guard + - &pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: 'false' + NPM_CONFIG_FUND: 'false' + NPM_CONFIG_UPDATE_NOTIFIER: 'false' + with: + version: 9.15.4 + - name: Install only the source-reviewed graph without lifecycle scripts + run: | + node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json + pnpm install --frozen-lockfile --ignore-scripts + - name: Build the existing TypeScript and native runner owners + run: | + pnpm --filter @paperclipai/paperclip-runner build:typescript + pnpm --filter @paperclipai/paperclip-runner build:runner-binaries + - name: Package source-bound outputs and resolved lock + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json + tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar + sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 + - name: Retain credential-free immutable outputs + id: bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 + path: | + chat-testops-build.tar.gz + chat-testops-build.tar.gz.sha256 + retention-days: 1 + if-no-files-found: error + + chat: + name: One real native OpenCode backlog request on AWS + needs: [authorize, select, build] + if: needs.authorize.outputs.operation == 'evaluate' + runs-on: ${{ needs.authorize.outputs.runner }} + timeout-minutes: 32 + environment: runner-e2e-private-paid + permissions: + contents: read + id-token: write + env: &retention + PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 + TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} + TARGET_SHA: ${{ needs.authorize.outputs.sha }} + MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' + SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + steps: + - *checkout + - *node + - *guard + - *pnpm + - name: Check out the same admitted source for trusted retention + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ needs.authorize.outputs.sha }} + path: .testops-trusted + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.build.outputs.artifact_id }} + path: chat-testops-build + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.select.outputs.artifact_id }} + path: chat-testops-selection + - name: Verify immutable outputs and token-free TestOps selection + env: + EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + run: | + set -euo pipefail + (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) + tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" + node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json + node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json + cmp chat-testops-lock.json chat-testops-worker-lock.json + cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml + pnpm install --frozen-lockfile --ignore-scripts + (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") + - name: Materialize and qualify pinned OpenCode runtime and browser + run: | + node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs + node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' + google-chrome --version + pnpm exec playwright install ffmpeg + - name: Prepare private evidence with shipped-main authority + id: private + env: + TARGET_ROOT: ${{ github.workspace }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare + - name: Obtain only the exact private cell storage session + id: cell_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.private.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Verify strict private custody before the model key + env: &cell_session + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify + - name: Execute only the fixed original request with zero retries + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog + PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} + PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome + run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 + - name: Retain exact original grade and private evidence, including failures + if: always() && steps.cell_credentials.outcome == 'success' + env: *cell_session + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell + + report: + name: Retain private originals and export only reviewed fixture results + needs: [authorize, select, chat] + if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: runner-e2e-private-history + permissions: + contents: read + actions: read + id-token: write + env: *retention + steps: + - *checkout + - *node + - *guard + - *pnpm + - name: Install trusted reporting dependencies only + run: | + node tests/runner-e2e/prepare-report-lock.mjs + pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" + - name: Prepare exact private report and cell-read policies + id: session + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session + - name: Obtain private report writer + id: report_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Obtain exact private cell reader + id: read_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Download the exact private original and verify all inventories + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '0' + AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: | + node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell + node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads + - name: Build the existing normalized campaign without exposing original logs + env: + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} + run: | + mkdir -p runner-e2e-merged-report + pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 + - name: Retain the complete merged report privately even when its grade failed + id: private_report + if: always() && steps.report_credentials.outcome == 'success' + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report + AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report + - name: Export only the bound normalized result, JUnit and marked PNGs + id: export + if: always() && steps.private_report.outcome == 'success' + env: + TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts + - name: Upload only the sanitized TestOps projection + if: always() && steps.export.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-e2e-testops-${{ github.run_id }}-1 + path: runner-e2e-testops/ + retention-days: 30 + if-no-files-found: error + + upload: + name: Report the original outcome to its exact TestOps case + needs: [authorize, select, chat, report] + if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: runner-e2e-testops-reporting + permissions: + contents: read + actions: read + steps: + - *checkout + - *node + - *guard + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-e2e-testops-${{ github.run_id }}-1 + path: runner-e2e-testops + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.select.outputs.artifact_id }} + path: chat-testops-selection + - *allure + - name: Verify and upload only the bound safe case outcome + id: allure_upload + env: &testops_reporter + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results + run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops + - name: Verify the actual TestOps result after upload processing + id: verify_upload + if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') + env: *testops_reporter + run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops + - name: Retain only closed actual TestOps verification metadata + if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-testops-verification-${{ github.run_id }}-1 + path: | + ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json + ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json + retention-days: 30 + if-no-files-found: error diff --git a/DESIGN.md b/DESIGN.md index 1b740713f13..63e8b37c49f 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -84,6 +84,8 @@ Background refreshes and history expansion keep the conversation and composer mounted. A failed history request shows an inline retry without hiding readable content. Full-history and classic views retain the bounded 15-second initial wait. +Standing agent chats open with the latest 20 contiguous messages and hydrate only their runs. Upward scrolling requests earlier pages; a fixed-height status slot says “Loading earlier messages…” while the page is pending. Preserve the logical row anchor and mounted composer through prepends and background refresh. Retry failed history inline without automatic retry loops. Answer receipts and created-task history use the same scrolling affordance. + Only the viewer's own human comments sit on the right in blue bubbles. Agents and other humans sit on the left on the page surface, with their own name and avatar above the message. Human identity comes from the company directory; diff --git a/doc/DATABASE.md b/doc/DATABASE.md index b794f1e51ee..ac0d63bb728 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1545,6 +1545,15 @@ named `review_wait_benchmark...` can be supplied through `PAPERCLIP_TEST_DATABASE_URL` to compare another PostgreSQL major version. DataRow bytes exclude response metadata and transport overhead. +## Object-backed run evidence + +Optional segmented run logs and raw provider traces store their payloads and +range indexes directly in S3-compatible storage. Existing `heartbeat_runs` and +`provider_trace_records` contain backend/reference and lifecycle metadata; +there is no new payload table or per-segment SQL publication. Object manifests +use conditional writes to fence publication. Existing structured events retain +their database append contract. See [run evidence storage](run-evidence-storage.md). + ## Cloud runtime configuration singleton The private `instance_settings` key `cloud-runtime-config:v1` holds a strictly diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3ab1e2dea46..17712b94903 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -3883,6 +3883,16 @@ in `_test` for `PAPERCLIP_MCP_SLEEP_TEST_DATABASE_URL`, also accepts `PAPERCLIP_INTERACTIONS_TEST_DATABASE_URL`. The suites migrate and write to those URLs; the sleep suite truncates company data between tests. +## S3-backed run evidence + +New logs and opted-in provider traces automatically use the instance's existing +S3 storage configuration, bucket, credentials and prefix, under `run-logs/` and +`provider-traces/`. No extra storage switches are needed. Local-disk instances +remain local. See [run evidence storage](run-evidence-storage.md) for required provider +capabilities, stable keys, operating limits, automatic history migration and qualification. +Do not remove a web volume until its complete retained evidence inventory and +the independent disposable-server requirements are satisfied. + ## Cloud startup configuration Cloud stores nonsecret startup choices in a private PostgreSQL singleton. diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index ec85824d7e7..1bbb8ad26fb 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -580,22 +580,33 @@ transaction or an intentional freshness/authorization discard. These retained homes are recovery evidence, not a background replay queue; persistent database or lock failures still require operator intervention. -Quota OAuth replacement tokens are encrypted with the instance secrets master -key and fsynced under `/quota-credential-recovery//` -before vault, grant, or activity writes. Storage and encryption are checked before -exchanging a single-use refresh token. Failed saves retry without another OAuth -exchange; persistent failures keep the encrypted record for the next quota poll, -including after a restart. New OpenAI subscription runtimes serialize credential -reads with quota exchanges and recover a matching pending replacement before -materializing an auth home. Authentication-failure handling also recovers a -matching replacement before marking the grant invalid. A failed recovery save -defers these actions and leaves both the active grant and journal intact. -The journal is removed only after database commit (or -when an authorized newer credential makes it obsolete). Replay checks the grant, -connection, secret, and original credential fingerprint and cannot reactivate a -revoked grant or overwrite a reconnect. Back up this directory and the instance -secrets key with the instance data. Loss of durable storage while receiving a -provider token can still require reconnecting the account. +Server-owned quota refresh validates and locks the current grant and secret in +Postgres before exchanging a token. A failed database prerequisite prevents the +exchange. A returned replacement is saved directly into the existing secret +store. The same logical operation keeps it in memory for at most three save +attempts. A lost commit response is reconciled against the current saved value; +reconnect, revocation, secret-version and session-epoch changes win over a delayed +save. Save retries never repeat the OAuth exchange and no new recovery file is +written for this path, in cloud or local deployments. + +If all save attempts fail, the request fails. If the server process then exits, +the unsaved replacement is lost. V1 accepts signing in again after the next failed +use; it does not report successful refresh or keep a second credential vault. + +Remote Codex Runner tasks retain their private credential until secret writeback +and the lease's credential handoff commit. Provider completion follows that +handoff. Failed exports or database saves leave it pending for the existing +cleanup path, including a replacement web receiver. Its authority still checks +the company, grant, secret, version selector and session epoch. + +Existing encrypted records in +`/quota-credential-recovery//` remain readable for +cutover settlement. Do not delete unresolved files or remove the old volume +before inventory and settlement. Legacy replay cannot reactivate a revoked grant +or overwrite a reconnect. Local Google Antigravity retirement still uses its +existing encrypted recovery record and private-home ownership; remote and warm +Google subscription sessions remain unsupported. This local path is not a cloud +fallback. Cloud cutover must settle any observed legacy credential files first. ## Advanced provider routing (2026-10-02) diff --git a/doc/evals.md b/doc/evals.md index 5acd205811b..78c9c3ce467 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -43,7 +43,10 @@ two natural-language task scenarios into local native Codex subscription and OpenCode Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, using ordinary production instructions, real browser messages and independent -state/provenance checks. The broader YAML catalog remains planning inventory. +state/provenance checks. A separately selected native OpenCode/DeepSeek1017 +backlog cell measures TestOps integration acceptance with the same oracle and +original bounds. Its source registration is not an AWS or TestOps pass. The +broader YAML catalog remains planning inventory. The explicit-only [Daytona restart continuity journey](../tests/runner-e2e/README.md) extends the existing native restart oracle to a 125-second controller outage, diff --git a/doc/observability.md b/doc/observability.md index 5065e511ecc..e2bb4065caf 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -254,6 +254,17 @@ It does not extend an earlier execution attempt or include time waiting for the background worker to run. Retries have separate roots. A check that finds no eligible work can end without a restore child. +`task.response_presentation` owns recovery of an already committed native reply. +It starts a separate linked root because deferred presentation can run after the +execution attempt and workspace finalization have ended. Worker discovery for +a specific run and worker checks include candidate reads under +`response.resolve`. Each discovery or check owns its root; direct recovery and +its worker check share one root when nested. `response.transaction` +includes the recovery transaction; `response.transaction.entry` measures entry +into its callback, including connection acquisition. These spans retain only the +same hashed run ID and never capture reply content. A successful task status is +separate from the time the reply is saved. + These spans exist before provider dispatch, including setup failures. Preparation phases have only the existing `paperclip.task.run.run_id` attribute: the first 12 hexadecimal characters of the run UUID's SHA-256 hash. Failed operations set error status @@ -641,7 +652,8 @@ Without an OTLP endpoint, tracing remains silent and does not export data. The run link in Honeycomb matches `task.provider_session` for native runs, `task.run` for legacy adapters, `task.execution_attempt` for server preparation and settlement, and -`task.workspace_finalization` for deferred native copyback. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the four root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged. +`task.workspace_finalization` for deferred native copyback and `task.response_presentation` +for committed native reply recovery. It retains the exact hashed run-ID and service filters and groups by trace ID. The URL builder uses Honeycomb’s supported `in` operator for the five root names; see the [query specification](https://github.com/honeycombio/honeycomb-mcp#query-specification-for-run_query). This changes only the query link. The operator endpoint gate and span attribute allowlist remain unchanged. The canonical lifecycle is: diff --git a/doc/plans/2026-10-30-chat-feature-testops-smoke.md b/doc/plans/2026-10-30-chat-feature-testops-smoke.md new file mode 100644 index 00000000000..220be450b52 --- /dev/null +++ b/doc/plans/2026-10-30-chat-feature-testops-smoke.md @@ -0,0 +1,89 @@ +# Chat feature TestOps smoke — 2026-10-30 + +## Finish line + +Create one dedicated runnable TestOps plan and obtain one genuine passing +`tasks.create-backlog` result through TestOps → GitHub Actions/AWS → native +OpenCode/OpenRouter `deepseek/deepseek-v4.1-flash-1017`. The result must prove +the saved task, assignment, backlog-at-creation state, no worker execution, +agent attribution, rendered link and reload persistence. Confirm cleanup, +safe screenshot/report access and the final TestOps result link. + +This is a new integration-acceptance measurement. It preserves the original +244-scenario campaign, attempts, grades and existing Codex measurements. +Delegation and the other catalog scenarios are deferred. + +## Source and execution contract + +- Public selector: `chat-features.runner-opencode-deepseek-1017.local.create-backlog`. +- Existing YAML query, fixture and independent backlog oracle stay unchanged. +- One human query and one accounted controller work run; 120-second turns, + 420-second attempt, 300-second settled-state observation, zero retries. +- Existing 500-cent company/agent policies and unknown setup usage remain + explicit. Morgan is fixture context and must not execute this backlog task. +- Only the existing protected `OPENROUTER_API_KEY` is required. Agent + configuration uses the normal encrypted secret-reference path. +- OpenCode 1017 is selected only for this cell and remains a qualification + candidate until the real run. Other suites retain their existing models. + +## Ownership and integration + +Campaign owns public catalog/profile calibration and these progress notes. +Core owns the public AWS workflow/execution adapter, one coherent public PR, +and the sole actual operation. Coordinator supplies the public TestOps +selection/reporting helper and the separate private credential sealer. +The public workflow owns the dedicated plan. Source peers precede publication. + +The current Cloud TestOps adapter is scoped to `qa-basics` Product journeys; +the new chat plan must have its own exact plan/case mapping. Reuse normalized +result/check/provenance and existing packaged evidence. Do not upload private +traces, provider sessions, raw logs or unreviewed media as TestOps attachments. + +## Current progress + +The public patch adds only the OpenCode backlog selection while preserving +the two Codex cells. A genuine parent selector control failed; all 68 focused +oracle/catalog controls and the owning type check now pass. The expected +suite-size assertion was updated for the single added cell. Independent source +review passed and the nine-file profile slice is committed as `0a996033`. +No local app, database, native SDK, +provider, model, install or build qualification was run. + +The separate protected credential bootstrap seals the existing reporting token +for the source-pinned public reporting environment. Delivery is complete without +exporting plaintext. The public adapter has 53 focused authority/export/workflow +controls and 39 API/CLI/result controls passing, plus owning typechecking and +workflow lint. Unchanged controls are reused after narrow source corrections. +The independent composition source peer passed. +The public infrastructure [PR 18322](https://github.com/paperclipai/paperclip/pull/18322) +is open. Its first normal CI run rejected the new workflow in the existing paid +workflow allowlist and reported a separate heartbeat scheduling assertion; both +failures remain recorded. The policy correction preserves the older workflow +guards and rejects unsafe new-route variants. The reporting token is configured +through the reviewed encrypted bootstrap. + +Review also added a one-time initial administrator launch fence. A protected +expected GitHub run number and actual own-run metadata must match before any +TestOps mutation. REST catalog reconciliation preserves exact case aliases; +job wiring uses only the reviewed existing project GitHub integration. These +provider-free helper controls pass. Actual registration IDs, tenant API behavior, +provider execution and the TestOps result remain unqualified. Dedicated plan, +AWS evaluation and final result links remain pending. Source checks alone do not +satisfy the finish line. + +The final source consolidation adds actual admission-wrapper filesystem coverage, +the existing reviewed lock normalization with frozen installs and a matched +build/worker receipt, plus same-job server-result readback after Allure upload. +Focused checks cover 41 admission controls, 31 lock/owner/workflow controls and +44 API/reporter controls; unchanged profile/oracle evidence is reused. This is +source verification. The plan and actual model result are still not created. + +Current-main composition preserves the four existing Codex and production-qualified +OpenCode cells introduced by #18319. The TestOps cell has its own +`runner-opencode-deepseek-1017` profile, so the existing OpenCode model remains +unchanged. It uses the shipped v2 grader, queue accounting and `final-state.png` +capture. The new profile keeps the original query, one work run, one human message, +120-second turn, 420-second attempt, 300-second observation bound and zero retries. +The existing private session and trace-asset owners admit only this additional +exact selector; the frozen 244-case candidate-approval registry remains unchanged. +Source checks do not establish a TestOps registration or model outcome. diff --git a/doc/plans/2026-10-30-external-run-logs-and-provider-traces.md b/doc/plans/2026-10-30-external-run-logs-and-provider-traces.md new file mode 100644 index 00000000000..b8f758b9e29 --- /dev/null +++ b/doc/plans/2026-10-30-external-run-logs-and-provider-traces.md @@ -0,0 +1,567 @@ +# Durable run logs and provider traces for disposable web servers + +Status: implementation and PR qualification in progress. Runtime storage, +history migration, and canonical native source replay are implemented using the +instance's existing storage selection. Production-provider qualification remains +a rollout gate. +Owner: this Codex chat, through the reviewable PR. Date: 2026-10-30. +Revision: object-storage-first, following Dotta's operational correction on 2026-10-30. +Source revision inspected: `0c70cedce9a803bdc201324528abb35e423e7cd0`. +Parent effort: near-zero-downtime Railway deployment of the main web/API server. + +## Goal and boundary + +After the main Paperclip web server and its local filesystem disappear, a +replacement connected to the same Postgres, object storage, and secrets can +read all acknowledged run-log output and retained provider traces, then continue +recording the same remote execution without gaps or duplicate records. + +The ordinary journey is: open a task's run, inspect its output, replace the web +server while its remote agent is working, and continue reading the run. An +authorized administrator can still inspect, reveal, download, and delete a +captured provider trace. Existing consumers of run output continue working. + +This is the **run logs and provider traces** slice of the parent plan. Railway +still documents that volume-backed deployments cannot overlap, which makes +removing durable filesystem dependencies necessary. This slice alone does not +remove every volume dependency or prove zero-downtime HTTP deploys. +[Railway volume limitations](https://docs.railway.com/volumes/reference#caveats). + +Firm requirements: + +- All retained historical logs and unexpired traces survive migration. Dotta + explicitly selected this on 2026-10-30. Do not shorten existing retention. +- A successful durable append survives abrupt process and disk loss. An orderly + shutdown flush is useful but insufficient evidence. +- Preserve company/task authorization, redaction, stable log byte offsets, + trace integrity checks, opt-in capture, existing limits, expiry, and deletion. +- Store log and raw trace payloads directly in S3-compatible object storage. + Postgres holds small ownership/lifecycle/reference records only; no new + per-token or per-frame SQL, payload tables, or DB-to-S3 payload compaction. + Local files may be caches or unacknowledged buffers. Preserve local mode. +- Keep task results, usage settlement, and optional diagnostics independent. +- Preserve bounded work and existing sleep behavior. +- Reuse existing instance storage settings and per-stack bucket credentials. + New logs and opted-in traces follow S3 file storage automatically, with their + own folders beneath the existing prefix. Add no required configuration. + +Dotta approved implementation on 2026-10-30. Delivery ends at a reviewable PR +with relevant checks and review findings resolved. Deployment, volume removal, +and production history migration require the parent effort's target and cutover. +The acceptance matrix below remains the full storage-slice contract; an +implemented primitive does not establish that every row has passed. + +Deferred to the parent effort: managed agent files, workspace recovery, +accounting/credential journals, workspace-operation logs, readiness/draining, +global live-event fanout, scheduler ownership, volume detach, and fleet rollout. +These remain blockers to claiming the entire web server disposable. No changes +to first-party Telemetry or operator OpenTelemetry export are proposed. Raw +provider protocol traces here are a separate, restricted diagnostic store. + +## Current implementation and implications + +| Owner | Current behavior | Required change | +| --- | --- | --- | +| `server/src/services/run-log-store.ts` | Only `local_file`; local NDJSON append; best-effort final/in-flight S3 mirroring; local-first reads | Add an explicit external store with shared durable append and reads | +| `server/src/services/heartbeat/run-output.ts` | Redacts/compacts before append; process-local output sequence; progress persisted on a 60-second cadence | Preserve stable source identities; batch output to S3; distinguish live delivery from durable receipts | +| `server/src/services/heartbeat-run-events.ts` | Structured events already use atomic DB sequence allocation and native source deduplication | Preserve this authority; do not turn its event table into a duplicate blob store | +| `server/src/services/provider-trace-store.ts` | DB metadata points to local native and `.rehydration` sidecars; 24-hour retention and 64 MiB capture limit | Remote payload storage, recoverable capture cursors, durable expiry/deletion obligations | +| Runner trace sinks and `runnerd-codex-transport.ts` | Rust/TypeScript capture and interpretation producers use paths; TypeScript capture is bounded and fire-and-forget | Introduce a durable capture delivery boundary and preserve channel/frame lineage | +| `doc/testing-native-remote-continuity.md` | Sparse checkpoints deliberately omit streaming output; acknowledged display output may be lost before app projection | Qualify replay/retention through web replacement without restoring per-token DB checkpoint writes | +| Run readers | Heartbeat, issues, feedback, and customer-success inspection assume `local_file` | Resolve each persisted backend, including historical records, through one reader | + +Do not infer cloud durability from `RunLogStore.append()` succeeding today, or +from a trace's local `acknowledgedDebugSequence`. Neither proves remote commit. +Generic stdout/stderr and raw traces cannot be reconstructed from +`heartbeat_run_events`. The implementation investigation found a narrower +surviving source: native control ingestion already commits the exact validated +PRP event before ACK. Its canonical `paperclip.prp.event` log export can replay +that existing source with an S3 cursor, without adding payload writes. This +replaces the tentative second Runner spool below for canonical native output. + +## Operational decision + +The original proposal used Postgres for payload chunks and then copied them to +S3. That choice was premature. It adds payload writes, WAL/replication/backup +bytes, index churn, deletion/vacuum work, and extra database connections to the +same database serving the control plane. Batching would reduce transaction +count but would not remove the byte load. No workload measurement justified it. + +Use **S3 as the durable payload store from the first persisted batch**. Local +append files remain useful for buffering and cache speed; the web server's disk +cannot be their only acknowledged durable home. Existing S3 mirroring is a useful +starting point, but repeatedly uploading the entire growing file amplifies +network and storage work. Upload each newly sealed segment once instead. + +| Location | Responsibility | +| --- | --- | +| Browser live stream / bounded server memory | Immediate progress display; not a durable receipt | +| Existing native control event source / Runner outbox | Existing committed native records survive ACK; unacknowledged events remain replayable at the Runner | +| S3-compatible storage | Immutable log/trace segments and bounded manifest/index pages | +| Postgres | Existing company/run ownership, backend/reference, final status and expiry/deletion state; no per-batch writes | +| Disposable web disk | Optional bounded buffer/read cache; never the only acknowledged copy | + +Existing structured `heartbeat_run_events`, task results, usage, and controller +checkpoints retain their contracts. This is not a rewrite of those data paths. + +### Cost model and the first gate + +Let `A` be actively producing streams, `r` output records/second/stream, `b` +bytes/second/stream, `T` maximum dirty-buffer age, and `S` target segment bytes. +For a steady stream, approximate segment rate is +`A × max(1/T, b/S)`; record rate is `A × r`. Empty streams emit no segments. +Bursts, final flushes, retries, indexes, and trace channels add work and must be +counted separately. A stream is not necessarily an agent: raw tracing can add +multiple streams to one run. + +Illustrative arithmetic, **not a fleet measurement or chosen production setting**: +100 continuously active streams at 10 records/second give 1,000 records/second. +If five-second flushing dominates, they produce 20 segments/second, or 72,000 +segments/hour. With one small metadata publication transaction per segment, +that is still 20 transactions/second. A 30-second interval gives about 3.3 +segments/second but retains more pending output. S3 index PUTs, GETs, retries, +and transfer costs are additional. Neither option is free. + +The first gate measures real output sizes/rates and concurrency, then compares +candidate size/age thresholds (initial experiment: 256 KiB–1 MiB and 5–30 +seconds, not defaults). Include quiet text, bursty tool output, raw tracing, +long runs, slow S3, and the expected concurrent-run envelope. Record: + +- Incremental Postgres transactions, row updates, WAL bytes, connections, and + application p95 latency; no log/trace payload bytes inserted into Postgres. +- S3 PUT/GET/HEAD/LIST counts, index overhead, object count, stored bytes, + compression CPU, transfer volume, and upload/read p95 latency. +- Pending bytes, upload lag, read recovery time, and behavior at queue limits. +- A daily/monthly cost projection using the **actual configured provider's** + rates and observed workload/duty cycle; do not substitute AWS prices for an + S3-compatible vendor or assume 24-hour production. +- Zero added recurring SQL/network work after buffers and obligations settle. + +Record a concrete operating envelope, request-rate/backlog limits, and latency +budget from this evidence before enabling the feature. If the batched metadata +load is material, evaluate an object-store manifest head with conditional writes +and lifecycle-only SQL. Choose one publication scheme for the release; do not +ship two unqualified distributed state machines. Do not return to DB payload +buffering as an automatic workaround for S3 latency or failure. + +## Design + +### 1. External backend and batched publication + +Keep local mode and add a versioned S3-backed store. Inject the existing object +provider plus narrow metadata/ownership operations. Readers dispatch from the +persisted backend/version, including historical records. Remove hardcoded +`local_file` checks in heartbeat, issues, feedback, and customer-success +inspection together. Preserve current `RUN_LOG_S3_*` mirror configuration for +legacy mode, without redirecting unrelated product file storage. + +Reuse the existing instance storage configuration without additional switches. +Cloud already provisions a bucket and bucket-scoped credentials per stack. +Preserve any configured storage prefix and place journals in `run-logs/` and +`provider-traces/`, with company/run namespaces underneath. New journals must not +use a separately configured legacy mirror bucket. Configuration +errors fail explicitly; never select local-only persistence silently. Add only +small company/run-bound stream metadata, with unique identities and constraints. +There are **no proposed run-log payload-chunk or raw-trace payload tables**. + +Selected publication protocol after the local operational experiment: + +1. Redact/compact log output through the existing owner and enqueue exact NDJSON + bytes with stable source/attempt identities in a bounded buffer. A buffered + enqueue and a durable receipt are different results. +2. Seal on byte threshold, the first dirty buffer's actual deadline, explicit + flush, completion, or graceful handoff. Allocate sequence/byte offsets within + the current writer generation; do not write SQL per record. Seal channels + separately when their trace ordering requires it. +3. Upload a new immutable segment with digest, length, source range, and logical + byte range. Then upload the bounded immutable index/manifest update. Keep + manifest pages bounded; do not rewrite a growing full-history JSON document + every few seconds. Retain exact logical bytes even when compressing storage. +4. Publish one bounded head with S3 ETag `If-Match` (or `If-None-Match` for + creation). The head carries owner generation, committed cursor, and at most + 53 immutable index roots. Check the admitted database execution authority + before claiming a new generation. No per-batch SQL or new stream table is + needed. Conditional head publication fences stale generations and tombstones. +5. Release durable receipts for the published contiguous range. Reconcile a + lost object response from the exact head before retrying. Same-source replay + must not duplicate bytes; conflicting replay fails explicitly. + +Upload/index retries reuse immutable identities and verify ambiguous results. +Do not hold a DB transaction or connection during network upload. New immutable +index nodes combine completed binary subtrees; all normally published nodes +remain referenced, avoiding an obsolete index object for every flush. Metadata must +not grow with the number or size of individual text records: segment descriptors +and source indexes live in S3. Keep lifecycle/audit metadata bounded. + +Use a company/agent/run prefix with a generation-fenced head. A stale owner may finish an upload, +but cannot publish it after takeover. A successor adopts the last published +manifest and replays its uncommitted tail. Per-stream prefix tracking plus scoped, +paginated enumeration must recover unpublished-object cleanup after a crash; +never expose bucket-wide listing through public APIs. Do not garbage-collect a +pending publication merely because one DB query did not find its result. + +AWS supports conditional object creation and ETag-based updates, but the current +storage interface does not expose them and compatible vendors need verification. +If eliminating per-segment SQL becomes necessary, qualify conditional writes, +ownership fencing, and manifest recovery explicitly before selecting that design. +[Conditional writes](https://docs.aws.amazon.com/AmazonS3/latest/userguide/conditional-writes.html). + +### 2. Live viewing, durability, and the pending tail + +Live output need not wait for an S3 PUT to appear in the browser. Preserve its +stable source identity so reconnect/reload can reconcile it with durable history. +An open viewer can see the pending tail; a different server can read the published +S3 prefix. Do not promise zero-lag cross-server tail reads until a replay/relay +path proves them. The flush threshold bounds normal durable-read lag, not a +promise that storage is always available. + +The hard problem is **who retains output until publication**, not which database +column it uses. A timer uploading from disposable web memory alone cannot +preserve the pending tail after a crash. Distinguish these boundaries: + +- Store durable receipt: only after the segment/index and metadata publication. +- Runner control-protocol ACK: currently may allow streaming output to leave + replay retention before log projection. Inspect and qualify this exact path. +- Live browser delivery: presentation, not permission to discard the source. + +Selected after inspecting the real ACK path: reuse the native control port's +already-durable canonical event rows. The S3 head advances their export cursor +atomically with the log bytes. Takeover reads later native rows in bounded +16-row pages, verifies source identity and hash, then uses the same formatter, +redaction and compaction as live recording. The ordinary contiguous stream adds +no SQL. Startup, missing callback ranges, and interrupted settled archives use +bounded recovery reads; settled repair waits for the old native lease to expire. +This avoids a second payload copy, new per-record DB writes, and a new Runner +retention protocol. Other stdout and raw trace bytes are not inferred from this +source. A retained native control row remains readable after Runner cleanup. +The existing Runner outbox retains unacknowledged records. No second spool, +broker, always-on logging service, or full transcript checkpoint is added. + +An alternative is holding the existing transport ACK until a batch is published, +but use it only if a real protocol experiment proves bounded pipelining and +prompt Stop/control handling. Serial awaited callbacks plus a five-second timer +can become a five-second wait per record; that is unacceptable. Do not rebuild +full transcript checkpoints in Postgres to sidestep this issue. + +For server-only diagnostics with no surviving source, successful durable receipts +still require S3 publication. Local enqueue alone is never advertised as durable. +Missing uncommitted optional diagnostics are explicitly incomplete. Preserve the +original strict guarantee for every acknowledged durable record; any proposal +to accept a bounded crash-loss window for ordinary output is a separate product +decision, not an automatic relaxation in this revision. Host-local agent-process +survival remains outside the web-server replacement guarantee. + +On S3 failure, retain bounded pending work at the source, back off, and surface +capture/persistence state without falsely completing the log. At capacity, +backpressure or stop new admission according to the existing execution owner; +do not allocate unbounded memory/disk or switch payloads into Postgres. Optional +trace overflow can become incomplete/truncated. Preserve essential control, +Stop, task results, and usage settlement. Runner cleanup cannot destroy a pending +required-output obligation; either finish export or leave a durable cleanup hold. + +### 3. Reading and storage amplification + +Read the published pointer, then bounded manifest/index pages and the segments +covering the requested byte range. Cache immutable contents by digest. Preserve +byte offsets, NDJSON/UTF-8 alignment, EOF, cancellation, and bounded memory. +Compressed segments need logical offset metadata and bounded decompression; +HTTP ranges over compressed bytes are not logical log offsets. Preserve exact +full-stream digest semantics when finalizing. + +The first release can serve a finalized log by streaming its segments. Do not +require a full re-upload or database compaction pass at every completion. Add +optional segment coalescing only if measured read/request savings justify its +extra I/O, publication/cleanup complexity, and temporary storage. Retention and +read-path limits must account for small-object growth and indexes. Avoid one +object per token/frame and repeated whole-log mirroring. + +S3 publication is not atomic with Postgres; object upload precedes pointer +publication. An orphan object is recoverable cleanup, not proof of an accepted +append. A missing/corrupt published object is a storage integrity error, not an +empty log. Verify the actual provider's PUT/GET/range/checksum behavior rather +than assuming AWS guarantees for every compatible backend. +[S3 consistency model](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html#ConsistencyModel). + +### 4. Provider trace boundaries + +Use the same batched object-payload mechanism with trace-specific ownership. +Keep capture opt-in, existing secret/identity suppression, admin access, +company/task-read checks, no-store responses, reveal/download audit, and trusted +workspace-diff reprojection. Raw bytes never enter ordinary events, live logs, +activity payloads, or general attachments. Provider parsing remains in drivers. + +Preserve native and rehydration channels, frame IDs, source/debug sequences, +interpretation links, and raw digests. Capture transport must retain pending +records outside disposable web storage until a storage receipt. Reconstruct +frame lookup on takeover. A complete trace requires all required channel ranges +and terminal records durably published and the integrity assessment passing. +Capture failure yields incomplete/truncated without restarting the provider or +blocking task/accounting settlement. Qualify all currently supported producer +families; do not silently advertise a legacy path-only producer as durable. + +Encrypt raw trace objects with the existing stable-key facilities and bind them +to company/trace/channel/segment identity. Keep only metadata and the appropriate +key reference in Postgres. Preserve the existing 24-hour expiry from original +capture and 64 MiB capture limit, with separate bounds for encoded/index overhead. +Restart/import never extends expiry. Historical imports preserve exact bytes, +including native-then-rehydration download ordering and digest. + +Expiry/deletion first creates a durable tombstone and fences publication, then +cleans segments, index pages, abandoned uploads, and caches. Retain unresolved +cleanup obligations through restart and concurrent uploads; no resurrection. +Preserve `{ok:true}` only after required payload cleanup. Verify object-version +behavior for the actual bucket, and document historical backup retention +separately. Restore must honor expiry/deletion receipts before exposing traces. +Audit contains content-free metadata only. + +### 5. Idle behavior and recovery + +A nonempty buffer schedules one actual flush deadline; successful settlement +cancels it. Use the existing coordinator/scheduler for server-owned work and the +Runner lifecycle for producer-owned work. Retries exist only for unresolved +uploads/publications/deletes. Coalesce work and bound global as well as per-run +upload concurrency; prevent many runs flushing at once from starving HTTP or +connection pools. Do not create per-agent storage clients or idle keepalives. + +Start/resume/takeover reconciles active stream metadata, source retention, and +unfinished cleanup. Existing DB work signals are process-local; do not rely on +them to wake another server. Until the parent's cross-instance wake mechanism +exists, retain explicit owner/recovery/sleep obligations. Completed history alone +creates no recurring scan. Trace expiry uses its real deadline and the existing +supported wake/blocker contract. + +## Delivery slices + +| Slice | Useful result and owned surfaces | Evidence/dependency | +| --- | --- | --- | +| 1. Operational and durability prototype | Workload measurement, S3 segments, publication/cost model, actual Runner pending-tail/ACK experiment | Select batching and one publication scheme; demonstrate acceptable cost and control responsiveness before schema commitment | +| 2. S3-backed run logs | Store/readers, bounded buffering/indexes, small metadata schema, replay/ownership, cleanup and local compatibility | Slice 1; two-process replacement, ambiguous PUT/commit, stale-writer and backlog tests | +| 3. S3-backed provider traces | Producer sinks/rehydration, trace store/routes, expiry/deletion | Slice 2 primitives; actual capture integrity, access and cleanup races | +| 4. History migration and qualification | Resumable importer, compatibility/authority switch, documentation and staging evidence | All retained inventory accounted for; empty-disk reads and supported rollback after new writes | + +These are reviewable boundaries, not a fixed PR count. Keep each usable with the +existing instance storage profile. Raw trace capture remains opt-in. Keep storage +logic in its owners, not large heartbeat/executor additions. +Generate necessary additive metadata migrations with `pnpm db:generate` and +follow snapshot-retention rules. There is no DB payload migration or compactor. + +Update `doc/run-log-events.md`, `doc/DATABASE.md`, `doc/DEVELOPING.md`, and +`doc/testing-native-remote-continuity.md` with actual guarantees and operating +limits. Update affected run/inspection feature/access map entries and run +`node scripts/check-maps.mjs`. Change shared types/UI only when their contract +changes; keep incidental storage mechanics out of normal product controls. + +## Migration and rollback + +1. Ship a compatibility release that reads both stores, understands new formats, + and enforces the persisted minimum reader/writer capability on external runs. + Keep the volume and legacy writing enabled initially. Record this release as + the rollback floor before switching any authority; an environment flag alone + cannot stop an old binary from mishandling new rows. The deployment/rollback + preflight must reject binaries older than that floor: a pre-feature binary + cannot be expected to understand a newly introduced DB guard. +2. Inventory every retained run log and unexpired trace by company/run. Classify + local complete files, remote mirrors, active files, missing payloads, conflicting + copies, and expired/deleted traces. A mirror may be a partial prefix: verify + against authoritative bytes/metadata, never assume it is complete. +3. Backfill settled resources with bounded indexed keyset batches and resumable + receipts. Preserve original bytes, offsets, digests, attempt information, + source attribution, and trace expiry. Expired/deleted traces stay unavailable. +4. Under the parent's permitted one-time maintenance window, stop admission and + settle/fence remaining legacy writers. Copy final active tails and both trace + channels. Recheck hashes and byte totals after fencing; no copy-while-writing + race can establish completion. +5. Publish each resource's backend/reference transactionally. Record a migration + manifest of migrated, already-remote, expired/deleted, and unresolved resources. + Missing/conflicting retained bytes block the preservation gate. Report genuine + pre-existing loss; neither fabricate history nor quietly waive Dotta's choice. +6. Start B with the log/trace directories absent and old storage inaccessible. + Read historical data and append new data through normal APIs. Confirm every + in-scope inventory item has a valid disposition. Keep the old volume retained + under the parent's recovery policy; this slice does not detach/delete it. +7. Roll back to the compatibility release against the same external data and + prove it reads writes made after cutover. Do not switch authoritative reads + back to stale local files. Defer destructive schema cleanup until rollback + readers have retired. + +## Acceptance matrix + +| Case | Observable finish condition | Evidence | +| --- | --- | --- | +| Operational envelope | No new payload/per-token SQL; batch metadata and object traffic fit measured budgets | Real workload baseline, WAL/transactions, request/storage/transfer model and app/control latency | +| Active/completed run | B retrieves every durable receipt exactly once after A and its disk disappear | Separate processes/directories; normal authorized reader; byte/digest comparison | +| Pending tail | Web replacement retains/replays required output without duplicating provider work | Actual Runner spool/ACK experiment; kill between generation, upload and publication | +| Existing history | All retained inventory preserved under original retention | Full checksums, resumable manifest; missing/conflicting data blocks cutover | +| Ambiguous PUT/commit | Retry reconciles publication and never duplicates bytes | Fault after object PUT or DB commit but before response | +| Ownership | Stale A cannot publish/finalize after B takes over | Concurrent server and manifest-generation tests | +| Reads | Stable offsets/digests across segments, compression, caches and replacement | UTF-8/boundary/EOF/cancellation; bounded memory and GET counts | +| Storage failure | No unbounded queues or SQL-payload fallback; Stop/API remain usable | Slow/failed S3, queue capacity, recovery and source cleanup hold | +| Traces | Accepted frames survive; completeness and privacy remain truthful | All producer families, channel hashes, disabled capture and denied actors | +| Downstream consumers | Issue previews, feedback, inspection and reprojection keep working | Normal routes and current authorization checks | +| Expiry/delete | Deny after tombstone; clean payloads; no late-upload resurrection | Capture/upload/delete races, restart, actual bucket version behavior | +| Rollback/local mode | Compatible release reads post-cutover writes; local defaults unchanged | Adjacent-version and local-store tests | +| Idle | No recurring SQL/network work after settlement; real work wakes owners | Configured idle, completed history, retry/drain/resume observations | + +## First useful experiment and qualification order + +Use real Postgres, the existing S3 provider implementation against a disposable +S3-compatible target, and two actual service processes with separate temporary +directories. Produce representative output through the production recorder, +including repeated chunks, multibyte text, and tool bursts. Measure the candidate +batch settings and baseline API traffic. Test both published data and the pending +tail: kill A at each boundary and read/replay through B. Never pre-populate the +storage result being tested. Record revision, configuration, receipts, timing, +SQL/WAL and object-operation evidence. + +Use the existing remote-continuity harness with a deterministic provider and +actual Runner to qualify retention independently of control ACKs and responsive +Stop. The decisive question is whether durable batching can coexist with ongoing +provider execution and web replacement. Do not commit to a protocol mechanism +based on a mocked callback or a successful single PUT. + +Before staging, record the chosen publication scheme, batching/concurrency +limits, backlog capacity, supported recovery behavior, and measured cost/latency +operating envelope. If those fail, revise the design before a broad schema or +migration implementation. No production sizing/cost has yet been measured. + +Then verify the actual staging bucket and remote capture path with a bounded +live provider run, normal UI/inspection/download, and post-replacement reads. +Make the old web filesystem inaccessible. Preserve prior failures alongside +successful retries. Full Railway HTTP handoff and volume removal remain parent +plan gates. + +Run narrow storage/output/trace/continuity/auth/migration checks while iterating. +At implementation PR handoff run `pnpm -r typecheck`, `pnpm test:run`, and +`pnpm build`, plus affected Runner checks and user-journey verification. Do not +claim live-provider or fleet evidence from deterministic fixtures. + +## Implementation state and remaining gates + +- Configuration follow-up: shared storage resolution now serves product files, + run logs and traces. S3 instances need no extra switches or connection fields. + Cloud's existing per-stack bucket and credentials remain the isolation + boundary; existing prefixes and company/run subfolders are preserved. The + configuration regression failed before the change, then 70 affected tests + passed, including real S3-compatible bucket/prefix isolation and cleanup tests. + Four existing Cloud environment-contract tests also passed without Cloud code + or deployment changes. +- User follow-up: remove the added run/adapters map sections. Retained-history + conversion now belongs to an automatic, bounded delivery worker; the command + is optional diagnosis/repair. Existing local references remain durable work. + Startup reconstructs it, active or blocked history retries with backoff, and + completed history has no recurring SQL/network scan. Migration stays with its + owning service; index.ts only registers its lifecycle, and the shared delivery + registry adds the queue name. The source volume must survive until all retained + history is verified. Mixed versions must stop creating new local-file runs. +- Trace encryption reuses either existing vault key source, including the + configured key file, and never creates a replacement key for external traces. + Both strict missing-key and file-key round-trip regressions failed before the fix. + The configuration/storage regression run passed 91 tests. Automatic migration + and retention passed 10 tests, including a real Postgres/S3 upgrade followed by + source-directory deletion. Harness failures remain in the local evidence: a + cold import exceeded a test deadline (moved module setup into the fixture), a + sandbox denied localhost access (rerun with network permission), and the first + automatic fixture bucket name exceeded S3's length limit (shortened the name). +- Shutdown review: automatic trace import now propagates cancellation through + source reads, opening, conditional-write qualification, upload and verification. + Five real-SDK stalled-request regressions failed before the fix, then passed. + A shared capability probe survives one cancelled claimant and aborts when its + last claimant leaves. Source and remote trace hashing stream bounded chunks. + A multi-chunk native sidecar without a final newline retains exact download + ordering and digest. Existing file-key and empty-web-home app checks passed at + a142424fc9; subsequent changes affect copy cancellation and verification only. +- Base integration: 879ba0d1fd reached Greptile 5/5 and all CI checks passed, + then master advanced and produced two append-only documentation conflicts. + Preserve both documentation sections and integrate the new Cloud snapshot + startup contract. The startup regression reproduced a redundant config read; + worker registration now accepts only the already-loaded storage-provider value. + All 103 startup controls plus storage/lifecycle controls passed (112 tests), + including saved Cloud profile reuse after the legacy config file is absent. + A further combined run passed all 124 storage and real-Postgres Cloud runtime + configuration controls on the merged source. +- Further review: trace retention now registers independently of heartbeat + scheduling, and optional issue-comment attribution reads the published log + prefix after a failed best-effort native repair. Both defects reproduced red; + all 116 selected startup, retention, attribution and cancellation controls pass. + The 19f702d780 browser session-recovery check timed out; its job log is retained. + No UI behavior or browser assertion was changed to suppress that failure. +- Branch: `codex/external-run-logs-traces`; this chat owns integration and PR review. +- S3 holds payloads from the first committed batch. Conditional object heads + hold cursors and writer generations. Existing Postgres rows hold lifecycle and + references only. No schema change, per-record SQL, or DB payload compactor. +- Implemented: stored-backend reader dispatch, bounded batching and I/O, + immutable binary range indexes, exact digest/range reads, ownership fencing, + encrypted optional trace channels, durable deletion/expiry obligations, and a + bounded resumable history importer. Local defaults remain unchanged. +- Exercised locally with real Postgres and Moto 5.2.5 through the real AWS SDK: + encrypted trace replacement/deletion and history import with original source + directories removed. Missing, corrupt, and active history blocks migration. +- Abrupt process qualification: a writer commits 1,000 records, receives storage + receipts, dies by `SIGKILL`, and loses its working directory. A separate + process reads every record exactly and continues the same journal. +- App journey: create a deterministic process agent, click Run now, open its + run history, kill the web process, restart against the same external services, + and reload the viewer. All 31 NDJSON records remained byte-identical through + the normal authorized API (6,206 bytes; SHA-256 + `3e8415c79379c583b1c4c37902187593fd69ebe85db78d6486d6077f205bacba`). +- Local workload qualification reports burst and eight-stream steady request, + transfer, stored-object, and timing counts. It is a synthetic envelope, + not a fleet cost forecast. The first index prototype left obsolete COW + objects; the selected immutable binary index removes that normal growth. +- Full workspace typecheck and build passed after native replay and the storage + review fixes. The pushed server revision was rebuilt for the final app check. + The native Postgres package initially lacked its postinstall-created library + symlinks; its standard hydration helper repaired that local test prerequisite. +- Native pending-tail qualification passed with the real Rust recovery fixture, + `PaperclipControlPlanePort`, production recorder, real Postgres and Moto. A + received ACK released 32 records from the Runner while S3's source cursor was + still zero. After `SIGKILL` and deletion of the old controller directory, B + exported all 64 records once while the same Runner PID continued. Snapshot and + Stop commands completed in 453 ms and 122 ms. The first attempt failed because + its new controller directory had mode 0755; the fixture now creates mode 0700. + Both failed and successful evidence remain in this chat's temporary artifacts. + This is a deterministic protocol experiment, not a live model or Railway test. +- Final app verification used a new web home on `c2cedcd116`. Retained history + remained byte-identical (31 records, 6,206 bytes), and a new UI-triggered process + run succeeded with every generated record present in its S3-backed transcript. + The browser initially reached the listener before startup completed; after + readiness, the normal page and API checks passed. +- Durable log receipts and live log messages follow S3 publication. The native + export reuses existing control rows, adds no payload writes, and repairs missing + callback ranges. Interrupted settled archives can be repaired on normal reads + after the old native lease expires. Optional raw capture and server-only + diagnostics without a surviving source can remain incomplete after a crash. +- Review fixes cover absent-head deletion fencing, shared capture ownership, + abort/drain on deletion, local expiry deadlines and multi-page cleanup, bounded + transient retries and admission backpressure, partial-startup cleanup, and + explicit trace-import cleanup markers. A source integrity failure leaves the + native archive's final size unknown instead of certifying a partial result. +- Full local Vitest could not run on host embedded Postgres because concurrent + clusters exhausted macOS SysV semaphores. Affected suites are running against + isolated Docker databases. One local full typecheck attempt also lacked sandbox + permission for tsx IPC; the authorized rerun uses the normal workspace command. + The isolated Docker run then passed 1,014 affected storage/native tests and + all 27 privacy-route tests. Subsequent targeted review regressions passed for + native source classification, native import coverage, shared-reader repair + cancellation, serving the published prefix on repair conflicts, and a failed + local-file cleanup retaining its deadline. Live and recovered native output + redact existing agent identity material; runtime log secrets are registered + once through the existing encrypted redaction registry, without copying agent + private keys into run contexts. +- Native history migration accepts finalized files whose recorded size and + digest match, preserving the existing formatter's truncated large display + chunks. Files missing either completeness field still require ordered source + coverage. Corrupt metadata blocks publication; the conditional switch checks + that the original completeness fields have not changed. +- PR: https://github.com/paperclipai/paperclip/pull/18327. Initial CI passed 51 + checks with one E2E teardown lock timeout. That E2E shard passed on the next + revision; its Runner shard then failed the Antigravity authentication fixture. + These failures are retained, and fresh-head CI remains required. +- Production remains disabled. Railway Buckets are the parent fleet's current + S3 provider; their conditional-write, version/deletion, workload cost, remote + capture, and sleep/wake behavior need actual-target qualification. No fleet + environment or volume was changed. +- Next gate: finish fresh-source checks and PR review; retain actual-provider, + live remote capture, and parent cutover gates before any volume-free rollout. +- No Paperclip issue/run context was supplied. This repository plan is the state + record; no issue work product or production migration is claimed. diff --git a/doc/recovery-file-cutover.md b/doc/recovery-file-cutover.md new file mode 100644 index 00000000000..225a81da1bd --- /dev/null +++ b/doc/recovery-file-cutover.md @@ -0,0 +1,112 @@ +# Check existing recovery files before web-server replacement + +This check covers only observed legacy files on one explicitly selected instance. +There is no verified production backlog. An empty installation needs no migration. +This check does not inspect Postgres obligations, Runner delivery, other volumes, +other instances, or the rest of web-server replacement readiness. + +Use the retained volume and the instance root from the old server configuration. +The instance root is the directory that contains `accounting-receipts/` and +`quota-credential-recovery/`. It is not the parent directory of all instances. +Use canonical absolute paths. The check rejects symlink components, including +system aliases such as `/var` on macOS; use the actual `/private/var` path there. +The chosen instance directory must exist. Missing recovery directories are normal. + +Run this command from a checkout that includes the check. Node is the only runtime +dependency. It needs no database connection, master key, or package installation. + +```sh +node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance +``` + +The default command reports JSON and exits successfully even when work remains. +For an automation gate, add `--check`: + +```sh +node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance --check +``` + +| Exit code | Meaning | +| --- | --- | +| `0` | Report produced; with `--check`, all three scans are clear | +| `1` | With `--check`, unresolved files or an unknown/incomplete scan block cutover | +| `2` | Invalid arguments or the inventory could not run | + +The sandbox directory defaults to `data/sandbox-orphan-cleanup/` below the chosen +instance. If the old server uses `SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR`, supply that same +environment value when running the check. An explicit `--orphan-directory` takes +precedence. A relative or empty override is rejected; supply its canonical absolute +path for this check. An override is an additional explicitly selected root and can +be outside the instance directory. The check does not search for other overrides. + +```sh +node scripts/recovery-cutover.mjs --instance-root /absolute/retained/instance \ + --orphan-directory /absolute/retained/orphans --check +``` + +The report gives one summary for each of the three recovery kinds. It never +prints filenames, input paths, file contents, identifiers, token values, decrypted +material, or raw filesystem errors. `unverifiedFiles` includes every regular file, +including JSON, temporary, probe, truncated, corrupt, and unreadable files. The +check does not read file bytes and cannot claim that a file is valid or settled. +That decision belongs to the existing recovery owner and its format definitions. + +Only the credential directory's immediate UUID company directories are traversed; +empty company directories are normal after recovery. Accounting and orphan files +use flat directories. Unexpected directories, deeper nesting, symlinks, and +special entries count as `unknownEntries` and block. The check does not follow +symlinks or inspect unexpected subtrees. Directory/stat/close failures increment +`readFailures` or leave the scan incomplete. An invalid or missing instance root +also blocks; it is not an empty installation. + +The default shared budget is 10,000 observed entries across all three scans, +including company directories. `--max-entries` accepts 1 through 100,000. At the +limit the scanner reads at most one extra entry per open directory to detect +truncation. An unfinished scan reports `complete: false` and status `unknown`. +Counts are observed lower bounds, not total records or total pending operations. +A complete scan with any files reports `blocked`; any uncertainty reports +`unknown`. Only a complete empty scan reports `clear`. + +For the final cutover gate, stop the old producers first and retain the old volume. +Run the check after the relevant recovery owners finish their work and producers +are stopped again. A live scan is a point-in-time inventory, not an atomic drain +certificate. Concurrent renames or writes invalidate the snapshot; the tool does +not lock writers or protect against a hostile process that changes paths during +inspection. Do not use it to certify a live writable or untrusted volume. + +If files exist, keep the old volume and let the existing owners settle actual +entries with their existing permissions and database checks: + +- Accounting: `run-cost-accounting.ts` uses the `usage-receipts.ts` replay and + per-run recovery paths. The receipt owner validates the accounting envelope and + handles database conflicts and settlement. +- Credentials: `ai-connections.ts` and `ai-connection-runtime.ts` use + `quota-credential-recovery.ts` during account credential access and refresh. + The owner validates encrypted material with the original instance master key + and preserves grant, company and credential version fences. +- Orphan cleanup: `environment-runtime.ts` exposes `flushDeferredOrphanCleanups` + to the existing cleanup lifecycle. `sandbox-orphan-cleanup-spool.ts` owns the + record format and removes a file only after the recovery path saves ownership. + +These are service entry points, not standalone operator commands. This change +adds no import, deletion, replay, decryption, or bulk migration command. Do not +trigger new paid runs or token exchanges merely to empty the report. If an owner +cannot settle a record, preserve it and record an explicit reconciliation outcome +through the existing incident/operations process. Corrupt files need owner review; +they do not disappear from the gate because a recovery reader skipped them. +Never delete unresolved files to obtain a clear result. An approved exception +requires separate evidence; this tool continues to block while the files remain. +Retain the volume and required secrets through rollback compatibility review. + +Verify the check without any real instance data: + +```sh +node --test scripts/recovery-cutover.test.mjs +``` + +The tests use disposable synthetic directories and spawn the actual CLI. They +cover pending legacy layouts, malformed bytes, temporary files, empty directories, +symlinks, overrides, scan limits, safe diagnostics, failure injection, exit codes, +and preservation of source bytes. They establish file inventory behavior only. +They do not establish that any production volume is clear or that domain recovery +has completed. diff --git a/doc/run-evidence-storage.md b/doc/run-evidence-storage.md new file mode 100644 index 00000000000..64a69221d81 --- /dev/null +++ b/doc/run-evidence-storage.md @@ -0,0 +1,276 @@ +# Run log and provider trace object storage + +Instances configured for S3-compatible file storage automatically use the +`s3_segments` backend for new run logs and opted-in provider traces. Postgres +retains the existing run/trace references and lifecycle rows. +It adds no payload table, per-record SQL, or database-to-object compactor. +Structured `heartbeat_run_events` retain their existing database path. Native +events already committed there before a Runner ACK also provide the recovery +source for an interrupted canonical native log export. No additional event copy +or payload write is introduced. +When native object logging uses runtime observability credentials, their log +redaction values are registered once in the existing encrypted run-secret +registry. This is bounded redaction metadata, not a transcript copy or a +per-event write. Agent identity private material stays in its existing key store. + +## Configuration + +No additional storage configuration is required. The existing instance +`storage.provider` setting, or `PAPERCLIP_STORAGE_PROVIDER`, selects storage. +The same config-file S3 fields and `PAPERCLIP_STORAGE_S3_*` overrides select the +bucket, endpoint, region, prefix, and path style used by product files. Credentials +use the same AWS SDK chain. Local-disk instances keep `local_file` logs and traces. + +The instance's existing S3 prefix is preserved. The folders underneath it are: + +```text +/run-logs/segments-v1//// +/provider-traces/segments-v1///// +//... existing product files +``` + +Cloud's existing provisioning supplies a separate bucket and bucket-scoped +credentials for each stack, plus the stack's encryption key. The app reuses that +boundary rather than selecting a global customer bucket. Within a stack, journal +keys remain company/run scoped. Cleanup lists only the resource's own prefix; +it cannot sweep other companies, other stacks' prefixes, or product files. +Folders are organization, not a substitute for bucket permissions or API access +checks. Shared-bucket self-hosted deployments must retain their existing distinct +instance prefixes and appropriate storage credentials. + +Legacy run rows retain their recorded backend until verified migration. Their +pre-existing `RUN_LOG_S3_*` mirror settings and key paths remain supported for +legacy reads and writes. They do not override instance storage for new journals. +Use stable bucket/prefix settings across replacements and releases. +As with existing product files, changing the instance bucket or prefix is a +separate storage move and requires copying the complete namespace and preserving +its keys before changing configuration. Switching to local storage does not move +existing S3 evidence. Readers reject an unavailable external store; they never +substitute an empty local log. Restore the original stack's storage profile to +read its history. This upgrade migrates legacy local evidence into the existing +stack store; it does not introduce bucket relocation or reverse migration. + +Raw capture still requires its existing explicit capture request. Its encryption +uses the existing vault key, from `PAPERCLIP_SECRETS_MASTER_KEY` or the existing +configured key file. Cloud already supplies the environment key. No separate +trace key is introduced, and a missing vault key fails rather than creating a +replacement. Disposable deployments must preserve the existing vault key along +with their other secrets. +Raw payload and index objects +use AES-GCM with a key derived for the particular company/run/trace/channel. +The bounded head holds only ownership, range, digest, and publication metadata. +Existing capture opt-in, redaction, instance-admin access, task/company access, +reveal/download auditing, and the 24-hour capture expiry remain in effect. + +The provider must enforce `If-None-Match: *` and ETag `If-Match`, return ETags, +and support prefix-scoped paginated listing. Claiming the first journal performs +a conditional-write capability probe. A backend which ignores preconditions is +rejected. Before deploying this release to an S3-configured production instance, +qualify the provider's conditional writes, range/read consistency, encryption, +object versions, and cleanup. AWS API compatibility alone is insufficient evidence. + +## Publication and operating limits + +Each batch uploads immutable payload segments and merges equal-height index +roots into immutable binary nodes, then conditionally publishes one head object. +The head carries at most 53 ordered roots. Every normally published payload and +index node remains reachable; appends do not leave superseded index copies. +Interrupted or fenced uploads can leave unpublished objects, which remain inside +the resource prefix for scoped deletion. The head is the commit +boundary. A stale writer can upload an orphan but cannot replace the current +owner's published history. An uncertain response is reconciled against the exact +head before returning a durability receipt. Committed reads use immutable roots, +so publication cannot change the bytes behind an already selected snapshot. + +An individual store `append` resolves only after publication. Heartbeat's +`onLog` diagnostic callback waits for bounded admission without serially +waiting for each network batch. At capacity it waits for space; admission beyond +the bounded waiting allowance is rejected without poisoning accepted batches. +Transient network, 409, 429 and server errors receive bounded retries. A 412 +or a changed head fences the stale writer; a 409 alone does not prove ownership +loss or conditional-write support. Live log messages and progress are published +after storage confirmation. Finalization drains admitted writes before closing +and hashing the stream. Logging failure reports incomplete persistence and +leaves the final size/digest unknown rather than claiming a complete snapshot. +The original progress projection remains coarsely throttled; concurrent batch +completions share a single progress write. + +Defaults and fixed safeguards: + +| Limit | Value | +| --- | --- | +| Dirty batch deadline | 1 second, configurable from 1 ms to 30 seconds | +| Segment target | 1 MiB | +| Pending bytes per writer | 8 MiB | +| Pending run-log bytes per process | 64 MiB | +| Additional waiting admission | 8 MiB / 32 calls per process | +| Concurrent run-log flushes | 4 | +| Concurrent journal object operations | 8, with 32 queued operations | +| Object operation/batch cancellation deadline | 15 seconds | +| Current index fanout / maximum roots | 2 children / 53 roots | +| Journal head read bound | 64 KiB | +| Trace encoded payload bound per sidecar | 192 MiB, in addition to the existing raw capture limit | + +A configured but empty log store sends no periodic request. Dirty buffers, +capture file-change events, and actual retention/retry obligations drive work. +Use request/transfer counts, byte volume, steady versus bursty producers, +concurrent runs, and control/API latency to size a deployment. The local S3 +qualification is a synthetic workload, not a production cost forecast. + +Finalization streams the committed bytes once to compute the existing complete +file SHA-256. It does not re-upload the entire log. Range indexes avoid loading +all segment descriptors to read a tail. Historical consumers, UTF-8/NDJSON record +alignment, issue previews, feedback, and customer-success inspection dispatch +through the persisted backend. Logical reads retain the existing authorization. +On-demand native repair closes its verified committed range with an exact byte +count and leaves an unavailable full-file SHA-256 null. It does not make a log +page request re-download the entire archive to compute that optional digest. + +## Durability boundary + +Committed log receipts and captured object prefixes survive web disk loss. +Output still buffered only by the current web process has no durable receipt. +The live log path waits for publication before showing that output. Raw sidecar +capture remains optional; a crash can leave an incomplete uncommitted tail, and +only durably closed channels plus valid terminal markers qualify as complete. + +Native PRP events have a surviving source: the control port already commits +validated, redacted events before acknowledging them. The S3 head checkpoints +the highest exported database event cursor with the batch's bytes. A replacement +replays later native source rows in bounded 16-row pages, checks their identity +and canonical hash, and exports the same `paperclip.prp.event` representation +through normal log redaction/compaction. Late callbacks and duplicate delivery +cannot append the same source record again. This is a canonical native export; +generic stdout, stderr, and raw provider traces are not reconstructed from event +rows. The control ACK and sparse checkpoint formats remain unchanged. + +Recovery runs before native execution reconnects. If the run settled before its +archive finished, the normal log API, issue preview, or feedback reader can repair +it after the native controller lease expires. Concurrent requests share recovery; +final metadata is saved only when no later native source remains. Read-only +customer-success inspection reads the currently published prefix. A failed +repair leaves final size/digest unknown and preserves the existing source. +Retained native events are not deleted by Runner cleanup. Recovery adds bounded +reads and one final metadata update, with no new per-record SQL on the normal +contiguous output path. Raw optional capture can still be incomplete after a crash. +Credential-update, retained-cleanup, and trace-reprojection records have separate +source namespaces and do not enter this log export. Corrupt canonical records +fail integrity validation. Shared on-demand repair has a 30-second lifetime; +cancellation removes a waiting reader and aborts storage work when no readers +remain. It preserves already published cursors so a later request can resume. +Live export and recovery both redact the existing agent identity and known run +secrets before publication. Recovery reads existing identity material without +provisioning a key. If best-effort repair loses ownership or fails, ordinary log +reads serve the published prefix and leave completion metadata unknown; request +cancellation still propagates. + +The [remote continuity](testing-native-remote-continuity.md) protocol and the +parent deployment's other filesystem and HTTP handoff requirements still apply. + +## Trace deletion, expiry, and sleep + +Deletion/expiry denies reads through a database tombstone before payload cleanup. +Object journal tombstones prevent further publication. If an active writer may +have an in-flight upload, cleanup waits at least 30 seconds and remains a durable +retry obligation. A successful manual DELETE requires payload cleanup; an +unsettled cleanup returns a retryable failure, and the shared retention worker +retries it. Tombstones remain after object removal. Historical backups and +object-version retention need a separate operator policy; an ordinary DELETE +must not be mistaken for erasure of historical versions. + +The shared coordinator restores expiry/cleanup work on startup and registers +actual deadlines for local and external captures. New local inserts also wake it; +expired backlogs drain in bounded pages. No completed-history polling is added. Trace expiry +is not yet part of Cloud's durable wake contract, so outstanding trace deadlines +conservatively block instance sleep. Removing that blocker requires qualification +of the corresponding external wake, not a longer polling interval. + +Trace expiry and failed-deletion recovery remain registered when +`HEARTBEAT_SCHEDULER_ENABLED=false`. Disabling agent scheduling does not disable +retained-data cleanup. The worker still respects warm standby, admission drain, +actual deadlines, and shutdown. + +## History migration and rollback + +S3-configured servers automatically migrate retained local history in the background. +No command or extra configuration is required. The existing delivery coordinator +starts the upgrade outside request handling, independently of the heartbeat +scheduler toggle. Each batch processes at most one log and one trace, then yields +for one second. Existing references are the durable work inventory. Restarting +reconstructs unfinished work from those references; already migrated rows are +excluded. No new payload or job table is added. + +Active writers, missing files, corrupt data, and storage failures leave history +unresolved. Other pages continue. The server logs each blocked resource and a +pass summary. Unresolved passes retry after five minutes, with backoff up to +30 minutes. Transport failures retry through the shared coordinator. After all +retained history is migrated, the worker stops querying and contacting storage. +Trace source hashing and remote verification stream bounded chunks instead of +loading a complete capture into server memory. The worker cancellation signal +reaches file reads, trace opening, conditional probing, uploads and verification. +Local-disk instances do not register this worker. Outstanding migration conservatively +blocks instance sleep, and shutdown aborts the batch and waits for owned work. + +Deploy compatible readers and stop old binaries from starting new local runs. +The upgrade waits for existing writers to settle and their leases to expire. +Keep the source volume attached until migration completes and clean-filesystem +reads pass. Automatic copying cannot recover a file after its only copy is lost. + +`server/scripts/migrate-run-evidence.ts` remains an optional diagnostic and repair +tool. Its default is a read-only inventory; `--apply` runs the same verified +importer used by the automatic worker. `--company`, `--after-run`, `--after-trace` +and `--limit` allow scoped investigation against an explicitly selected +`DATABASE_URL`. Normal upgrades do not require this tool. It does not stop agents, +select a production target, delete source files, or certify volume removal. + +For logs, migration verifies exact local bytes or a mirror with proven length +and digest, compares any previously uploaded prefix, verifies remote SHA-256, +then conditionally changes the run's backend/reference. Unknown-completeness +mirrors, missing/corrupt payloads, and active writers block publication. Existing +trace bytes, download ordering, integrity state and original expiry are retained. +Neither path fabricates missing historical data or extends trace retention. +For finalized native logs, matching the recorded size and SHA-256 proves the +complete legacy file, including display chunks truncated by the existing 64 KiB +formatter. Migration preserves those exact bytes and records the settled source +cursor. If either completeness field is absent, a bounded NDJSON scan must prove +ordered coverage of the existing canonical source records. Missing ranges or +chunks whose identities cannot be verified block publication as +`native_log_source_coverage_unproven`. A database's latest event cursor alone +does not establish completeness. Publication checks that the original reference, +completeness metadata, and source cursor have not changed. Keep the original +volume and resolve blocked inventory before cutover. +An interrupted trace import keeps an `s3-import-v1:` reference: reads still use +the original local sidecars, while deletion and expiry retain responsibility for +the staged object prefix. The automatic worker resumes the verified switch after restart or retry. +Ordinary local traces have no S3 cleanup dependency. + +Keep the volume until every retained item has been verified from a clean web +filesystem. Rollback must use a compatible reader against the same remote state. +An older binary or stale local volume is not a valid rollback target after new +external writes. Enforce that floor in the owning deployment procedure. + +## Qualification + +Run the storage, recorder, migration, privacy and retention tests. The opt-in +S3/Postgres integration fixtures use `PAPERCLIP_LOG_TEST_S3_ENDPOINT` and a dedicated +`PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL` ending in `_test`. They create +unique test buckets and schemas, and remove only those resources. The privacy +route suite also accepts `PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL` for a +dedicated, fully migrated database ending in `_test`; it clears its fixtures. + +`server/scripts/qualify-object-run-logs.ts` exercises the actual AWS SDK against +a loopback test backend, confirms exact replacement reads and stale-writer +rejection, and reports requests/bytes/timing. It uses synthetic credentials and +refuses a non-loopback target. This proves its tested transport and workload; +production-provider qualification and actual Railway deployment remain separate. + +`server/scripts/qualify-native-log-recovery.ts` uses the real Rust +`recovery-fixture` example, real Postgres, the control port, production recorder, +and S3 backend. It kills the controller after Runner ACKs but before the S3 +cursor catches up, deletes that controller's directory, and restores sparse +control state into a fresh controller. It checks exact source order, no duplicate +records, the same surviving Runner PID, and control/Stop responsiveness. Build +the example with `cargo build --locked --manifest-path packages/paperclip-runner/runner/Cargo.toml -p paperclip-runner-core --example recovery-fixture`. +It accepts the same two explicit loopback test endpoints and creates a private +schema and bucket. This deterministic protocol experiment does not qualify a +live provider, remote execution environment, or production S3 vendor. diff --git a/doc/run-log-events.md b/doc/run-log-events.md index f3937486b9c..e2bf5455176 100644 --- a/doc/run-log-events.md +++ b/doc/run-log-events.md @@ -694,3 +694,13 @@ telemetry-read authorization. Ordinary PRP reads/live delivery retain run-secret and current-user redaction. The task page uses one history read on mount/reconnect and the existing live transcript stream, then makes a bounded plugin readiness check. No panel discovery timer scans runs or provider state while connected. + +## Payload storage + +Structured events in this guide remain in `heartbeat_run_events`. When the +instance's existing file storage uses S3, new stdout/stderr NDJSON logs use the +same stack bucket and configured prefix under `run-logs/`, with existing run +references in Postgres. Local-disk instances remain local. Raw provider +traces have their own restricted, encrypted object path. See +[run evidence storage](run-evidence-storage.md) for batching, durability, +configuration, migration, retention, and qualification boundaries. diff --git a/doc/testing-native-remote-continuity.md b/doc/testing-native-remote-continuity.md index 60d821e9f0f..91887a1b71b 100644 --- a/doc/testing-native-remote-continuity.md +++ b/doc/testing-native-remote-continuity.md @@ -172,10 +172,15 @@ disk outbox. No second remote journal or new polling loop is introduced. The live event window keeps the latest identity, turn, result and usage anchors, as well as unsettled semantic inputs, through long output streams. The 4,096-event and 500-command limits remain. This checkpoint is intentionally not a complete -transcript: an abrupt crash can lose display output already acknowledged but not -yet projected to the existing run log. The same provider turn continues; recovery -never reruns work to reconstruct display text. Tool outcomes and terminal state -retain their durable barriers. +transcript. With local-only logs, an abrupt crash can lose a display projection +already acknowledged by the controller. The opt-in `s3_segments` log backend +checkpoints its canonical native export cursor in S3 and replays missing records +from the existing validated `heartbeat_run_events` source on takeover. This adds +no payload writes or per-record checkpoint transactions. Generic process output +and optional raw provider sidecars have separate durability boundaries. See +[run evidence storage](run-evidence-storage.md). The same provider turn continues; +recovery never reruns provider work to reconstruct display text. Tool outcomes +and terminal state retain their durable barriers. Rollback must use a controller that understands sparse checkpoints and the signed recovery cursor. A v2 decoder alone is insufficient. Runners admitted before this @@ -278,3 +283,121 @@ This is protocol and persistence evidence. The separate `everyday-workflows.runner-codex.daytona.recover-controller` Product E2E cell qualifies the actual Daytona/provider path, including completion and workspace finalization. Neither test substitutes for the other's failure boundary. + +## Required native usage acceptance + +Built-in native remote Runner execution accepts structured accounting before +its PRP transport ACK. `usage.reported`, turn-start, and turn-terminal facts +produce immutable `run_usage_receipts` rows. The receipt stores only structured +usage and turn identity. It does not depend on output logs or raw trace capture. +The source UUID derives from the run and Runner identity. Provider-turn IDs and +turn-start source facts preserve aggregation when several provider turns share +one PRP envelope turn. Source event IDs and +sequences remain stable across receiver replacement. Repeated delivery checks +the saved fact and does not create another receipt for the same report. + +A failed receipt transaction rejects acceptance. The remote Runner keeps its +outbox report and reconnects through its existing protocol lifecycle. A lost +commit response can replay the same immutable receipt. A replacement receiver +restores accepted per-turn aggregation from Postgres, then accepts pending +reports from the same producer. The existing source fence prevents a retired +producer from replacing the current source's usage. Missing or partial usage +keeps accounting pending; it does not produce a zero-cost charge. + +The native session authority epoch supplies the current run's accounting owner +to reused warm transports. The usual controller checkpoint and generation checks +still control transport acknowledgement and attachment. Ledger settlement, +pricing, reservations, and budget enforcement retain their existing owners. +No accounting receipt directory is created for this supported remote path. + +Local native execution, legacy adapters, and external Dot/Muse execution retain +their existing recorder behavior. This change does not establish disposable-web +accounting durability for those paths. Server-originated model calls keep their +existing admission and settlement records. No S3 accounting spool is introduced. +The independent run-log/trace storage change can consume its own persisted +sources; output acceptance is not accounting acceptance. + +Run the focused PostgreSQL/Runner experiment with: + +```sh +pnpm exec vitest run server/src/services/native-runtime/native-usage-delivery.integration.test.ts --maxWorkers=1 +``` + +The provider fixture emits a real PRP usage report through the durable Rust +Runner. A PostgreSQL trigger fails receipt insertion. The test removes the first +receiver directory, replaces its accounting owner, reconnects the same Runner, +and independently reads the receipt and ledger. The provider is controlled; +no paid provider request is made. Additional cases cover lost commit responses, +replay conflicts, per-turn aggregation, partial usage, source replacement, and +budget holds. Optional log and trace callbacks are absent from the experiment. + +The cold fixture build runs before opening PostgreSQL, with the existing native +integration build budget. PR CI routes this suite through the existing cached +native Runner lane. CI builds the release fixture; local execution uses debug. +The behavior test has its own bounded receiver/process deadline after compilation. + +### Codex usage authority and release order + +The qualified Codex producer keeps its current-run baseline and latest session +totals in its existing durable provider state. It observes historical resume +snapshots, preserves that baseline across process recovery, and resets the +baseline to the latest totals when attaching the next run. Multiple request +reports therefore emit the whole run delta, rather than the provider's last +request. Regressing or incomplete live counters cannot certify completion. + +Authoritative Codex reports carry `runDeltaBasis: "run"`. This requires a Runner +artifact built from the provider baseline implementation in this change. A +version string alone does not establish support; use the qualified artifact +digest. Existing unmarked Codex reports remain incomplete and unpriced in the +new accounting path. Their budget debt remains pending for reconciliation. +No deployed Runner artifact has been qualified by this source-only change. + +PRP negotiates envelope versions and the admitted Runner digest. It does not +negotiate this usage payload field. The existing event envelope validators leave +usage payloads open, but an old standalone usage-schema consumer rejects this +new field. Update receivers and strict usage-schema consumers first, then +release the matching Runner artifact before claiming this durability guarantee. +Finish or explicitly reconcile older admitted Codex runs during cutover. Do not +claim that an old producer's final-request delta is a whole-run receipt. + +Heartbeat completion reads the current immutable accepted receipt, including its +saved exact rate-card amount and provenance. It does not reprice with current +rates or replace that accepted estimate with an absent provider dollar amount. +The terminal UPDATE preserves the current producer-owned accounting projection +atomically. If another report arrived after completion read its checkpoint, the +newer receipt, readiness, price, and budget exposure win; unrelated session and +completion metadata can still update. A newer partial receipt stays pending. +Non-accounting attachment control events keep their existing handoff acceptance; +an old run's accounting report still cannot acquire a successor's authority. + +The provider drains its bounded settled-turn usage tail before publishing the +terminal event that closes the run. An authoritative active goal keeps its +existing continuation path unless an uncorrected completion rejection closes +the run. The +same owner predicate selects both final usage drain and terminal outcome. A +completed turn remains authoritative after a +fully drained provider exit; ordinary session-exit handling still records an +unavailable process independently. This keeps a late cumulative increase on the old run before the +next run resets its baseline. Readiness and attachment retain accounting facts +in the existing provider checkpoint instead of discarding them, including facts +consumed before a later quiescence error. An increase +that arrives after authoritative completion requires explicit reconciliation: +PostgreSQL rejects changed finalized native usage or authoritative monetary evidence, so the existing Runner +outbox retains it under the old source identity without another charge. Equal +snapshots are harmless. No recurring scan or automatic adjustment is added. + +If delayed usage arrives after attachment and cannot belong to the current +provider turn, the provider retains its cumulative evidence and the matching +saved prior identity, leaves the successor baseline unchanged, and emits a +failed run terminal. The existing reconciliation lifecycle blocks further work, +including after restart. Unknown older ownership stays explicit rather than +being inferred from provider-supplied run identity. Manual reconciliation or a +fresh session is required for this rare unsupported timing case. + +Prior-turn increases are checked against the saved attachment baseline, even +when a successor has already reported a higher session total. Required usage +bypasses the optional diagnostic limit. An affected successor publishes an +explicit incomplete usage report before failure; this preserves known budget +exposure without settling ambiguous usage. That required report and its receipt +retain the accounting measurements and matching saved prior identity, so an +optional diagnostic acknowledgement cannot discard reconciliation evidence. diff --git a/docs/deploy/storage.md b/docs/deploy/storage.md index 9c0ff5b804f..c65e025b9f9 100644 --- a/docs/deploy/storage.md +++ b/docs/deploy/storage.md @@ -4,6 +4,9 @@ summary: Local disk vs S3-compatible storage --- Paperclip stores uploaded files (issue attachments, images) using a configurable storage provider. +New run logs and explicitly requested provider traces reuse the same settings. +S3 instances store these under `run-logs/` and `provider-traces/` within the +existing prefix. Local-disk instances keep them local. ## Local Disk (Default) diff --git a/maps/agent-access-map/agent-chat.md b/maps/agent-access-map/agent-chat.md index 64fe049f155..fe50d30deff 100644 --- a/maps/agent-access-map/agent-chat.md +++ b/maps/agent-access-map/agent-chat.md @@ -46,7 +46,9 @@ Review: `baseline`. API: `present` — representative operation IDs: -- `GET /api/issues/{id}/comments`. +- `GET /api/issues/{id}/recent-comments` (20 contiguous comments for a standing chat). +- `GET /api/issues/{id}/comments` (cursor pages for older history). +- `GET /api/issues/{id}/runs?runIds=...` (at most 100 requested IDs, constrained to the accessible issue and company). - `POST /api/issues/{id}/comments`. - `GET /api/issues/{id}/decision-receipts` (own standing-chat board read). - `GET /api/issues/{id}/chat-handoffs` (own standing-chat board read, paginated). diff --git a/maps/feature-map/agent-chat.md b/maps/feature-map/agent-chat.md index 506fec74016..916eadf31ff 100644 --- a/maps/feature-map/agent-chat.md +++ b/maps/feature-map/agent-chat.md @@ -27,6 +27,8 @@ Open `/chats`, choose an agent, or use `/chats/:agentRef` and its conversation s With Coordinator enabled, the composer exposes the shared model/thinking-level picker for supported agents. The agent stays fixed in this control; switching agents remains in the chat header. Choices save as conversation overrides on send, preserving the agent's configuration. A first message creates the conversation before saving settings and submitting through the ordered comments endpoint. +Standing chats open with the latest 20 contiguous messages. Scroll upward to fetch earlier messages, answer receipts, and created tasks. The loading indicator stays in the thread; history failures offer an inline Retry while preserving the draft and reading position. Opening a chat does not fetch its lifetime comment, activity, or run history. Already observed messages survive live refresh and Back within the query-cache session. + Coordinator answer receipts from this conversation jump to the answer bubble without leaving chat. Answer updates from an execution task link to that source task and its decision context. Compact document links read “Open [document title] in the sidebar”. ### `project-handoff` @@ -43,6 +45,8 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Enab ### `agent-conversation` +Local browser recipe: [long-chat startup and scrolling](../../tests/perf/agent-chat/agent-chat-startup.spec.ts), with fixture and command details in the [performance README](../../tests/perf/agent-chat/README.md). It covers recent-only reveal, delayed and failed pages, Back, older comment links, and live refresh. Synthetic local data is not deployed acceptance evidence. + Automated: [chat landing](../../ui/src/pages/AgentChats.test.tsx) checks recent-chat routing; [conversation service](../../server/src/__tests__/agent-conversations.test.ts) covers persisted conversation behavior. Automated: [composer picker](../../ui/src/components/task-chat/ComposerRunSettingsPicker.test.tsx) covers fixed-agent model/effort selection on desktop and mobile; [composer](../../ui/src/components/task-chat/TaskChatComposer.test.tsx) covers Coordinator gating and settings submission without reassignment. [Task detail](../../ui/src/pages/IssueDetail.test.tsx) verifies canonical first/existing-chat IDs, settings-before-comment ordering, and no comment submission when settings fail to save. These tests mock the API and do not prove provider execution with the chosen model. diff --git a/maps/feature-map/projects.md b/maps/feature-map/projects.md index b793d57d678..382c3992682 100644 --- a/maps/feature-map/projects.md +++ b/maps/feature-map/projects.md @@ -44,6 +44,14 @@ Preconditions: follow the [baseline](./README.md#before-driving-a-journey). Use Automated: [projects UI](../../ui/src/pages/Projects.test.tsx) and [project detail](../../ui/src/pages/ProjectDetail.test.tsx) are component tests. +Automated: the explicit [`project-task-context` deployment journey](../../tests/runner-e2e/product-journeys/project-context-flow.ts) +creates a project and a self-assigned task through the production dialogs. It +reopens both, verifies the project task list and task properties after reload, +and independently checks the saved company, project, prompt, assignment and +standard work mode with zero agent runs. [Oracle tests](../../tests/runner-e2e/product-journeys/project-context-evidence.test.ts) +reject lost links and mismatched records. This describes coverage; each target +still needs its own live pass before daily promotion. + Manual: Create a project, add a task from its view, and reload. Verify the task belongs to that project, list counts are plausible, and links preserve company scope. Archive only the disposable project and inspect list/task behavior. ### `repository-settings` diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index f9a92a8811d..cd3dfef556f 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -348,7 +348,7 @@ revised-source live onboarding and follow-up proof. `/company/settings/connections` configures decision/fast-response models, not the general model-account library. -- The run log is instance database data; it is not the opt-out first-party Telemetry system or operator-configured OpenTelemetry tracing. +- Structured run events stay in the instance database. Optional stdout/stderr and raw trace payload storage can use S3; see [run evidence storage](../../doc/run-evidence-storage.md). Both are separate from first-party Telemetry and operator OpenTelemetry export. - Authentication/model discovery does not establish tool, image, steering, or resumption capability parity. - Use [recovery](./recovery.md) for interrupted runs and [execution environments](./execution-environments.md) for local/SSH/sandbox differences. diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index 6e2bd463f70..6082b67c456 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -140,7 +140,7 @@ { "path": "schemas/usage.schema.json", "id": "https://paperclip.dev/schemas/prp/v1/usage.schema.json", - "sha256": "34b1478d2054f898d64b54b62a9bb738670087893bb1e5967da541539d2daf36" + "sha256": "be9609768b224459546b800236045b375448f2cb2cb47ccb337e149e0b016361" }, { "path": "schemas/workspace-diff.schema.json", diff --git a/packages/paperclip-runner/protocol/schemas/usage.schema.json b/packages/paperclip-runner/protocol/schemas/usage.schema.json index c1a57e17bf3..1f6bf2e1094 100644 --- a/packages/paperclip-runner/protocol/schemas/usage.schema.json +++ b/packages/paperclip-runner/protocol/schemas/usage.schema.json @@ -11,6 +11,73 @@ "providerRequestId": { "type": ["string", "null"], "maxLength": 240 }, "cumulative": { "$ref": "#/$defs/measurement" }, "runDeltaAvailable": { "type": "boolean" }, + "runDeltaBasis": { "const": "run" }, + "reconciliation": { + "type": "object", + "required": [ + "reason", + "receivedTurnId", + "cumulative", + "priorOrigin" + ], + "properties": { + "reason": { + "const": "accounting_reconciliation_required" + }, + "receivedTurnId": { + "type": [ + "string", + "null" + ], + "maxLength": 240 + }, + "cumulative": { + "$ref": "#/$defs/measurement" + }, + "priorOrigin": { + "type": [ + "object", + "null" + ], + "required": [ + "runnerInstanceId", + "runId", + "normalizedSessionId", + "turnId", + "itemId" + ], + "properties": { + "runnerInstanceId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "normalizedSessionId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "turnId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "runDelta": { "$ref": "#/$defs/measurement" } }, "additionalProperties": false, diff --git a/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs b/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs index 0398a6b2f3f..9a514bc3de9 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/examples/recovery-fixture.rs @@ -21,6 +21,8 @@ struct Script { retention_limit: usize, release_file: PathBuf, receipt_file: PathBuf, + #[serde(default)] + accounting: bool, } #[derive(Default)] @@ -86,11 +88,37 @@ fn workload(script: Script, shared: Shared) { digest = Sha256::digest(digest); } work = lock.lock().unwrap(); + let (event_type, payload) = if script.accounting { + if step == 1 { + ( + "usage.reported", + json!({"provider":"openai", "model":"test-model", + "runDeltaAvailable":true, "runDeltaBasis":"run", "runDelta":{"inputTokens":120,"outputTokens":10,"providerCostUsd":0.25}}), + ) + } else if step == 2 { + ("turn.completed", json!({"status":"completed"})) + } else { + ( + "usage.reported", + json!({"provider":"openai","model":"test-model","runDeltaAvailable":true,"runDeltaBasis":"run", + "runDelta":{"inputTokens":150,"outputTokens":15,"providerCostUsd":0.5}}), + ) + } + } else { + ( + "item.delta", + json!({"text": format!("step={step} sha256={digest:x}\n")}), + ) + }; work.events.push_back(PolledEvent { executor_event_id: format!("fixture-step-{step}"), - event_type: "item.delta".to_owned(), - priority: EventPriority::P2, - payload: json!({"text": format!("step={step} sha256={digest:x}\n")}), + event_type: event_type.to_owned(), + priority: if script.accounting { + EventPriority::P0 + } else { + EventPriority::P2 + }, + payload, }); work.completed = step; work.peak_retained = work.peak_retained.max(work.events.len()); diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs index 47c83bc81b5..88f6c18c27e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/bin/fake-codex-app-server.rs @@ -333,20 +333,60 @@ fn finish_turn_with_send( "text": "Codex completed the fake turn." }} }))?; + let cumulative_accounting = std::env::args().any(|arg| arg == "--cumulative-accounting-usage"); + let warm_tail_accounting = std::env::args().any(|arg| arg == "--warm-tail-accounting"); + let total_input = if warm_tail_accounting { + if state.next_turn == 1 { + 100 + } else { + 150 + } + } else if cumulative_accounting { + 100 + state.next_turn * 100 + } else { + 12 + }; + let total_output = if warm_tail_accounting { + if state.next_turn == 1 { + 10 + } else { + 15 + } + } else if cumulative_accounting { + 20 + state.next_turn * 10 + } else { + 3 + }; + if cumulative_accounting { + send( + json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id, + "tokenUsage":{"total":{"inputTokens":total_input-40,"outputTokens":total_output-4},"last":{"inputTokens":10,"outputTokens":1}}}}), + )?; + } send(json!({ "method": "thread/tokenUsage/updated", "params": { "threadId": state.thread_id, "tokenUsage": { - "total": {"inputTokens": 12, "outputTokens": 3}, - "last": {"inputTokens": 12, "outputTokens": 3, "requests": 1} + "total": {"inputTokens": total_input, "outputTokens": total_output}, + "last": {"inputTokens": if cumulative_accounting {40} else {12}, "outputTokens": if cumulative_accounting {4} else {3}, "requests": 1} } } }))?; + if state.next_turn > 1 && std::env::args().any(|arg| arg == "--hold-second-accounting-terminal") + { + return Ok(()); + } send(json!({ "method": "turn/completed", "params": {"turn": {"id": turn_id, "status": status}} }))?; + if std::env::args().any(|arg| arg == "--accounting-tail-after-terminal") { + send( + json!({"method":"thread/tokenUsage/updated","params":{"threadId":state.thread_id,"turnId":turn_id, + "tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}), + )?; + } Ok(()) } @@ -1285,7 +1325,7 @@ fn run() -> Result<(), Box> { for _ in 0..2 { send(json!({"method": "thread/tokenUsage/updated", "params": { "threadId": state.thread_id, "turnId": format!("provider-turn-{}", state.next_turn), - "tokenUsage": {"total": {"inputTokens": 100, "outputTokens": 20}, "last": {"inputTokens": 50, "outputTokens": 10}} + "tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {100 + state.next_turn * 100} else {100}, "outputTokens": if args.iter().any(|arg| arg == "--cumulative-accounting-usage") {20 + state.next_turn * 10} else {20}}, "last": {"inputTokens": 50, "outputTokens": 10}} }}))?; } } @@ -1889,6 +1929,43 @@ fn run() -> Result<(), Box> { } }))?; } + if state.next_turn == 1 { + if let Some(gate) = + argument(&args, "--late-accounting-tail-gate").map(PathBuf::from) + { + let thread_id = state.thread_id.clone(); + let turn_id = provider_turn_id.clone(); + let unsafe_tail = + args.iter().any(|arg| arg == "--unsafe-after-late-usage"); + let duplicate_burst = args + .iter() + .any(|arg| arg == "--late-accounting-duplicate-burst"); + thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !gate.is_file() && std::time::Instant::now() < deadline { + thread::sleep(Duration::from_millis(1)); + } + if gate.is_file() { + if duplicate_burst { + for _ in 0..33 { + let _ = send( + json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id,"tokenUsage":{"total":{"inputTokens":100,"outputTokens":10}}}}), + ); + } + } + let _ = send( + json!({"method":"thread/tokenUsage/updated","params":{"threadId":thread_id,"turnId":turn_id, + "tokenUsage":{"total":{"inputTokens":120,"outputTokens":12},"last":{"inputTokens":20,"outputTokens":2}}}}), + ); + if unsafe_tail { + let _ = send( + json!({"method":"turn/started","params":{"threadId":thread_id,"turn":{"id":"unsafe-next-turn"}}}), + ); + } + } + }); + } + } if emit_post_completion_warning { send(json!({ "method": "warning", @@ -1900,7 +1977,7 @@ fn run() -> Result<(), Box> { json!({ "method": "thread/tokenUsage/updated", "params": {"threadId": state.thread_id, "turnId": provider_turn_id, - "tokenUsage": {"total": {"inputTokens": 120, "outputTokens": 12}, + "tokenUsage": {"total": {"inputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {150} else {120}, "outputTokens": if args.iter().any(|arg| arg == "--warm-tail-accounting") && state.next_turn > 1 {15} else {12}}, "last": {"inputTokens": 20, "outputTokens": 2}}} }), json!({ diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs index 5f16baf89df..50f00d18adf 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_provider.rs @@ -1342,6 +1342,7 @@ impl CodexProvider { &mut self, authorized_tools: impl IntoIterator, completion_contract: Option<(&str, &[String])>, + accounting: &mut Vec<(String, Value)>, ) -> Result { let authorized_tools = authorized_tools.into_iter().collect::>(); let completion_contract = @@ -1354,7 +1355,15 @@ impl CodexProvider { { return Ok(false); } - let blockers = self.warm_run_attachment_blockers(true)?; + let (tail, result) = self.drain_completed_turn_tail_for_warm_attachment(); + accounting.extend(tail); + result?; + if !accounting.is_empty() { + return Err(LocalRunnerError::invalid( + "Codex warm run attachment requires pending accounting reconciliation", + )); + } + let blockers = self.warm_run_attachment_blockers()?; if !blockers.is_empty() { return Err(LocalRunnerError::invalid( format!( @@ -1375,49 +1384,53 @@ impl CodexProvider { Ok(true) } - fn drain_completed_turn_tail_for_warm_attachment(&mut self) -> Result<(), LocalRunnerError> { - let Some(completed_turn_id) = self - .completed_turn_authority - .as_ref() - .map(|authority| authority.provider_turn_id.clone()) - else { - return Ok(()); - }; - if self.active_provider_turn_id.is_some() { - return Ok(()); - } - - // Readiness probes run over the PRP command channel while provider - // stdout is drained by runnerd's adjacent control-loop iteration. A - // final usage/warning/item frame can therefore land after the last - // successful probe but before run.attach executes. Close that race in - // the same critical section as authority rotation. Only bounded tail - // notifications for the already-settled turn may be discarded: a new - // turn, provider request, process exit, or mismatched turn remains a - // fail-closed attachment error. - let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE; - let mut quiet_since: Option = None; - let mut drained = 0usize; - loop { - if std::time::Instant::now() >= deadline { - return Err(LocalRunnerError::invalid( - "Codex warm run attachment tail did not become quiescent", - )); + pub(crate) fn drain_completed_turn_tail_for_warm_attachment( + &mut self, + ) -> (Vec<(String, Value)>, Result<(), LocalRunnerError>) { + let mut accounting = Vec::new(); + let result = (|| { + let Some(completed_turn_id) = self + .completed_turn_authority + .as_ref() + .map(|authority| authority.provider_turn_id.clone()) + else { + return Ok(()); + }; + if self.active_provider_turn_id.is_some() { + return Ok(()); } - match self.poll()? { - Some(CodexProviderEvent::Notification { method, params }) => { - quiet_since = None; - drained = drained.saturating_add(1); - if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT { - return Err(LocalRunnerError::invalid( - "Codex warm run attachment tail exceeded its bounded frame limit", - )); - } - let names_other_turn = notification_turn_id(¶ms) - .is_some_and(|turn_id| turn_id != completed_turn_id); - let safe_tail_method = matches!( - method.as_str(), - "warning" + + // Readiness probes run over the PRP command channel while provider + // stdout is drained by runnerd's adjacent control-loop iteration. A + // final usage/warning/item frame can therefore land after the last + // successful probe but before run.attach executes. Close that race in + // the same critical section as authority rotation. Only bounded tail + // notifications for the already-settled turn may be discarded: a new + // turn, provider request, process exit, or mismatched turn remains a + // fail-closed attachment error. + let deadline = std::time::Instant::now() + WARM_ATTACHMENT_DRAIN_DEADLINE; + let mut quiet_since: Option = None; + let mut drained = 0usize; + loop { + if std::time::Instant::now() >= deadline { + return Err(LocalRunnerError::invalid( + "Codex warm run attachment tail did not become quiescent", + )); + } + match self.poll()? { + Some(CodexProviderEvent::Notification { method, params }) => { + quiet_since = None; + drained = drained.saturating_add(1); + if drained > WARM_ATTACHMENT_TAIL_DRAIN_LIMIT { + return Err(LocalRunnerError::invalid( + "Codex warm run attachment tail exceeded its bounded frame limit", + )); + } + let names_other_turn = notification_turn_id(¶ms) + .is_some_and(|turn_id| turn_id != completed_turn_id); + let safe_tail_method = matches!( + method.as_str(), + "warning" | "configWarning" | "remoteControl/status/changed" | "mcpServer/startupStatus/updated" @@ -1436,73 +1449,82 @@ impl CodexProvider { | "thread/status/changed" | "turn/diff/updated" | "turn/plan/updated" - ); - if names_other_turn || !safe_tail_method { - return Err(LocalRunnerError::invalid(format!( + ); + if names_other_turn || !safe_tail_method { + return Err(LocalRunnerError::invalid(format!( "Codex warm run attachment observed unsafe post-terminal provider method {}", bounded_method(&method) ))); - } - if let Some(frame_id) = self.take_provider_trace_frame_id() { - self.record_provider_trace_interpretation( + } + if matches!( + method.as_str(), + "thread/tokenUsage/updated" | "paperclip/resumeUsageSnapshot" + ) { + accounting.push((method, params)); + continue; + } + if let Some(frame_id) = self.take_provider_trace_frame_id() { + self.record_provider_trace_interpretation( frame_id, "codex.warm_attachment.completed_turn_tail", "ignored", Vec::new(), "Provider emitted a bounded tail notification after the prior turn terminal and before run attachment", ); + } } - } - Some(CodexProviderEvent::ResourceLimit { .. }) => { - return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation")); - } - Some(CodexProviderEvent::ProtocolFailure { .. }) => { - return Err(LocalRunnerError::invalid( - "Codex protocol integrity failed during warm attachment", - )); - } - Some(CodexProviderEvent::DescendantNotification { .. }) => { - // Child output cannot certify a quiescent root attachment. - return Err(LocalRunnerError::invalid( - "Codex descendant remains active during warm run attachment", - )); - } - Some(CodexProviderEvent::ToolCall { .. }) - | Some(CodexProviderEvent::RuntimeRequest { .. }) - | Some(CodexProviderEvent::McpPermissionRequest { .. }) => { - return Err(LocalRunnerError::invalid( - "Codex warm run attachment observed a post-terminal provider request", - )); - } - Some(CodexProviderEvent::Exited { .. }) => { - return Err(LocalRunnerError::invalid( - "Codex exited while quiescing for warm run attachment", - )); - } - None => { - let now = std::time::Instant::now(); - let quiet_start = quiet_since.get_or_insert(now); - if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW { - return Ok(()); + Some(CodexProviderEvent::ResourceLimit { .. }) => { + return Err(LocalRunnerError::invalid("Codex descendant capacity requires reconciliation before fresh-session continuation")); + } + Some(CodexProviderEvent::ProtocolFailure { .. }) => { + return Err(LocalRunnerError::invalid( + "Codex protocol integrity failed during warm attachment", + )); + } + Some(CodexProviderEvent::DescendantNotification { .. }) => { + // Child output cannot certify a quiescent root attachment. + return Err(LocalRunnerError::invalid( + "Codex descendant remains active during warm run attachment", + )); + } + Some(CodexProviderEvent::ToolCall { .. }) + | Some(CodexProviderEvent::RuntimeRequest { .. }) + | Some(CodexProviderEvent::McpPermissionRequest { .. }) => { + return Err(LocalRunnerError::invalid( + "Codex warm run attachment observed a post-terminal provider request", + )); + } + Some(CodexProviderEvent::Exited { + completed_turn_authoritative, + completion_reconciles_exit, + .. + }) if completion_reconciles_exit + || (completed_turn_authoritative && self.process.stdout_drained()) => + { + return Ok(()) + } + Some(CodexProviderEvent::Exited { .. }) => { + return Err(LocalRunnerError::invalid( + "Codex exited while quiescing for warm run attachment", + )); + } + None => { + let now = std::time::Instant::now(); + let quiet_start = quiet_since.get_or_insert(now); + if now.duration_since(*quiet_start) >= WARM_ATTACHMENT_QUIET_WINDOW { + return Ok(()); + } + std::thread::sleep(Duration::from_millis(1)); } - std::thread::sleep(Duration::from_millis(1)); } } - } + })(); + (accounting, result) } pub(crate) fn warm_run_attachment_blockers( &mut self, - quiesce_completed_tail: bool, ) -> Result, LocalRunnerError> { - // Only an explicit attachment-readiness probe may consume the bounded, - // already-settled provider suffix. Ordinary checkpoint snapshots run - // immediately after a terminal frame while the provider can still be - // unwinding; turning those observations into quiescence barriers can - // quarantine an otherwise reusable runner before its next turn. - if quiesce_completed_tail { - self.drain_completed_turn_tail_for_warm_attachment()?; - } let mut blockers = Vec::new(); if self.process.try_wait()?.is_some() { blockers.push("process_exited"); @@ -1517,7 +1539,18 @@ impl CodexProvider { blockers.push("ambiguous_turn_start"); } if !self.pending_messages.is_empty() { - blockers.push("pending_messages"); + blockers.push( + if self.pending_messages.iter().any(|message| { + matches!( + message.value.get("method").and_then(Value::as_str), + Some("paperclip/resumeUsageSnapshot" | "thread/tokenUsage/updated") + ) + }) { + "pending_accounting_reconciliation" + } else { + "pending_messages" + }, + ); } if !self.deferred_ambiguous_messages.is_empty() { blockers.push("deferred_messages"); @@ -2919,13 +2952,11 @@ impl CodexProvider { .and_then(Value::as_str); if notification_turn_id.is_some() && notification_turn_id != self.active_provider_turn_id.as_deref() - && notification_turn_id + && (notification_turn_id .is_some_and(|turn_id| self.settled_provider_turn_ids.contains(turn_id)) + || (self.active_provider_turn_id.is_none() + && method == "thread/tokenUsage/updated")) { - self.notification_identity_diagnostics += 1; - if self.notification_identity_diagnostics > 32 { - return Ok(None); - } // Resume replays the cumulative usage of the last settled turn. // Keep its identity and baseline, but never bill its `last` // measurement to the newly attached run. @@ -2941,6 +2972,10 @@ impl CodexProvider { }), })); } + self.notification_identity_diagnostics += 1; + if self.notification_identity_diagnostics > 32 { + return Ok(None); + } return Ok(Some(CodexProviderEvent::Notification { method: "warning".to_owned(), params: json!({ @@ -4392,9 +4427,58 @@ done }); // poll() normalizes settled-turn usage to paperclip/resumeUsageSnapshot. // That accounting fact is not new work and must not force replacement. - let result = provider.drain_completed_turn_tail_for_warm_attachment(); + let (accounting, result) = provider.drain_completed_turn_tail_for_warm_attachment(); provider.shutdown().unwrap(); - result.expect("historical usage must not break provider continuity"); + result.expect("historical usage must remain available to its owner"); + assert_eq!(accounting.len(), 1); + assert_eq!(accounting[0].1["total"]["inputTokens"], 120); + } + + #[cfg(unix)] + #[test] + fn failed_warm_drain_retains_consumed_usage_before_unsafe_tail() { + for attach in [false, true] { + let mut provider = completion_tail_provider(); + provider + .restore_completed_turn_authority(true, Some(1), Some("reader-tail-1")) + .unwrap(); + provider.active_provider_turn_id = None; + for value in [ + json!({"method":"thread/tokenUsage/updated","params":{"threadId":"reader-tail-thread","turnId":"reader-tail-1","tokenUsage":{"total":{"inputTokens":120,"outputTokens":12}}}}), + json!({"method":"turn/started","params":{"threadId":"reader-tail-thread","turn":{"id":"unsafe-next-turn"}}}), + ] { + provider + .pending_messages + .push_back(BufferedProviderMessage { + value, + trace_frame_id: None, + }); + } + let (accounting, result) = if attach { + let mut accounting = Vec::new(); + let tools = provider.authorized_tools.clone(); + let contract = provider.completion_contract.clone(); + let result = provider + .attach_run_in_place( + tools, + contract.as_ref().map(|contract| { + ( + contract.revision.as_str(), + contract.criterion_ids.as_slice(), + ) + }), + &mut accounting, + ) + .map(|_| ()); + (accounting, result) + } else { + provider.drain_completed_turn_tail_for_warm_attachment() + }; + provider.shutdown().unwrap(); + assert!(result.is_err()); + assert_eq!(accounting.len(), 1); + assert_eq!(accounting[0].1["total"]["inputTokens"], 120); + } } #[cfg(unix)] diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs b/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs new file mode 100644 index 00000000000..41e282ffe97 --- /dev/null +++ b/packages/paperclip-runner/runner/crates/runner-core/src/codex_usage.rs @@ -0,0 +1,234 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +pub(crate) struct CodexRunUsage { + baseline: Option>, + latest: BTreeMap, +} + +impl CodexRunUsage { + pub(crate) fn new(requires_baseline: bool) -> Self { + Self { + baseline: (!requires_baseline).then(BTreeMap::new), + latest: BTreeMap::new(), + } + } + + pub(crate) fn is_valid(&self) -> bool { + self.baseline + .iter() + .flat_map(|values| values.values()) + .chain(self.latest.values()) + .all(|value| { + value + .as_f64() + .is_some_and(|value| value.is_finite() && value >= 0.0) + }) + } + + pub(crate) fn increases_known_usage(&self, total: &Value) -> bool { + !self.latest.is_empty() + && total.as_object().is_some_and(|total| { + total.iter().any(|(key, value)| { + value.as_f64().is_some_and(|value| { + value > self.latest.get(key).and_then(Value::as_f64).unwrap_or(0.0) + }) + }) + }) + } + + pub(crate) fn increases_run_baseline(&self, total: &Value) -> bool { + self.baseline.as_ref().is_some_and(|baseline| { + total.as_object().is_some_and(|total| { + total.iter().any(|(key, value)| { + value.as_f64().is_some_and(|value| { + value > baseline.get(key).and_then(Value::as_f64).unwrap_or(0.0) + }) + }) + }) + }) + } + + pub(crate) fn has_baseline(&self) -> bool { + self.baseline.is_some() + } + + pub(crate) fn begin_run(&mut self) { + if self.latest.is_empty() { + return; + } + self.baseline = (valid_token(self.latest.get("inputTokens")) + && valid_token(self.latest.get("outputTokens"))) + .then(|| self.latest.clone()); + } + + pub(crate) fn observe(&mut self, total: &Value, historical: bool) -> (Value, bool) { + let measurement: BTreeMap = total + .as_object() + .into_iter() + .flat_map(|value| value.iter()) + .filter(|(_, value)| { + value + .as_f64() + .is_some_and(|value| value.is_finite() && value >= 0.0) + }) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + let complete = self.baseline.is_some() + && (historical + || measurement.iter().all(|(key, value)| { + !key.ends_with("Tokens") + || value.as_u64().is_some_and(|value| { + value >= self.latest.get(key).and_then(Value::as_u64).unwrap_or(0) + && value + >= self + .baseline + .as_ref() + .and_then(|baseline| baseline.get(key)) + .and_then(Value::as_u64) + .unwrap_or(0) + }) + })) + && valid_token(measurement.get("inputTokens")) + && valid_token(measurement.get("outputTokens")) + && measurement + .iter() + .all(|(key, value)| !key.ends_with("Tokens") || valid_token(Some(value))) + && self + .latest + .keys() + .all(|key| !key.ends_with("Tokens") || valid_token(measurement.get(key))); + if historical + && self.baseline.is_none() + && valid_token(measurement.get("inputTokens")) + && valid_token(measurement.get("outputTokens")) + { + self.baseline = Some(measurement.clone()); + } + for (key, value) in measurement { + if value.as_f64().unwrap() + >= self.latest.get(&key).and_then(Value::as_f64).unwrap_or(0.0) + { + self.latest.insert(key, value); + } + } + let delta: serde_json::Map = self + .latest + .iter() + .map(|(key, value)| { + let baseline = self + .baseline + .as_ref() + .and_then(|baseline| baseline.get(key)); + let delta = if let Some(value) = value.as_u64() { + json!(value.saturating_sub(baseline.and_then(Value::as_u64).unwrap_or(0))) + } else { + json!((value.as_f64().unwrap() + - baseline.and_then(Value::as_f64).unwrap_or(0.0)) + .max(0.0)) + }; + (key.clone(), delta) + }) + .collect(); + (Value::Object(delta), complete) + } +} + +fn valid_token(value: Option<&Value>) -> bool { + value + .and_then(Value::as_u64) + .is_some_and(|value| value <= 9_007_199_254_740_991) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preserves_resumed_baselines_across_requests_recovery_and_attachment() { + let mut usage = CodexRunUsage::new(true); + usage.observe( + &json!({"inputTokens":100,"outputTokens":10,"cachedInputTokens":20}), + true, + ); + let (first, complete) = usage.observe( + &json!({"inputTokens":140,"outputTokens":16,"cachedInputTokens":25}), + false, + ); + assert!(complete); + assert_eq!(first["inputTokens"], 40); + let mut restored: CodexRunUsage = + serde_json::from_value(serde_json::to_value(&usage).unwrap()).unwrap(); + let (second, complete) = restored.observe( + &json!({"inputTokens":190,"outputTokens":25,"cachedInputTokens":30}), + false, + ); + assert!(complete); + assert_eq!( + second, + json!({"inputTokens":90,"outputTokens":15,"cachedInputTokens":10}) + ); + restored.begin_run(); + let (next, complete) = restored.observe( + &json!({"inputTokens":220,"outputTokens":29,"cachedInputTokens":32}), + false, + ); + assert!(complete); + assert_eq!( + next, + json!({"inputTokens":30,"outputTokens":4,"cachedInputTokens":2}) + ); + } + + #[test] + fn attachment_before_paid_work_preserves_a_proven_empty_baseline() { + let mut usage = CodexRunUsage::new(false); + usage.begin_run(); + let (delta, complete) = usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false); + assert!(complete); + assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10})); + let mut unknown = CodexRunUsage::new(true); + unknown.begin_run(); + assert!( + !unknown + .observe(&json!({"inputTokens":100,"outputTokens":10}), false) + .1 + ); + } + + #[test] + fn regressing_live_totals_remain_incomplete_without_losing_known_exposure() { + let mut usage = CodexRunUsage::new(false); + usage.observe(&json!({"inputTokens":100,"outputTokens":10}), false); + let (delta, complete) = usage.observe(&json!({"inputTokens":90,"outputTokens":9}), false); + assert!(!complete); + assert_eq!(delta, json!({"inputTokens":100,"outputTokens":10})); + let mut resumed = CodexRunUsage::new(true); + resumed.observe(&json!({"inputTokens":100,"outputTokens":10}), true); + assert!( + !resumed + .observe(&json!({"inputTokens":90,"outputTokens":9}), false) + .1 + ); + } + + #[test] + fn unknown_baselines_and_missing_counters_cannot_certify_usage() { + let mut usage = CodexRunUsage::new(true); + assert!( + !usage + .observe(&json!({"inputTokens":140,"outputTokens":16}), false) + .1 + ); + usage.observe(&json!({"inputTokens":100,"outputTokens":10}), true); + assert!(!usage.observe(&json!({"inputTokens":160}), false).1); + assert!( + usage + .observe(&json!({"inputTokens":170,"outputTokens":20}), false) + .1 + ); + } +} diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs index 5c1d58d335f..ffbc524010e 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/lib.rs @@ -13,6 +13,7 @@ pub mod aws_agentcore_provider; pub mod claude_managed_provider; pub mod codex_provider; mod codex_startup_trust; +mod codex_usage; mod configured_environment; mod credential_observer; pub mod dot_provider_backend; diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index c84f57773e8..74a14c53002 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -17,6 +17,7 @@ use crate::codex_provider::{ ProviderStartupObservation, ProviderStartupStage, RejectedAcceptedTurn, MAX_SETTLED_PROVIDER_TURN_IDS, }; +use crate::codex_usage::CodexRunUsage; use crate::durable::{ create_private_temporary_file, current_unix_ms, open_private_regular_file, sanitize_value, validate_semantic_tool_input, verify_private_directory, Command, CommandExecution, @@ -93,6 +94,13 @@ struct ProviderEventIdentity { item_id: String, } +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +struct CodexAccountingOrigin { + identity: ProviderEventIdentity, + provider_turn_id: String, +} + #[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase", deny_unknown_fields)] struct ProviderStartupCommand { @@ -608,6 +616,105 @@ fn normalize_provider_notification( method: &str, params: &Value, ) -> Result, DurableRunnerError> { + if state.config.provider == "codex" + && ["thread/tokenUsage/updated", "paperclip/resumeUsageSnapshot"].contains(&method) + { + let historical = method == "paperclip/resumeUsageSnapshot"; + let total = if historical { + params.get("total") + } else { + params.pointer("/tokenUsage/total") + } + .unwrap_or(&Value::Null); + let increases_known_usage = state + .codex_usage + .as_ref() + .is_some_and(|usage| usage.increases_known_usage(total)); + let received_turn = params.get("turnId").and_then(Value::as_str); + let increases_prior_baseline = state.previous_accounting_origin.is_some() + && received_turn != state.completed_provider_turn_id.as_deref() + && state + .codex_usage + .as_ref() + .is_some_and(|usage| usage.increases_run_baseline(total)); + if historical + && (increases_known_usage || increases_prior_baseline) + && params.get("turnId").and_then(Value::as_str) + != state.active_provider_turn_id.as_deref() + { + state.lifecycle = "reconciliation_required".to_owned(); + let owns_completed_turn = state.active_provider_turn_id.is_none() + && state + .completed_provider_turn_id + .as_deref() + .is_some_and(|id| params.get("turnId").and_then(Value::as_str) == Some(id)); + if !owns_completed_turn { + if state.active_provider_turn_id.is_some() { + state.settle_active_provider_turn_identity()?; + } + state.active_provider_turn_id = None; + let mut invalidation = normalize_codex_notification("thread/tokenUsage/updated", &json!({ + "threadId":params.get("threadId"), "model":state.config.model, "tokenUsage":{"total":total} + })).remove(0); + invalidation.payload["runDeltaBasis"] = json!("run"); + invalidation.payload["reconciliation"] = json!({"reason":"accounting_reconciliation_required", + "receivedTurnId":params.get("turnId"),"cumulative":invalidation.payload["cumulative"], + "priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| received_turn == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity)}); + return Ok(vec![ + NormalizedProviderEvent { + event_type: "harness.diagnostic".to_owned(), + priority: EventPriority::P0, + payload: json!({"code":"codex_usage_accounting_integrity", "classification":"accounting_reconciliation_required", + "receivedTurnId":params.get("turnId"),"cumulative":total,"priorOrigin":state.previous_accounting_origin.as_ref().filter(|origin| params.get("turnId").and_then(Value::as_str) == Some(origin.provider_turn_id.as_str())).map(|origin| &origin.identity), + "message":"Late provider usage cannot be attributed to the attached run; explicit reconciliation is required"}), + }, + invalidation, + NormalizedProviderEvent { + event_type: "turn.failed".to_owned(), + priority: EventPriority::P0, + payload: json!({"provider":"codex","status":"failed","code":"codex_usage_accounting_integrity","recoverable":false, + "message":"Late provider usage requires explicit accounting reconciliation"}), + }, + NormalizedProviderEvent { + event_type: "run.terminal".to_owned(), + priority: EventPriority::P0, + payload: json!({"schema":"paperclip.prp.terminal.v1","provider":"codex","turnTerminalState":"failed","runTerminalState":"failed","reportedWorkDisposition":"needs_review"}), + }, + ]); + } + } + let had_baseline = state + .codex_usage + .as_ref() + .is_some_and(CodexRunUsage::has_baseline); + let (delta, complete) = state + .codex_usage + .get_or_insert_with(|| CodexRunUsage::new(true)) + .observe(total, historical); + let mut normalized_params = params.clone(); + let emit_usage = !historical || (had_baseline && increases_known_usage); + if historical && emit_usage { + normalized_params = json!({"threadId": params.get("threadId"), "turnId": params.get("turnId"), + "tokenUsage":{"total":total,"last":delta}}); + } else if !historical { + normalized_params["tokenUsage"]["last"] = delta; + } + let mut events = normalize_codex_notification( + if emit_usage { + "thread/tokenUsage/updated" + } else { + method + }, + &normalized_params, + ); + for event in &mut events { + if event.event_type == "usage.reported" { + event.payload["runDeltaAvailable"] = json!(complete); + event.payload["runDeltaBasis"] = json!("run"); + } + } + return Ok(events); + } if method != "paperclip/runResult" { return Ok(normalize_codex_notification(method, params) .into_iter() @@ -640,14 +747,21 @@ fn normalize_provider_notification( } } +fn terminal_completion_rejected(state: &CodexProviderState) -> bool { + state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none() +} + +fn goal_continues_after_turn(state: &CodexProviderState, goal_active: bool) -> bool { + goal_active && !terminal_completion_rejected(state) +} + fn terminal_events( state: &CodexProviderState, event_type: &str, goal_status: Option<&str>, ) -> Vec { - let rejected_completion = - state.active_terminal_tool_rejected && state.active_provider_result_fingerprint.is_none(); - if goal_status == Some("active") && !rejected_completion { + let rejected_completion = terminal_completion_rejected(state); + if goal_continues_after_turn(state, goal_status == Some("active")) { return Vec::new(); } let Some(contract) = state.completion_contract.as_ref() else { @@ -1007,6 +1121,10 @@ struct CodexProviderState { #[serde(default)] opencode_launch_profile_digest: Option, #[serde(default)] + codex_usage: Option, + #[serde(default)] + previous_accounting_origin: Option, + #[serde(default)] completion_contract: Option, #[serde(default)] tool_bridge: ProviderToolBridge, @@ -1107,11 +1225,15 @@ impl CodexProviderState { tool_bridge: ProviderToolBridge, ) -> Self { let thread_id = config.provider_session_id.clone(); + let codex_usage = + (config.provider == "codex").then(|| CodexRunUsage::new(thread_id.is_some())); Self { schema: PROVIDER_STATE_SCHEMA.to_owned(), lifecycle: "prepared".to_owned(), config, opencode_launch_profile_digest: None, + codex_usage, + previous_accounting_origin: None, completion_contract, tool_bridge, thread_id, @@ -1154,6 +1276,13 @@ impl CodexProviderState { self.tool_bridge.validate_recovered().map_err(|error| { DurableRunnerError::invalid(format!("Codex semantic tool state is invalid: {error}")) })?; + if self + .codex_usage + .as_ref() + .is_some_and(|usage| !usage.is_valid()) + { + return Err(DurableRunnerError::invalid("invalid Codex usage baseline")); + } let mut pending_event_ids = HashSet::new(); if self.descendant_thread_ids.len() > crate::codex_provider::MAX_DESCENDANT_THREAD_IDS || self @@ -2501,7 +2630,43 @@ impl CodexCommandExecutor { stable } + fn persist_completed_accounting_tail( + &mut self, + tail: Vec<(String, Value)>, + failed: bool, + ) -> Result<(), DurableRunnerError> { + if !tail.is_empty() || failed { + let state = self.state.as_mut().ok_or_else(|| { + DurableRunnerError::invalid("Codex provider has not been prepared") + })?; + for (method, params) in tail { + let events = normalize_provider_notification(state, &method, ¶ms)?; + state.extend_events(events)?; + } + if failed { + state.lifecycle = "reconciliation_required".to_owned(); + } + self.save_state()?; + } + Ok(()) + } + + fn retain_completed_accounting_tail(&mut self) -> Result<(), DurableRunnerError> { + let (tail, result) = self + .provider + .as_mut() + .map(CodexProvider::drain_completed_turn_tail_for_warm_attachment) + .unwrap_or_else(|| (Vec::new(), Ok(()))); + self.persist_completed_accounting_tail(tail, result.is_err())?; + result.map_err(|error| { + DurableRunnerError::invalid(format!( + "failed to retain completed Codex accounting: {error}" + )) + }) + } + fn attach_run(&mut self, payload: &Value) -> Result<(), DurableRunnerError> { + self.retain_completed_accounting_tail()?; let mut next_state = self .state .clone() @@ -2600,6 +2765,17 @@ impl CodexCommandExecutor { "run.attach tool contract could not be rebound: {error}" )) })?; + if let Some(usage) = next_state.codex_usage.as_mut() { + usage.begin_run(); + } + next_state.previous_accounting_origin = self + .event_identity + .clone() + .zip(next_state.completed_provider_turn_id.clone()) + .map(|(identity, provider_turn_id)| CodexAccountingOrigin { + identity, + provider_turn_id, + }); next_state.completion_contract = completion_contract; next_state.completed_turn_authoritative = false; next_state.completed_turn_process_generation = None; @@ -2613,10 +2789,12 @@ impl CodexCommandExecutor { next_state.active_provider_result_disposition = None; next_state.active_terminal_tool_rejected = false; next_state.last_agent_message = None; - let retained_provider = if let Some(provider) = self.provider.as_mut() { - !runtime_launch_changed - && !upgraded_skill_config - && provider + let mut attachment_accounting = Vec::new(); + let retention = if let Some(provider) = self.provider.as_mut() { + if runtime_launch_changed || upgraded_skill_config { + Ok(false) + } else { + provider .attach_run_in_place( next_state.tool_bridge.authorized_tools().cloned(), next_state.completion_contract.as_ref().map(|contract| { @@ -2625,12 +2803,14 @@ impl CodexCommandExecutor { contract.criterion_ids.as_slice(), ) }), + &mut attachment_accounting, ) .map_err(|error| { DurableRunnerError::invalid(format!( "failed to retain Codex for warm run attachment: {error}" )) - })? + }) + } } else if next_state.lifecycle == "prepared" && next_state.provider_process_generation > 0 && next_state.pending_events.iter().all(is_startup_audit_event) @@ -2640,12 +2820,17 @@ impl CodexCommandExecutor { // validated run-scoped settings here; open_session then restores // that same thread under the new authority. Never restart during // drain/suspend or require the stopped process to still exist. - false + Ok(false) } else { return Err(DurableRunnerError::invalid( "run.attach requires the restored Codex provider process", )); }; + self.persist_completed_accounting_tail(attachment_accounting, retention.is_err())?; + let retained_provider = match retention { + Ok(retained) => retained, + Err(error) => return Err(error), + }; if !retained_provider { if let Some(provider) = self.provider.as_mut() { provider.shutdown().map_err(|error| { @@ -4044,10 +4229,13 @@ impl CodexCommandExecutor { .get("quiesceForWarmAttach") .and_then(Value::as_bool) .unwrap_or(false); + if quiesce_for_warm_attach { + self.retain_completed_accounting_tail()?; + } let mut warm_attach_blockers = self .provider .as_mut() - .map(|provider| provider.warm_run_attachment_blockers(quiesce_for_warm_attach)) + .map(|provider| provider.warm_run_attachment_blockers()) .transpose() .map_err(|error| { DurableRunnerError::invalid(format!( @@ -4071,6 +4259,9 @@ impl CodexCommandExecutor { if !state.queued_events.is_empty() { warm_attach_blockers.push("durable_queued_events"); } + if state.lifecycle == "reconciliation_required" { + warm_attach_blockers.push("reconciliation_required"); + } let warm_attach_ready = warm_attach_blockers.is_empty(); Ok(CommandExecution::result(json!({ "status": state.lifecycle, @@ -4281,6 +4472,22 @@ impl CodexCommandExecutor { } else { None }; + let goal_remains_active = match &goal_reconciliation { + Some(Ok(snapshot)) => normalize_codex_goal(snapshot, false) + .is_some_and(|goal| goal.status == "active"), + _ => self + .state + .as_ref() + .and_then(|state| state.goal.as_ref()) + .is_some_and(|goal| goal.status == "active"), + }; + let run_continues = self + .state + .as_ref() + .is_some_and(|state| goal_continues_after_turn(state, goal_remains_active)); + if terminal_event_type.as_deref() == Some("turn.completed") && !run_continues { + self.retain_completed_accounting_tail()?; + } let working_now = self .provider .as_ref() @@ -4370,7 +4577,9 @@ impl CodexCommandExecutor { state.receipt_limit_interrupt_attempts = 0; state.receipt_limit_interrupt_deadline_unix_ms = None; state.ambiguous_turn_start_pending = false; - state.lifecycle = "session_open".to_owned(); + if state.lifecycle != "reconciliation_required" { + state.lifecycle = "session_open".to_owned(); + } if let Some(goal) = state.goal.as_mut() { goal.working_now = false; state.goal_revision = state.goal_revision.saturating_add(1); @@ -4473,6 +4682,13 @@ impl CodexCommandExecutor { }) .unwrap_or_default(); self.save_state()?; + if self + .state + .as_ref() + .is_some_and(|state| state.lifecycle == "reconciliation_required") + { + break; + } if let (Some(frame_id), Some(provider)) = (trace_frame_id, self.provider.as_mut()) { @@ -5894,6 +6110,8 @@ mod tests { conversation_mode: None, }, opencode_launch_profile_digest: None, + codex_usage: Some(CodexRunUsage::new(false)), + previous_accounting_origin: None, completion_contract: None, tool_bridge: ProviderToolBridge::default(), thread_id: Some("thread-1".to_owned()), diff --git a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs index ead00ee0f33..09368feb54a 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/tests/codex_provider.rs @@ -3964,6 +3964,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta "--durable-turn-ids", "--emit-post-completion-warning", "--emit-post-completion-passive-statuses", + "--warm-tail-accounting", ], ); let runner_config = durable_config(&directory); @@ -3991,6 +3992,7 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta )) .expect("start the first turn"); + let mut old_usage = Vec::new(); let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); loop { assert!( @@ -3998,6 +4000,12 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta "the first turn must settle before attachment" ); let events = poll_and_ack(&mut executor).expect("poll the first turn terminal"); + old_usage.extend( + events + .iter() + .filter(|event| event.event_type == "usage.reported") + .cloned(), + ); if events .iter() .any(|event| event.event_type == "turn.completed") @@ -4010,6 +4018,11 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta std::thread::sleep(std::time::Duration::from_millis(1)); } + assert_eq!( + old_usage.last().unwrap().payload["runDelta"]["inputTokens"], + 120 + ); + assert_eq!(old_usage.last().unwrap().payload["runDeltaAvailable"], true); let readiness = executor .execute(&command( "readiness", @@ -4034,6 +4047,23 @@ fn durable_backend_drains_completed_turn_usage_and_passive_tail_during_warm_atta .iter() .any(|(event_type, _, _)| event_type == "run.attached")); + executor + .execute(&command( + "next-turn", + 6, + "turn.start", + json!({"text":"Complete the next paid turn."}), + )) + .unwrap(); + let next_events = wait_for_executor_event_batch(&mut executor, "turn.completed"); + let next_usage = next_events + .iter() + .filter(|event| event.event_type == "usage.reported") + .last() + .unwrap(); + assert_eq!(next_usage.payload["runDelta"]["inputTokens"], 30); + assert_eq!(next_usage.payload["runDelta"]["outputTokens"], 3); + assert_eq!(next_usage.payload["runDeltaAvailable"], true); executor.shutdown().expect("stop the warm provider"); fs::remove_dir_all(directory).expect("remove Codex integration-test directory"); } @@ -6256,6 +6286,9 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() { let mut config = provider_config(&directory, &["--complete-after-tool-call"]); if scenario == "controller-goal" { config.args.push("--agent-created-goal-on-open".to_owned()); + config + .args + .push("--accounting-tail-after-terminal".to_owned()); } config.args.extend([ "--terminal-result-file".to_owned(), @@ -6327,6 +6360,13 @@ fn rejected_declared_completion_stays_failed_when_the_provider_completes() { .unwrap(); assert!(failure < turn && turn < terminal); if scenario == "controller-goal" { + let final_usage = emitted + .iter() + .rposition(|event| event.event_type == "usage.reported") + .unwrap(); + assert!(final_usage < failure && final_usage < turn && final_usage < terminal); + assert_eq!(emitted[final_usage].payload["runDelta"]["inputTokens"], 120); + assert_eq!(emitted[final_usage].payload["runDeltaAvailable"], true); let settled_goal = emitted .iter() .rposition(|event| { @@ -6687,7 +6727,11 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() { let directory = temporary_directory("resume-usage-snapshot"); let config = provider_config( &directory, - &["--durable-turn-ids", "--resume-usage-snapshot"], + &[ + "--durable-turn-ids", + "--resume-usage-snapshot", + "--cumulative-accounting-usage", + ], ); let runner_config = durable_config(&directory); let mut first = CodexCommandExecutor::with_runner_config(&directory, &runner_config); @@ -6756,7 +6800,7 @@ fn resume_usage_is_a_historical_diagnostic_not_a_warning_or_charge() { .collect(); assert_eq!(snapshots.len(), 2); assert_eq!(snapshots[0].payload["receivedTurnId"], "provider-turn-1"); - assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 100); + assert_eq!(snapshots[0].payload["cumulative"]["inputTokens"], 200); assert!(!events.iter().any(|event| matches!( event.event_type.as_str(), "provider.notice.recorded" | "usage.reported" @@ -6988,3 +7032,361 @@ fn skill_flag_can_be_added_to_an_old_checkpoint_at_settled_run_attach() { ); fs::remove_dir_all(directory).unwrap(); } + +#[test] +fn durable_codex_usage_keeps_run_baselines_through_requests_recovery_and_attachment() { + let directory = temporary_directory("accounting-run-baseline"); + fs::write( + directory.join("fake-state.json"), + serde_json::to_vec(&json!({"threadId":"thread-1","activeTurnId":null,"nextTurn":1})) + .unwrap(), + ) + .unwrap(); + let mut config = provider_config( + &directory, + &[ + "--durable-turn-ids", + "--resume-usage-snapshot", + "--cumulative-accounting-usage", + ], + ); + config.provider_session_id = Some("thread-1".to_owned()); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider":config,"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + for _ in 0..20 { + if poll_and_ack(&mut executor) + .unwrap() + .iter() + .any(|event| event.payload["code"] == "codex_resume_usage_snapshot") + { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + fn run( + executor: &mut CodexCommandExecutor, + sequence: u64, + expected_input: u64, + expected_output: u64, + ) { + executor + .execute(&command( + &format!("turn-{sequence}"), + sequence, + "turn.start", + json!({"text":"Complete the accounting turn."}), + )) + .unwrap(); + let events = wait_for_executor_event_batch(executor, "turn.completed"); + let usage = events + .iter() + .filter(|event| event.event_type == "usage.reported") + .last() + .expect("provider emitted usage"); + assert_eq!(usage.payload["runDeltaBasis"], "run"); + assert_eq!(usage.payload["runDeltaAvailable"], true); + assert_eq!(usage.payload["runDelta"]["inputTokens"], expected_input); + assert_eq!(usage.payload["runDelta"]["outputTokens"], expected_output); + for _ in 0..20 { + if poll_and_ack(executor).unwrap().is_empty() { + break; + } + } + } + run(&mut executor, 3, 100, 10); + run(&mut executor, 4, 200, 20); + executor.shutdown().unwrap(); + drop(executor); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + run(&mut executor, 5, 300, 30); + executor + .execute(&command( + "attach", + 6, + "run.attach", + json!({"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + run(&mut executor, 7, 100, 10); + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); +} + +#[test] +fn late_codex_usage_requires_durable_reconciliation_before_or_after_handoff() { + for mode in ["before", "after", "after-paid", "unsafe"] { + let handoff_first = mode.starts_with("after"); + let directory = temporary_directory("late-accounting-reconciliation"); + let gate = directory.join("release-late-usage"); + let gate_arg = gate.to_string_lossy().into_owned(); + let config = provider_config( + &directory, + &[ + "--durable-turn-ids", + "--warm-tail-accounting", + "--late-accounting-tail-gate", + &gate_arg, + if mode == "after-paid" { + "--hold-second-accounting-terminal" + } else { + "--ordinary-terminal" + }, + if mode == "after-paid" { + "--late-accounting-duplicate-burst" + } else { + "--ordinary-tail-count" + }, + if mode == "unsafe" { + "--unsafe-after-late-usage" + } else { + "--ordinary-late-usage" + }, + ], + ); + let runner_config = durable_config(&directory); + let mut executor = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider":config,"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text":"Finish before the delayed accounting report."}), + )) + .unwrap(); + let events = wait_for_executor_event_batch(&mut executor, "turn.completed"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == "usage.reported") + .last() + .unwrap() + .payload["runDelta"]["inputTokens"], + 100 + ); + if handoff_first { + executor + .execute(&command( + "attach", + 4, + "run.attach", + json!({"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + let mut next = runner_config.clone(); + next.run_id = "run-2".to_owned(); + executor.rotate_authority(&next); + } + if mode == "after-paid" { + executor + .execute(&command( + "successor-turn", + 5, + "turn.start", + json!({"text":"Start successor paid work."}), + )) + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + assert!(std::time::Instant::now() < deadline); + let events = poll_and_ack(&mut executor).unwrap(); + assert!(!events + .iter() + .any(|event| event.event_type == "turn.completed")); + if let Some(usage) = events + .iter() + .find(|event| event.event_type == "usage.reported") + { + assert_eq!(usage.payload["runDelta"]["inputTokens"], 50); + break; + } + } + } + fs::write(&gate, "release").unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let retained = loop { + assert!(std::time::Instant::now() < deadline); + let events = if mode == "unsafe" { + match executor.execute(&command( + "quiesce", + 4, + "session.snapshot", + json!({"quiesceForWarmAttach":true}), + )) { + Ok(_) => { + std::thread::sleep(std::time::Duration::from_millis(1)); + continue; + } + Err(error) => { + assert!(error + .to_string() + .contains("failed to retain completed Codex accounting")); + executor.retained_events().unwrap() + } + } + } else { + executor.poll_events().unwrap() + }; + if events.iter().any(|event| { + event.event_type == "usage.reported" + || event.payload["code"] == "codex_usage_accounting_integrity" + }) { + break events; + } + executor.acknowledge_events(events.len()).unwrap(); + }; + if handoff_first { + assert!(retained + .iter() + .filter(|event| event.event_type == "usage.reported") + .all(|event| event.payload["runDeltaAvailable"] == false)); + assert!(retained + .iter() + .any(|event| event.event_type == "usage.reported" + && event.payload["runDeltaAvailable"] == false)); + assert!(retained + .iter() + .any(|event| event.event_type == "run.terminal" + && event.payload["runTerminalState"] == "failed")); + let required = retained + .iter() + .find(|event| event.event_type == "usage.reported") + .unwrap(); + assert_eq!( + required.payload["reconciliation"]["cumulative"]["inputTokens"], + 120 + ); + assert_eq!( + required.payload["reconciliation"]["priorOrigin"]["runId"], + "run-1" + ); + let evidence = retained + .iter() + .find(|event| event.payload["code"] == "codex_usage_accounting_integrity") + .unwrap(); + assert_eq!(evidence.payload["priorOrigin"]["runId"], "run-1"); + assert_eq!(evidence.payload["receivedTurnId"], "provider-turn-1"); + assert_eq!(evidence.payload["cumulative"]["inputTokens"], 120); + } else { + assert_eq!( + retained + .iter() + .find(|event| event.event_type == "usage.reported") + .unwrap() + .payload["runDelta"]["inputTokens"], + 120 + ); + } + let saved: Value = + serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap()) + .unwrap(); + assert_eq!(saved["lifecycle"], "reconciliation_required"); + assert!(!saved["pendingEvents"].as_array().unwrap().is_empty()); + let error = executor + .execute(&command( + "blocked-attach", + 5, + "run.attach", + json!({"authorizedTools":task_context_tool_set()}), + )) + .unwrap_err(); + assert!(error + .to_string() + .contains("requires explicit reconciliation")); + executor.shutdown().unwrap(); + drop(executor); + let mut restored = CodexCommandExecutor::with_runner_config(&directory, &runner_config); + let error = restored + .execute(&command( + "blocked-after-restart", + 6, + "run.attach", + json!({"authorizedTools":task_context_tool_set()}), + )) + .unwrap_err(); + assert!(error + .to_string() + .contains("requires explicit reconciliation")); + restored.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); + } +} + +#[test] +fn durable_codex_usage_keeps_completed_turn_through_reconciled_provider_exit() { + for exit_flag in [ + "--exit-after-turn-completion", + "--fail-after-turn-completion", + ] { + let directory = temporary_directory("accounting-reconciled-exit"); + let config = provider_config( + &directory, + &[ + "--durable-turn-ids", + "--warm-tail-accounting", + "--emit-post-completion-passive-statuses", + exit_flag, + ], + ); + let mut executor = + CodexCommandExecutor::with_runner_config(&directory, &durable_config(&directory)); + executor + .execute(&command( + "prepare", + 1, + "run.prepare", + json!({"provider":config,"authorizedTools":task_context_tool_set()}), + )) + .unwrap(); + executor + .execute(&command("open", 2, "session.open", json!({}))) + .unwrap(); + executor + .execute(&command( + "turn", + 3, + "turn.start", + json!({"text":"Complete accounting before exiting."}), + )) + .unwrap(); + let events = wait_for_executor_event_batch(&mut executor, "turn.completed"); + assert_eq!( + events + .iter() + .filter(|event| event.event_type == "usage.reported") + .last() + .unwrap() + .payload["runDelta"]["inputTokens"], + 120 + ); + assert!(!events.iter().any(|event| event.event_type == "turn.failed")); + let saved: Value = + serde_json::from_slice(&fs::read(directory.join("codex-provider-state.json")).unwrap()) + .unwrap(); + assert_ne!(saved["lifecycle"], "reconciliation_required"); + assert_eq!(saved["completedTurnAuthoritative"], true); + executor.shutdown().unwrap(); + fs::remove_dir_all(directory).unwrap(); + } +} diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index c09a2c50ea0..4b351a656f3 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -1123,6 +1123,43 @@ it("refuses a reusable close checkpoint when the local provider snapshot is unre } }, 15_000); +it("installs required accounting acceptance before the Runner acknowledgement callback returns", async () => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-required-accounting-")); + const OriginalCore = durableControlPlane.DurablePrpControlPlane; + let callbacks!: ConstructorParameters[0]; + const spy = vi.spyOn(durableControlPlane, "DurablePrpControlPlane").mockImplementation(function (options) { + callbacks = options; + return new OriginalCore(options); + }); + let release!: () => void; + const accepted = new Promise(resolve => { release = resolve; }); + const onAccountingEvent = vi.fn(async () => accepted); + const bundle = createCapabilityRunnerdCodexTransport({ stateDirectory: directory, sourceCodexHome: "", environment: {}, + runnerBinary: process.execPath, runnerArtifact: { version: "test", digest: `sha256:${"a".repeat(64)}` }, + codexCommand: process.execPath, codexArgs: [], onAccountingEvent, + controlPlaneRegistration: async () => { throw new Error("stop before Runner dispatch"); }, + }); + try { + await expect(bundle.transport.request("thread/start", { cwd: directory })).rejects.toThrow(); + const report = { eventType: "usage.reported", sourceEventId: "required-usage" } as PrpEvent; + let returned = false; + const result = callbacks.onCommittedEvent!(report).then(() => { returned = true; }); + await vi.waitFor(() => expect(onAccountingEvent).toHaveBeenCalledExactlyOnceWith(report)); + expect(returned).toBe(false); + release(); + await result; + expect(returned).toBe(true); + const failure = new Error("receipt unavailable"); + onAccountingEvent.mockRejectedValueOnce(failure); + await expect(callbacks.onCommittedEvent!(report)).rejects.toBe(failure); + } finally { + release(); + await bundle.transport.close().catch(() => undefined); + spy.mockRestore(); + await rm(directory, { recursive: true, force: true }); + } +}); + it("uses the same API result envelope for live dispatch and recovered receipts", async () => { const stateDirectory = await mkdtemp(join(tmpdir(), "runnerd-recovered-api-")); const OriginalCore = durableControlPlane.DurablePrpControlPlane; diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d994f6466de..23ab1f1721e 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -1240,6 +1240,7 @@ export interface CapabilityRunnerdCodexTransportOptions extends RunnerExecutionD provider?: "codex" | "opencode" | "claude_managed" | "aws_agentcore" | "acpx"; observeCredentials?: boolean; onCredentialsUpdated?: (event: PrpEvent) => Promise; + onAccountingEvent?: (event: PrpEvent) => Promise; recoverSemanticToolInput?: DurablePrpControlPlaneOptions["recoverSemanticToolInput"]; opencodePermissionMode?: NativeOpenCodePermissionMode; acpxAgent?: QualifiedAcpxAgent; @@ -5239,6 +5240,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { + await this.options.onAccountingEvent?.(event); if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); } @@ -5691,6 +5693,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { onSemanticToolInput: (call) => this.#handleSemanticToolInput(call), recoverSemanticToolInput: call => this.#recoverSemanticToolInput(call), onCommittedEvent: async (event) => { + await this.options.onAccountingEvent?.(event); if (event.eventType === "auth.credentials.updated") { await this.options.onCredentialsUpdated?.(event); } diff --git a/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts b/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts index 2720e4a5326..92905627855 100644 --- a/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts +++ b/packages/paperclip-runner/src/protocol/generated/schema-bundle.ts @@ -3385,6 +3385,75 @@ export const usageSchema = { "runDeltaAvailable": { "type": "boolean" }, + "runDeltaBasis": { + "const": "run" + }, + "reconciliation": { + "type": "object", + "required": [ + "reason", + "receivedTurnId", + "cumulative", + "priorOrigin" + ], + "properties": { + "reason": { + "const": "accounting_reconciliation_required" + }, + "receivedTurnId": { + "type": [ + "string", + "null" + ], + "maxLength": 240 + }, + "cumulative": { + "$ref": "#/$defs/measurement" + }, + "priorOrigin": { + "type": [ + "object", + "null" + ], + "required": [ + "runnerInstanceId", + "runId", + "normalizedSessionId", + "turnId", + "itemId" + ], + "properties": { + "runnerInstanceId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "runId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "normalizedSessionId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "turnId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + }, + "itemId": { + "type": "string", + "minLength": 1, + "maxLength": 240 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, "runDelta": { "$ref": "#/$defs/measurement" } diff --git a/packages/paperclip-runner/src/protocol/replay-contract.test.ts b/packages/paperclip-runner/src/protocol/replay-contract.test.ts index 3f17ecf2b62..bad4d2536ee 100644 --- a/packages/paperclip-runner/src/protocol/replay-contract.test.ts +++ b/packages/paperclip-runner/src/protocol/replay-contract.test.ts @@ -1,3 +1,5 @@ +import Ajv2020 from "ajv/dist/2020.js"; +import { usageSchema } from "./generated/schema-bundle.js"; import { readFile } from "node:fs/promises"; import { describe, expect, it } from "vitest"; @@ -6,6 +8,7 @@ import { negotiateProtocolVersion, parsePrpFixtureText, PRP_PROTOCOL_VERSION, + validatePrpEvent, } from "./replay-contract.js"; import { reducePrpFixture } from "../reducer/session-reducer.js"; @@ -371,3 +374,24 @@ describe("PRP v1 JSON Schema contract", () => { ).toBeNull(); }); }); + +it("accepts marked whole-run usage through the runtime standalone event validator", async () => { + const fixture = await readFixture(); + const event = (fixture.events as Array>)[0]; + const measurement = { inputTokens: 120, outputTokens: 10, cacheReadTokens: 0, cacheWriteTokens: 0, + activeSeconds: 0, requests: 1, providerCostUsd: 0 }; + const result = validatePrpEvent({ ...event, eventType: "usage.reported", priority: 0, + payload: { provider: "codex", model: "gpt-6.1-sol", cumulative: measurement, runDelta: measurement, + runDeltaAvailable: true, runDeltaBasis: "run" } }); + expect(result).toMatchObject({ ok: true }); +}); + +it("declares required incomplete reconciliation evidence in the strict usage contract", () => { + const validate = new Ajv2020({ strict: true, allErrors: true }).compile(usageSchema); + const measurement = { inputTokens: 120, outputTokens: 12, cacheReadTokens: 0, cacheWriteTokens: 0, activeSeconds: 0, requests: 0, providerCostUsd: 0 }; + const payload = { provider: "codex", model: null, cumulative: measurement, runDelta: { ...measurement, inputTokens: 0, outputTokens: 0 }, runDeltaAvailable: false, runDeltaBasis: "run", + reconciliation: { reason: "accounting_reconciliation_required", receivedTurnId: "provider-turn-1", cumulative: measurement, + priorOrigin: { runnerInstanceId: "runner-1", runId: "old-run", normalizedSessionId: "session", turnId: "turn", itemId: "item" } } }; + expect(validate(payload), JSON.stringify(validate.errors)).toBe(true); + expect(validate({ ...payload, reconciliation: { ...payload.reconciliation, arbitraryProviderData: "excluded" } })).toBe(false); +}); diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index ada9938813b..6c4c07c36cd 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -1779,6 +1779,7 @@ export interface IssueAttachment { /** Bounded cold-open window; the complete chronological feed stays paginated. */ export interface IssueRecentComments { latestCommentId: string | null; + oldestCommentId?: string | null; comments: IssueComment[]; hasMore: boolean; } diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 3d4ddd3fd5f..8efc8264bed 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -579,6 +579,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), + readWorkflow("chat-feature-testops.yml"), ]; for (const workflow of actionPinWorkflows) { @@ -618,6 +619,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", + "chat-feature-testops.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); const providerSecretReference = @@ -650,6 +652,38 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { (block) => [...block.matchAll(providerSecretReference)].length > 0, ); assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`); + if (name === "chat-feature-testops.yml") { + assert.equal(providerJobs.length, 1); + const assertBoundary = (block) => { + assert.match(block, /^ chat:\n/); + assert.match(block, /\n needs: \[authorize, select, build\]\n/); + assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); + assert.match(block, /\n environment: runner-e2e-private-paid\n/); + assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); + assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/); + assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); + assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/); + assert.doesNotMatch(block, /ALLURE_TOKEN/); + assert.equal([...block.matchAll(providerSecretReference)].length, 1); + assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); + assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference)); + }; + const block = providerJobs[0]; + assertBoundary(block); + assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); + assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/); + assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/); + assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/); + for (const mutant of [ + block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), + block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), + block.replace(" - *guard\n", ""), + block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"), + block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"), + block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"), + ]) assert.throws(() => assertBoundary(mutant)); + continue; + } for (const block of providerJobs) { if (name === "runner-protocol-live-evals.yml") { assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}")); diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index c4dd568a72f..52209944424 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -320,6 +320,7 @@ test("the real serialized shard partition is complete, non-overlapping and durat const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts"; const nativeRunnerSuitePath = "server/src/services/native-runtime/native-codex-runner.integration.test.ts"; +const usageDeliverySuitePath = "server/src/services/native-runtime/native-usage-delivery.integration.test.ts"; const dotRunnerSuitePath = "server/src/__tests__/dot-runner.test.ts"; const restartRecoverySuitePath = "server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts"; @@ -363,7 +364,8 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes assert.ok(!files.includes(nativeRunnerSuitePath)); assert.ok(!files.includes(dotRunnerSuitePath)); assert.ok(!files.includes(restartRecoverySuitePath)); - assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); + assert.ok(!files.includes(usageDeliverySuitePath)); + assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath, dotRunnerSuitePath, restartRecoverySuitePath, usageDeliverySuitePath, "server/src/services/native-runtime/plugin-runner-probe.test.ts"].sort(), full.selectedGeneralServerSuites.sort()); assert.equal(new Set(files).size, files.length); const defaultRun = dryRunJson([], prEnv); assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); @@ -412,6 +414,7 @@ test("the native-runner lane runs exactly the cargo-dependent suites", () => { "server/src/services/native-runtime/native-codex-runner.integration.test.ts", dotRunnerSuitePath, restartRecoverySuitePath, + usageDeliverySuitePath, ]); }); diff --git a/scripts/lib/recovery-cutover-inventory.mjs b/scripts/lib/recovery-cutover-inventory.mjs new file mode 100644 index 00000000000..6c9d74d2b17 --- /dev/null +++ b/scripts/lib/recovery-cutover-inventory.mjs @@ -0,0 +1,99 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { classifyRecoveryEntry, cutoverStatus, recoveryKinds } from "./recovery-cutover-policy.mjs"; + +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +// Inspect every ancestor: lstat on only the final component would follow an +// intermediate symlink. No directory/file names or OS errors enter the report. +async function directoryState(directory, io) { + const parsed = path.parse(directory); + let current = parsed.root; + for (const part of directory.slice(parsed.root.length).split(path.sep).filter(Boolean)) { + current = path.join(current, part); + try { + const stat = await io.lstat(current); + if (!stat.isDirectory() || stat.isSymbolicLink()) return "unsafe"; + } catch (error) { + return error?.code === "ENOENT" ? "absent" : "unreadable"; + } + } + return "directory"; +} + +/** Read-only, bounded entry inventory. Run with producers stopped on the retained + * volume. This is not an atomic snapshot and does not establish DB settlement. + * io exists for deterministic fault tests; production uses the actual filesystem. */ +export async function inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries = 10_000 }, io = fs) { + if (!path.isAbsolute(instanceRoot) || (orphanDirectory !== undefined && !path.isAbsolute(orphanDirectory))) { + throw new Error("Recovery inventory requires absolute directories"); + } + if (!Number.isSafeInteger(maxEntries) || maxEntries < 1 || maxEntries > 100_000) { + throw new Error("Entry limit must be between 1 and 100000"); + } + const root = path.resolve(instanceRoot); + const rootState = await directoryState(root, io); + const directories = [path.join(root, "accounting-receipts"), path.join(root, "quota-credential-recovery"), + orphanDirectory === undefined ? path.join(root, "data", "sandbox-orphan-cleanup") : path.resolve(orphanDirectory)]; + let remaining = maxEntries; + const summaries = []; + for (const [index, directory] of directories.entries()) { + const summary = { kind: recoveryKinds[index], presence: "unknown", observedEntries: 0, + unverifiedFiles: 0, unknownEntries: 0, readFailures: 0, complete: true, status: "unknown" }; + if (rootState !== "directory") { + summary.complete = false; + } else { + const state = await directoryState(directory, io); + if (state === "absent") summary.presence = "absent"; + else if (state !== "directory") { + summary.unknownEntries++; + summary.complete = false; + if (state === "unreadable") summary.readFailures++; + } else { + summary.presence = "present"; + async function scan(target, depth) { + let handle; + try { + handle = await io.opendir(target); + while (true) { + // Read one sentinel entry to distinguish exactly-at-limit from + // truncated scans. The sentinel is not included in observed counts. + const entry = await handle.read(); + if (entry === null) break; + if (remaining === 0) { summary.complete = false; break; } + remaining--; + summary.observedEntries++; + const entryKind = entry.isFile() ? "file" + : entry.isDirectory() && uuid.test(entry.name) ? "company-directory" : "other"; + const classification = classifyRecoveryEntry(summary.kind, depth, entryKind); + if (classification === "descend") { + const child = path.join(target, entry.name); + // Recheck the entry before traversing; never intentionally follow + // links. Operators must quiesce writers for the snapshot to hold. + if (await directoryState(child, io) !== "directory") { + summary.unknownEntries++; + summary.complete = false; + } else await scan(child, depth + 1); + } else summary[classification]++; + if (!summary.complete && remaining === 0) break; + } + } catch { + summary.readFailures++; + summary.complete = false; + } finally { + if (handle) { + try { await handle.close(); } + catch { summary.readFailures++; summary.complete = false; } + } + } + } + await scan(directory, 0); + } + } + summary.status = cutoverStatus(summary); + summaries.push(summary); + } + return { schema: "paperclip/recovery-cutover-inventory/v1", status: + summaries.some(s => s.status === "unknown") ? "unknown" : summaries.some(s => s.status === "blocked") ? "blocked" : "clear", + instanceStatus: rootState, contentInspection: "none", counts: "observed-lower-bounds", maxEntries, directories: summaries }; +} diff --git a/scripts/lib/recovery-cutover-policy.mjs b/scripts/lib/recovery-cutover-policy.mjs new file mode 100644 index 00000000000..19f5a7568fe --- /dev/null +++ b/scripts/lib/recovery-cutover-policy.mjs @@ -0,0 +1,14 @@ +/** File contents belong to each recovery owner. Inventory never parses them: + * even an apparently valid file cannot establish that its obligation settled. */ +export const recoveryKinds = ["accounting-receipts", "quota-credential-recovery", "sandbox-orphan-cleanup"]; + +export function classifyRecoveryEntry(kind, depth, entry) { + if (entry === "file") return "unverifiedFiles"; + if (entry === "company-directory" && kind === "quota-credential-recovery" && depth === 0) return "descend"; + return "unknownEntries"; +} + +export function cutoverStatus(summary) { + if (!summary.complete || summary.unknownEntries > 0 || summary.readFailures > 0) return "unknown"; + return summary.unverifiedFiles > 0 ? "blocked" : "clear"; +} diff --git a/scripts/recovery-cutover.mjs b/scripts/recovery-cutover.mjs new file mode 100644 index 00000000000..138a5f17947 --- /dev/null +++ b/scripts/recovery-cutover.mjs @@ -0,0 +1,33 @@ +#!/usr/bin/env node +import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs"; + +const usage = "Usage: node scripts/recovery-cutover.mjs --instance-root /absolute/instance [--orphan-directory /absolute/override] [--max-entries 10000] [--check]"; +const args = process.argv.slice(2); +try { + let instanceRoot; + let orphanDirectory = process.env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; + let maxEntries = 10_000; + let check = false; + const seen = new Set(); + for (let i = 0; i < args.length; i++) { + const argument = args[i]; + if (seen.has(argument)) throw new Error("duplicate"); + seen.add(argument); + if (argument === "--check") check = true; + else if (["--instance-root", "--orphan-directory", "--max-entries"].includes(argument)) { + const value = args[++i]; + if (value === undefined || value.startsWith("--")) throw new Error("missing"); + if (argument === "--instance-root") instanceRoot = value; + else if (argument === "--orphan-directory") orphanDirectory = value; + else maxEntries = Number(value); + } else throw new Error("unknown"); + } + if (!instanceRoot) throw new Error("missing"); + const report = await inventoryRecoveryFiles({ instanceRoot, orphanDirectory, maxEntries }); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + if (check && report.status !== "clear") process.exitCode = 1; +} catch { + // Never echo args, filesystem errors or paths: they may contain secrets. + process.stderr.write(`Recovery inventory could not run. ${usage}\n`); + process.exitCode = 2; +} diff --git a/scripts/recovery-cutover.test.mjs b/scripts/recovery-cutover.test.mjs new file mode 100644 index 00000000000..56bd6a0998d --- /dev/null +++ b/scripts/recovery-cutover.test.mjs @@ -0,0 +1,163 @@ +import { afterEach, test } from "node:test"; +import assert from "node:assert/strict"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { inventoryRecoveryFiles } from "./lib/recovery-cutover-inventory.mjs"; +import { classifyRecoveryEntry, cutoverStatus } from "./lib/recovery-cutover-policy.mjs"; + +const cli = fileURLToPath(new URL("./recovery-cutover.mjs", import.meta.url)); +const roots = []; +const company = "00000000-0000-4000-8000-000000000001"; +const grant = "00000000-0000-4000-8000-000000000002"; +async function fixture() { + // Canonicalize macOS /var -> /private/var, since the check rejects links. + const root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "cutover-"))); + roots.push(root); + return root; +} +async function write(root, relative, content = "secret-token-must-never-appear") { + const file = path.join(root, relative); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, content); + return file; +} +function run(root, args = [], env = {}) { + return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], { + encoding: "utf8", env: { ...process.env, SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: "", ...env }, + }); +} +// An unset override differs from an invalid empty override. +function cleanRun(root, args = []) { + const env = { ...process.env }; + delete env.SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR; + return spawnSync(process.execPath, [cli, "--instance-root", root, ...args], { encoding: "utf8", env }); +} +afterEach(async () => { await Promise.all(roots.splice(0).map(root => fs.rm(root, { recursive: true, force: true }))); }); + +test("pure policy never clears files or unknown entries", () => { + assert.equal(classifyRecoveryEntry("accounting-receipts", 0, "file"), "unverifiedFiles"); + assert.equal(classifyRecoveryEntry("quota-credential-recovery", 0, "company-directory"), "descend"); + assert.equal(classifyRecoveryEntry("quota-credential-recovery", 1, "company-directory"), "unknownEntries"); + assert.equal(cutoverStatus({ complete: false, unknownEntries: 0, readFailures: 0, unverifiedFiles: 0 }), "unknown"); +}); + +test("CLI: absent and empty legacy layouts clear without migration", async () => { + const root = await fixture(); + let result = cleanRun(root, ["--check"]); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).status, "clear"); + for (const directory of ["accounting-receipts", `quota-credential-recovery/${company}`, "data/sandbox-orphan-cleanup"]) { + await fs.mkdir(path.join(root, directory), { recursive: true }); + } + result = cleanRun(root, ["--check"]); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).status, "clear"); +}); + +test("CLI: real pending legacy files block, preserve bytes and reveal no secrets or names", async () => { + const root = await fixture(); + const receipt = JSON.stringify({ schema: "paperclip/accounting-receipt/v1", id: grant, companyId: company, + runId: company, sourceId: company, sequence: 1, receivedAt: "2026-10-30T00:00:00.000Z", adapterType: "codex_local", + receipt: { complete: true, usage: { inputTokens: 1, outputTokens: 1 } } }); + const files = await Promise.all([ + write(root, `accounting-receipts/${grant}.json`, receipt), + write(root, `quota-credential-recovery/${company}/${grant}.json`, '{"ciphertext":"secret-token-must-never-appear"}'), + write(root, "data/sandbox-orphan-cleanup/secret-filename.json", JSON.stringify({ companyId: company, + environmentId: company, executionWorkspaceId: null, issueId: null, heartbeatRunId: null, provider: "daytona", + providerLeaseId: "private-resource", metadata: {} })), + ]); + const before = await Promise.all(files.map(file => fs.readFile(file))); + const result = cleanRun(root, ["--check"]); + assert.equal(result.status, 1); + const report = JSON.parse(result.stdout); + assert.equal(report.status, "blocked"); + assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 1]); + assert.doesNotMatch(result.stdout + result.stderr, /secret-token|secret-filename|private-resource|ciphertext|00000000-/); + assert.deepEqual(await Promise.all(files.map(file => fs.readFile(file))), before); + assert.equal(cleanRun(root).status, 0, "report-only mode does not fail on obligations"); +}); + +test("truncated, temporary, and unreadable files are unverified, never clear", async () => { + const root = await fixture(); + await write(root, "accounting-receipts/truncated.json", '{"schema":'); + const file = await write(root, `quota-credential-recovery/${company}/${grant}.json.probe`); + await fs.chmod(file, 0); + const report = await inventoryRecoveryFiles({ instanceRoot: root }); + assert.equal(report.status, "blocked"); + assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [1, 1, 0]); +}); + +test("unknown nesting and symlinks block without traversing outside the chosen roots", async () => { + const root = await fixture(); + const outside = await fixture(); + await write(outside, "do-not-read.json"); + await fs.mkdir(path.join(root, "accounting-receipts")); + await fs.symlink(outside, path.join(root, "accounting-receipts", "link")); + await fs.mkdir(path.join(root, "quota-credential-recovery", company, "unexpected"), { recursive: true }); + const report = await inventoryRecoveryFiles({ instanceRoot: root }); + assert.equal(report.status, "unknown"); + assert.deepEqual(report.directories.map(s => s.unknownEntries), [1, 1, 0]); + assert.deepEqual(report.directories.map(s => s.unverifiedFiles), [0, 0, 0]); + await fs.symlink(outside, path.join(root, "data")); + assert.equal((await inventoryRecoveryFiles({ instanceRoot: root })).directories[2].status, "unknown"); + await fs.symlink(outside, path.join(root, "instance-link")); + assert.equal((await inventoryRecoveryFiles({ instanceRoot: path.join(root, "instance-link") })).status, "unknown"); +}); + +test("entry budget gives incomplete lower-bound counts and never clears omitted work", async () => { + const root = await fixture(); + for (let i = 0; i < 4; i++) await write(root, `accounting-receipts/${i}.json`); + const report = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 2 }); + assert.equal(report.status, "unknown"); + assert.equal(report.directories[0].observedEntries, 2); + assert.equal(report.directories[0].complete, false); + assert.equal(report.directories[0].unverifiedFiles, 2); + const exact = await inventoryRecoveryFiles({ instanceRoot: root, maxEntries: 4 }); + assert.equal(exact.directories[0].complete, true); +}); + +test("CLI respects configured orphan override and explicit override precedence", async () => { + const root = await fixture(); + const override = await fixture(); + await write(override, "pending.json"); + let result = run(root, ["--check"], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override }); + assert.equal(result.status, 1); + assert.equal(JSON.parse(result.stdout).directories[2].unverifiedFiles, 1); + const emptyOverride = await fixture(); + result = run(root, ["--check", "--orphan-directory", emptyOverride], { SANDBOX_ORPHAN_CLEANUP_SPOOL_DIR: override }); + assert.equal(result.status, 0); + assert.equal(JSON.parse(result.stdout).status, "clear"); +}); + +test("fault boundary reports stat, directory read and close failures without error data", async () => { + const root = await fixture(); + await fs.mkdir(path.join(root, "accounting-receipts")); + for (const operation of ["lstat", "opendir", "read", "close"]) { + const sensitiveError = Object.assign(new Error("secret-token-must-never-appear"), { code: "EACCES" }); + const io = { ...fs }; + if (operation === "lstat" || operation === "opendir") io[operation] = async () => { throw sensitiveError; }; + else io.opendir = async () => ({ + read: async () => { if (operation === "read") throw sensitiveError; return null; }, + close: async () => { if (operation === "close") throw sensitiveError; }, + }); + const report = await inventoryRecoveryFiles({ instanceRoot: root }, io); + assert.equal(report.status, "unknown"); + if (operation === "lstat") assert.equal(report.instanceStatus, "unreadable"); + else assert.ok(report.directories[0].readFailures > 0); + assert.doesNotMatch(JSON.stringify(report), /secret-token|EACCES/); + } +}); + +test("CLI fails closed for missing instance, unsafe roots and bad arguments", async () => { + const root = await fixture(); + assert.equal(cleanRun(path.join(root, "absent"), ["--check"]).status, 1); + for (const args of [["--max-entries", "0"], ["--max-entries", "100001"], ["--max-entries", "NaN"], + ["--orphan-directory", "relative-secret"], ["--bad-secret"], ["--check", "--check"]]) { + const result = cleanRun(root, args); + assert.equal(result.status, 2); + assert.doesNotMatch(result.stdout + result.stderr, /relative-secret|bad-secret/); + } +}); diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index ce3ee317239..d72bd532e57 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -92,6 +92,7 @@ const nativeRunnerSuites = [ "server/src/__tests__/dot-runner.test.ts", // These process cases otherwise skip in server shards without Runner binaries. "server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts", + "server/src/services/native-runtime/native-usage-delivery.integration.test.ts", ]; // In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable // workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip diff --git a/server/scripts/migrate-run-evidence.ts b/server/scripts/migrate-run-evidence.ts new file mode 100644 index 00000000000..71454359053 --- /dev/null +++ b/server/scripts/migrate-run-evidence.ts @@ -0,0 +1,32 @@ +import { createDb } from "@paperclipai/db"; +import { migrateRunEvidence } from "../src/services/run-evidence-migration.js"; + +if (!process.env.DATABASE_URL) + throw new Error("Set DATABASE_URL to the intended instance; run with legacy writers drained"); +const args = process.argv.slice(2); +const flags = new Set(["--apply", "--company", "--after-run", "--after-trace", "--limit"]); +for (let index = 0; index < args.length; index++) { + const flag = args[index]!; + if (!flags.has(flag)) throw new Error(`Unknown migration option: ${flag}`); + if (flag !== "--apply" && (!args[++index] || args[index]!.startsWith("--"))) { + throw new Error(`Missing value for ${flag}`); + } +} +const value = (flag: string) => { + const i = args.indexOf(flag); + return i < 0 ? undefined : args[i + 1]; +}; +const db = createDb(process.env.DATABASE_URL); +try { + const result = await migrateRunEvidence(db, { + apply: args.includes("--apply"), + companyId: value("--company"), + afterRunId: value("--after-run"), + afterTraceId: value("--after-trace"), + limit: Number(value("--limit") || 50), + }); + console.log(JSON.stringify(result, null, 2)); + if (result.items.some((item) => item.state === "blocked")) process.exitCode = 1; +} finally { + await db.$client.end(); +} diff --git a/server/scripts/qualify-native-log-recovery.ts b/server/scripts/qualify-native-log-recovery.ts new file mode 100644 index 00000000000..8f726f59af6 --- /dev/null +++ b/server/scripts/qualify-native-log-recovery.ts @@ -0,0 +1,445 @@ +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { spawn, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { setTimeout as delay } from "node:timers/promises"; +import { eq, sql } from "drizzle-orm"; +import { createDb, heartbeatRuns } from "@paperclipai/db"; +import { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, +} from "@aws-sdk/client-s3"; +import { + DurablePrpControlPlane, + decodeDurablePrpCheckpoint, +} from "../src/vendor/paperclip-runner/index.js"; +import { createS3StorageProvider } from "../src/storage/s3-provider.js"; +import { createObjectRunLogStore } from "../src/services/object-run-log-store.js"; +import { createObjectByteJournal } from "../src/services/object-byte-journal.js"; +import { createHeartbeatRunOutput } from "../src/services/heartbeat/run-output.js"; +import { createAgentIdentityRedactor } from "../src/services/agent-identity-redaction.js"; +import { appendHeartbeatRunEvent } from "../src/services/heartbeat-run-events.js"; +import { PaperclipControlPlanePort } from "../src/services/native-runtime/paperclip-control-plane-port.js"; +import { formatNativeRunLogEvent } from "../src/services/native-runtime/native-run-log-export.js"; +import { createNativeRunLogTestTables } from "../src/__tests__/helpers/native-run-log-database.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +const databaseUrl = process.env.PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL; +if ( + !endpoint || + !databaseUrl || + [endpoint, databaseUrl].some((url) => !["127.0.0.1", "localhost"].includes(new URL(url).hostname)) +) + throw new Error("Qualification requires explicit disposable loopback S3 and Postgres endpoints"); +const childMode = process.argv[2] === "--controller"; +type Config = { + schema: string; + bucket: string; + checkpoint: string; + companyId: string; + agentId: string; + runId: string; + sessionId: string; + runnerInstanceId: string; + environmentLeaseId: string; + turnId: string; + itemId: string; +}; +const config: Config = childMode + ? JSON.parse(await fs.readFile(process.argv[3]!, "utf8")) + : { + schema: `native_log_proof_${randomUUID().replaceAll("-", "")}`, + bucket: `pc-native-proof-${randomUUID()}`, + checkpoint: "", + companyId: randomUUID(), + agentId: randomUUID(), + runId: randomUUID(), + sessionId: randomUUID(), + runnerInstanceId: randomUUID(), + environmentLeaseId: randomUUID(), + turnId: randomUUID(), + itemId: randomUUID(), + }; +const db = createDb(databaseUrl, { maxConnections: 1, idleTimeoutSeconds: 0 }); +await db.execute(sql`set client_min_messages=warning`); +const provider = createS3StorageProvider({ + endpoint, + bucket: config.bucket, + region: "us-east-1", + forcePathStyle: true, +}); +const logStore = () => + createObjectRunLogStore({ + provider, + keyPrefix: "logs", + flushMs: 30_000, + assertWriterAuthority: async (input) => { + await input.assertWriterAuthority?.(); + }, + }); +const identity = { + runnerInstanceId: config.runnerInstanceId, + environmentLeaseId: config.environmentLeaseId, + runId: config.runId, + normalizedSessionId: config.sessionId, + turnId: config.turnId, + itemId: config.itemId, +}; +const expectedRunnerVersion = "0.3.0", + expectedRunnerDigest = `sha256:${"c".repeat(64)}`; +async function waitFor(read: () => Promise, done: (value: T) => boolean, timeout = 30_000) { + const until = Date.now() + timeout; + while (Date.now() < until) { + const value = await read(); + if (done(value)) return value; + await delay(40); + } + throw new Error("Native log qualification deadline exceeded"); +} +if (childMode) { + const home = process.argv[4]!, + portNumber = Number(process.argv[5] || 0); + await db.execute(sql.raw(`set search_path=${config.schema}`)); + if (portNumber) { + const records = new Map( + JSON.parse(await fs.readFile(config.checkpoint, "utf8")), + ); + decodeDurablePrpCheckpoint(records).restoreSessionDirectory(home); + } + const store = logStore(); + const [row] = await db + .select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + runtimeMode: heartbeatRuns.runtimeMode, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, + logBytes: heartbeatRuns.logBytes, + lastOutputSeq: heartbeatRuns.lastOutputSeq, + lastOutputAt: heartbeatRuns.lastOutputAt, + }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, config.runId)); + type Run = typeof heartbeatRuns.$inferSelect; + const output = createHeartbeatRunOutput(db, { + run: row as Run, + runId: config.runId, + issueId: null, + runLogStore: store, + identityRedactor: createAgentIdentityRedactor(), + secretKeys: new Set(), + getCurrentUserRedactionOptions: async () => ({ enabled: false }), + appendRunEvent: async (_run, event) => { + await appendHeartbeatRunEvent(db, { + companyId: config.companyId, + agentId: config.agentId, + runId: config.runId, + ...event, + }); + }, + assertWriterAuthority: async () => { + const [current] = await db + .select({ status: heartbeatRuns.status }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, config.runId)); + if (current?.status !== "running") throw new Error("fixture_writer_revoked"); + }, + }); + const handlers = await output.begin(row as Run); + const record = async (event: Parameters[0], cursor: number) => { + await handlers.onPersistedNativeLog!(cursor, formatNativeRunLogEvent(event)); + }; + const persistence = new PaperclipControlPlanePort( + db, + { + ...config, + issueId: randomUUID(), + sourceInstanceId: config.runnerInstanceId, + controlPlaneSourceInstanceId: `control-${config.runId}`, + completionContractId: randomUUID(), + completionContractSha256: "qualification", + }, + { onCommittedEvent: record, onDuplicateEvent: record }, + ); + const core = new DurablePrpControlPlane({ + stateDirectory: path.join(home, "control-plane"), + identity, + expectedRunnerVersion, + expectedRunnerDigest, + onCommittedEvent: async (event) => { + await persistence.appendEvent(event); + }, + }); + core.setCheckpointPersistence(async (checkpoint) => { + const temporary = `${config.checkpoint}.pending`; + await fs.writeFile(temporary, JSON.stringify([...checkpoint.records])); + await fs.rename(temporary, config.checkpoint); + }); + await core.start(portNumber || undefined); + const ticket = core.issueBootstrapTicket(); + await core.flushCheckpoint(); + process.on("message", async (message: { id: string; type: string }) => { + try { + if (message.type === "checkpoint") await core.flushCheckpoint(); + else if (message.type === "finish-log") { + const result = await store.finalize(output.handle!); + await output.flushOutputProgress({ force: true }); + if (result.bytes === null) throw new Error("qualification_log_incomplete"); + } else { + core.queueCommand(message.type, {}, message.id, true); + await waitFor( + async () => core.getCommand(message.id)?.status, + (status) => status === "completed", + ); + } + process.send!({ id: message.id, ok: true }); + } catch (error) { + process.send!({ id: message.id, error: String(error) }); + } + }); + process.send!({ ready: true, connectUrl: core.connectUrl, ticket }); +} else { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "native-log-qualification-")); + config.checkpoint = path.join(root, "checkpoint.json"); + const configFile = path.join(root, "config.json"), + receiptFile = path.join(root, "receipt.json"), + releaseFile = path.join(root, "release"), + scriptFile = path.join(root, "script.json"); + const children: ChildProcess[] = []; + const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + let bucketCreated = false; + async function controller(home: string, portNumber = 0) { + await fs.mkdir(home, { mode: 0o700 }); + const child = spawn( + process.execPath, + [ + "--import", + path.resolve("server/node_modules/tsx/dist/loader.mjs"), + fileURLToPath(import.meta.url), + "--controller", + configFile, + home, + String(portNumber), + ], + { + cwd: process.cwd(), + env: { ...process.env, PAPERCLIP_HOME: home }, + stdio: ["ignore", "ignore", "pipe", "ipc"], + }, + ); + children.push(child); + let diagnostics = ""; + child.stderr!.on("data", (bytes) => { + diagnostics = (diagnostics + String(bytes)).slice(-8000); + }); + const ready = await new Promise<{ connectUrl: string; ticket: string }>((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error(`Controller startup timeout: ${diagnostics}`)), + 45_000, + ); + child.on("message", (message: { ready?: boolean; connectUrl: string; ticket: string }) => { + if (message.ready) { + clearTimeout(timeout); + resolve(message); + } + }); + child.once("exit", () => { + clearTimeout(timeout); + reject(new Error(`Controller startup failed: ${diagnostics}`)); + }); + }); + async function command(type: string) { + const id = randomUUID(); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + child.off("message", onMessage); + reject(new Error(`Command timeout: ${type}: ${diagnostics}`)); + }, 35_000); + function onMessage(message: { id: string; error?: string }) { + if (message.id !== id) return; + clearTimeout(timer); + child.off("message", onMessage); + message.error ? reject(new Error(message.error)) : resolve(); + } + child.on("message", onMessage); + child.send({ id, type }); + }); + } + return { child, ready, command }; + } + try { + await db.execute(sql.raw(`create schema ${config.schema}`)); + await db.execute(sql.raw(`set search_path=${config.schema}`)); + await createNativeRunLogTestTables(db); + await db.execute( + sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${config.runId},${config.companyId},${config.agentId},'running')`, + ); + await client.send(new CreateBucketCommand({ Bucket: config.bucket })); + bucketCreated = true; + await fs.writeFile(configFile, JSON.stringify(config)); + await fs.writeFile( + scriptFile, + JSON.stringify({ steps: 64, waitAfter: 32, retentionLimit: 8, receiptFile, releaseFile }), + ); + const homeA = path.join(root, "web-a"), + a = await controller(homeA); + const runner = spawn( + path.resolve("packages/paperclip-runner/runner/target/debug/examples/recovery-fixture"), + [ + "--fixture-script", + scriptFile, + "--connect-url", + a.ready.connectUrl, + "--state-dir", + path.join(root, "runner"), + "--runner-id", + config.runnerInstanceId, + "--environment-lease-id", + config.environmentLeaseId, + "--run-id", + config.runId, + "--session-id", + config.sessionId, + "--turn-id", + config.turnId, + "--item-id", + config.itemId, + "--runner-version", + expectedRunnerVersion, + "--runner-digest", + expectedRunnerDigest, + "--max-outbox-bytes", + "65536", + "--p0-reserve-bytes", + "16384", + "--reconnect-grace-ms", + "60000", + ], + { + env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: a.ready.ticket }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + children.push(runner); + let runnerDiagnostics = ""; + runner.stderr!.on("data", (bytes) => { + runnerDiagnostics = (runnerDiagnostics + String(bytes)).slice(-4000); + }); + const receipt = async () => + JSON.parse(await fs.readFile(receiptFile, "utf8").catch(() => "{}")) as { + pid: number; + acknowledgedEvents: number; + completedSteps: number; + retainedEvents: number; + peakRetainedEvents: number; + }; + await a.command("turn.start"); + await fs.writeFile(releaseFile, "go"); + const acknowledged = await waitFor( + receipt, + (value) => value.acknowledgedEvents === 32 && value.retainedEvents === 0, + ); + const journal = createObjectByteJournal({ + provider, + prefix: `logs/segments-v1/${config.companyId}/${config.agentId}/${config.runId}`, + }); + const before = (await journal.head())!.head; + assert( + before.sourceCursor < 32, + "Experiment must kill a controller with acknowledged output still pending in S3", + ); + const controlStarted = performance.now(); + await a.command("session.snapshot"); + const controlMs = performance.now() - controlStarted; + await a.command("checkpoint"); + const exitA = once(a.child, "exit"); + a.child.kill("SIGKILL"); + await exitA; + await fs.rm(homeA, { recursive: true }); + const b = await controller(path.join(root, "web-b"), Number(new URL(a.ready.connectUrl).port)); + await b.command("turn.steer"); + const completed = await waitFor( + receipt, + (value) => value.acknowledgedEvents === 64 && value.retainedEvents === 0, + ); + assert.equal(completed.pid, acknowledged.pid); + assert.equal(runner.exitCode, null, runnerDiagnostics); + const stopStarted = performance.now(); + await b.command("turn.stop"); + const stopMs = performance.now() - stopStarted; + await b.command("finish-log"); + const store = logStore(), + content = ( + await store.read({ + store: "s3_segments", + logRef: `${config.companyId}/${config.agentId}/${config.runId}`, + }) + ).content; + const events = content + .trim() + .split("\n") + .map((line) => JSON.parse(JSON.parse(line).chunk).event); + assert.deepEqual( + events.map((event) => Number(event.payload.text.match(/^step=(\d+)/)[1])), + Array.from({ length: 64 }, (_, index) => index + 1), + ); + assert.equal(new Set(events.map((event) => event.sourceEventId)).size, 64); + const [source] = await db.execute( + sql`select count(*)::int as count from heartbeat_run_events where run_id=${config.runId}`, + ); + assert.equal(source!.count, 64); + const shutdown = once(runner, "exit"); + await b.command("runner.shutdown"); + assert.equal((await shutdown)[0], 0); + process.stdout.write( + `${JSON.stringify({ + result: "passed", + source: "real Rust recovery fixture and PaperclipControlPlanePort", + acknowledgedBeforeKill: 32, + objectSourceCursorBeforeKill: before.sourceCursor, + controllerLoss: "SIGKILL", + oldDirectoryRemoved: true, + sameRunnerProcess: true, + recoveredRecords: events.length, + duplicates: 0, + existingControlRows: source!.count, + controlCommandMs: Math.round(controlMs), + stopMs: Math.round(stopMs), + peakRunnerRetained: completed.peakRetainedEvents, + })}\n`, + ); + } finally { + for (const child of children) + if (child.exitCode === null && child.signalCode === null) { + const exit = once(child, "exit"); + child.kill("SIGKILL"); + await exit; + } + if (bucketCreated) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: config.bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: config.bucket, + Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: config.bucket })); + } + client.destroy(); + await db.execute(sql.raw(`drop schema if exists ${config.schema} cascade`)); + await db.$client.end(); + await fs.rm(root, { recursive: true, force: true }); + } +} diff --git a/server/scripts/qualify-object-run-logs.ts b/server/scripts/qualify-object-run-logs.ts new file mode 100644 index 00000000000..2890894c999 --- /dev/null +++ b/server/scripts/qualify-object-run-logs.ts @@ -0,0 +1,213 @@ +import { randomUUID } from "node:crypto"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, +} from "@aws-sdk/client-s3"; +import { createS3StorageProvider } from "../src/storage/s3-provider.js"; +import { createObjectRunLogStore } from "../src/services/object-run-log-store.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +if (!endpoint || !["127.0.0.1", "localhost"].includes(new URL(endpoint).hostname)) { + throw new Error( + "Qualification requires PAPERCLIP_LOG_TEST_S3_ENDPOINT on a disposable loopback backend", + ); +} +const childMode = process.argv[2] === "--child-writer"; +const bucket = childMode ? process.argv[3]! : `paperclip-logs-test-${randomUUID()}`; +const raw = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, +}); +const counts = { get: 0, put: 0, uploadedBytes: 0 }; +const latencies: number[] = []; +const provider = { + ...raw, + async getObject(...args: Parameters) { + counts.get++; + return raw.getObject(...args); + }, + async putObjectConditional(...args: Parameters>) { + counts.put++; + counts.uploadedBytes += args[0].contentLength; + const start = performance.now(); + try { + return await raw.putObjectConditional!(...args); + } finally { + latencies.push(performance.now() - start); + } + }, +}; +const input = { companyId: "company", agentId: "agent", runId: "crash" }; +const options = { + provider, + keyPrefix: "qualification", + flushMs: 1000, + assertWriterAuthority: async () => undefined, +}; +const output = (seq: number) => ({ + stream: "stdout" as const, + seq, + chunk: `${seq}: 再開😀 ${"x".repeat(900)}`, + ts: "2026-10-30T00:00:00Z", +}); +if (childMode) { + const store = createObjectRunLogStore(options), + handle = await store.begin(input); + const before = counts.put; + await Promise.all(Array.from({ length: 1000 }, (_, i) => store.append(handle, output(i + 1)))); + process.stdout.write( + JSON.stringify({ receipt: true, batchPuts: counts.put - before, counts }) + "\n", + ); + setTimeout(() => process.exit(2), 30_000); +} else { + const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + const directory = await fs.mkdtemp(path.join(tmpdir(), "run-log-crash-")); + let child: ReturnType | undefined; + try { + const started = performance.now(); + child = spawn( + process.execPath, + [ + "--import", + path.resolve("server/node_modules/tsx/dist/loader.mjs"), + import.meta.filename, + "--child-writer", + bucket, + ], + { cwd: directory, env: process.env, stdio: ["ignore", "pipe", "pipe"] }, + ); + const exited = once(child, "exit"); + let childError = ""; + child.stderr!.on("data", (bytes) => { + childError = (childError + bytes).slice(-8192); + }); + const receipt = await new Promise<{ batchPuts: number; counts: typeof counts }>( + (resolve, reject) => { + let text = ""; + const timeout = setTimeout( + () => reject(new Error(`Writer receipt timed out: ${childError}`)), + 30_000, + ); + child!.stdout!.on("data", (bytes) => { + text += bytes; + if (!text.includes("\n")) return; + clearTimeout(timeout); + try { + resolve(JSON.parse(text.split("\n")[0]!)); + } catch (error) { + reject(error); + } + }); + child!.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + void exited.then(() => { + clearTimeout(timeout); + if (!text) reject(new Error(`Writer exited before its receipt: ${childError}`)); + }); + }, + ); + child.kill("SIGKILL"); + const [, signal] = await exited; + if (signal !== "SIGKILL") throw new Error("Writer did not die by SIGKILL"); + await fs.rm(directory, { recursive: true, force: true }); + const successorStore = createObjectRunLogStore(options); + const successor = await successorStore.begin(input); + let content = "", + offset = 0; + for (;;) { + const page = await successorStore.read(successor, { offset, limitBytes: 131072 }); + content += page.content; + if (page.nextOffset === undefined) break; + offset = page.nextOffset; + } + const lines = content + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + if ( + lines.length !== 1000 || + !lines.every((line, i) => line.seq === i + 1 && line.chunk === output(i + 1).chunk) + ) + throw new Error("Replacement bytes did not match"); + await successorStore.append(successor, output(1001)); + const summary = await successorStore.finalize(successor); + + const steadyBefore = { ...counts }, + steadyStarted = performance.now(); + const steady = createObjectRunLogStore(options); + const handles = await Promise.all( + Array.from({ length: 8 }, (_, i) => steady.begin({ ...input, runId: `steady-${i}` })), + ); + const pending: Promise[] = []; + for (let tick = 0; tick < 30; tick++) { + for (const handle of handles) pending.push(steady.append(handle, output(tick + 1))); + await delay(100); + } + await Promise.all(pending); + await Promise.all(handles.map((handle) => steady.finalize(handle))); + const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (objects.IsTruncated) throw new Error("Qualification inventory exceeded one bounded page"); + latencies.sort((a, b) => a - b); + console.log( + JSON.stringify( + { + backend: "loopback S3-compatible test service", + flushMs: options.flushMs, + burst: { + records: lines.length, + batchPuts: receipt.batchPuts, + writerCounts: receipt.counts, + }, + steady: { + streams: handles.length, + records: pending.length, + elapsedMs: Math.round(performance.now() - steadyStarted), + putRequests: counts.put - steadyBefore.put, + getRequests: counts.get - steadyBefore.get, + uploadedBytes: counts.uploadedBytes - steadyBefore.uploadedBytes, + }, + readerCounts: counts, + committedCrashBytes: summary.bytes, + objects: objects.KeyCount, + storedBytes: objects.Contents?.reduce((sum, object) => sum + (object.Size ?? 0), 0), + putP95Ms: Math.round(latencies[Math.floor(latencies.length * 0.95)] ?? 0), + elapsedMs: Math.round(performance.now() - started), + abruptProcessLoss: signal, + oldDirectoryRemoved: true, + replacement: "passed", + }, + null, + 2, + ), + ); + } finally { + child?.kill("SIGKILL"); + await fs.rm(directory, { recursive: true, force: true }); + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket, MaxKeys: 1000 })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + } +} diff --git a/server/src/__tests__/activity-routes.test.ts b/server/src/__tests__/activity-routes.test.ts index e8f7d67d258..0c507b0a157 100644 --- a/server/src/__tests__/activity-routes.test.ts +++ b/server/src/__tests__/activity-routes.test.ts @@ -333,6 +333,20 @@ describe("activity routes", () => { expect(res.body).toEqual([{ runId: "run-1", adapterType: "codex_local" }]); }); + it("validates bounded run IDs before using the authorized issue projection", async () => { + const runId = "10000000-0000-4000-8000-000000000001"; + mockIssueService.getByIdentifier.mockResolvedValue({ id: "issue-uuid-1", companyId: "company-1" }); + mockActivityService.runsForIssue.mockResolvedValue([{ runId }]); + const app = await createApp(); + const response = await requestApp(app, base => request(base).get(`/api/issues/PAP-1/runs?runIds=${runId}`)); + expect(response.status).toBe(200); + expect(mockActivityService.runsForIssue).toHaveBeenCalledWith("company-1", "issue-uuid-1", "user-1", undefined, [runId]); + for (const ids of ["", "not-a-uuid", Array.from({ length: 101 }, () => runId).join(",")]) { + const denied = await requestApp(app, base => request(base).get("/api/issues/PAP-1/runs").query({ runIds: ids })); + expect(denied.status).toBe(400); + } + }); + it("requires company access before creating activity events", async () => { const app = await createApp(); const res = await requestApp(app, (baseUrl) => request(baseUrl) diff --git a/server/src/__tests__/agent-identity.test.ts b/server/src/__tests__/agent-identity.test.ts index d0e7dcbe246..69a29a40049 100644 --- a/server/src/__tests__/agent-identity.test.ts +++ b/server/src/__tests__/agent-identity.test.ts @@ -10,6 +10,7 @@ import { eq } from "drizzle-orm"; import { agentIdentityKeys, agents, companies, createDb, closeRegisteredClients, ensurePostgresDatabase, runDatabaseBackup, runDatabaseRestore, type Db } from "@paperclipai/db"; import { resolveWorktreeSeedPlan } from "../../../cli/src/commands/worktree-lib.js"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { agentIdentityService, supportsManagedAgentIdentity } from "../services/agent-identity.js"; import { agentService } from "../services/agents.js"; import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; @@ -21,9 +22,10 @@ import { actorMiddleware } from "../middleware/auth.js"; import { errorHandler } from "../middleware/error-handler.js"; import { drainHeartbeatRunsToQuiescence } from "./helpers/drain-heartbeat-runs.js"; +const externalIdentityDatabase = resolveExternalTestDatabaseUrl("PAPERCLIP_AGENT_IDENTITY_TEST_DATABASE_URL"); describe("agent cryptographic identity", () => { let db: Db; - let database: Awaited>; + let database: Pick>, "connectionString" | "cleanup">; let directory: string; let companyId: string; @@ -32,7 +34,9 @@ describe("agent cryptographic identity", () => { vi.stubEnv("PAPERCLIP_HOME", directory); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", ""); vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", join(directory, "master.key")); - database = await startEmbeddedPostgresTestDatabase("paperclip-identity-"); + database = externalIdentityDatabase ? { connectionString: externalIdentityDatabase, + cleanup: () => closeRegisteredClients(externalIdentityDatabase) } + : await startEmbeddedPostgresTestDatabase("paperclip-identity-"); db = createDb(database.connectionString); companyId = randomUUID(); await db.insert(companies).values({ id: companyId, name: "Identity test", issuePrefix: "KEY", defaultResponsibleUserId: "identity-test-owner" }); @@ -51,6 +55,7 @@ describe("agent cryptographic identity", () => { it("migration and public reads leave existing agents unprovisioned", async () => { const agent = await oldAgent(); expect(await agentIdentityService(db).getPublicIdentity(companyId, agent.id)).toBeNull(); + expect((await agentIdentityService(db).createExistingLogRedactor(companyId, agent.id)).values).toEqual([]); expect(await db.select().from(agentIdentityKeys).where(eq(agentIdentityKeys.agentId, agent.id))).toHaveLength(0); for (const type of ["http", "openclaw_gateway", "hermes_gateway"]) expect(supportsManagedAgentIdentity(type)).toBe(false); for (const provider of ["claude_managed", "aws_agentcore"]) expect(supportsManagedAgentIdentity("paperclip_runner", { provider })).toBe(false); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index 463a6463c25..e408200aa50 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -9,7 +9,8 @@ vi.mock("../services/subscription-refresh.js", () => ({ refreshSubscriptionConne import * as localCredentials from "../services/local-ai-credentials.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createHash, randomUUID } from "node:crypto"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; import { promises as fs } from "node:fs"; import { mkdtemp, realpath, rm, access, readFile, writeFile, stat } from "node:fs/promises"; @@ -38,6 +39,8 @@ import { secretService } from "../services/secrets.js"; import { resolveExecutionRunAdapterConfig } from "../services/heartbeat.js"; import { aiConnectionBindingSchema, connectionPurposeTransportSchema, isAiConnectionCompatible } from "@paperclipai/shared"; import { openRouterClaudeHeaders } from "@paperclipai/shared/openrouter-attribution"; +import { createServer } from "node:http"; +import { once } from "node:events"; import express from "express"; import request from "supertest"; import { aiConnectionRoutes, canInstallSharedAiConnectionForNewAgent, responsibleUserForAiRequest } from "../routes/ai-connections.js"; @@ -385,8 +388,8 @@ describe("managed AI connections", () => { await expectSelectionFailure(service.select({ ...input, userId: owner }), "ai_connection_unavailable"); }); - it("persists first-time recovery directories before consuming a single-use token", async () => { - const owner = "quota-durable-path"; + it("commits one exchanged token without writable credential recovery storage", async () => { + const owner = "quota-database-first"; await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); const original = JSON.stringify({ tokens: { account_id: owner, access_token: "expired", refresh_token: "single-use" } }); await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original); @@ -394,32 +397,32 @@ describe("managed AI connections", () => { const file = quotaCredentialRecoveryPath(companyId, row.grant.id); const recoveryRoot = path.dirname(path.dirname(file)); await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" }); - const synced: string[] = []; - let rejectRoot = true; - const realOpen = fs.open.bind(fs); + const writes: string[] = []; + const rejectRecoveryWrite = (target: unknown) => { + if (String(target).startsWith(recoveryRoot)) { + writes.push(String(target)); + throw Object.assign(new Error("Fixture recovery storage is read-only"), { code: "EROFS" }); + } + }; + const realOpen = fs.open.bind(fs), realMkdir = fs.mkdir.bind(fs); const open = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters) => { - const handle = await realOpen(...args); - const sync = handle.sync.bind(handle); - vi.spyOn(handle, "sync").mockImplementation(async () => { - if (rejectRoot && args[0] === recoveryRoot) throw new Error("Injected recovery directory flush failure"); - await sync(); synced.push(String(args[0])); - }); - return handle; + rejectRecoveryWrite(args[0]); + return realOpen(...args); }); - const request = vi.spyOn(globalThis, "fetch").mockImplementation(async () => { - expect(synced).toContain(recoveryRoot); - expect(synced).toContain(path.dirname(recoveryRoot)); - return Response.json({ access_token: "fresh-durable", refresh_token: "replacement-durable" }); + const mkdir = vi.spyOn(fs, "mkdir").mockImplementation(async (...args: Parameters) => { + rejectRecoveryWrite(args[0]); + return realMkdir(...args); }); + const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "fresh-direct", refresh_token: "replacement-direct" })); try { - await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow("Injected recovery directory flush failure"); - expect(request).not.toHaveBeenCalled(); - expect(await service.credential(row)).toBe(original); - rejectRoot = false; - await service.refreshQuotaCredential(row, original, new AbortController().signal); + const refreshed = await service.refreshQuotaCredential(row, original, new AbortController().signal); + expect(JSON.parse(refreshed)).toMatchObject({ tokens: { access_token: "fresh-direct", refresh_token: "replacement-direct" } }); + expect(await service.credential(row)).toBe(refreshed); expect(request).toHaveBeenCalledTimes(1); - expect(JSON.parse(await service.credential(row))).toMatchObject({ tokens: { access_token: "fresh-durable", refresh_token: "replacement-durable" } }); - } finally { open.mockRestore(); request.mockRestore(); } + expect(writes).toEqual([]); + await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(access(recoveryRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { open.mockRestore(); mkdir.mockRestore(); request.mockRestore(); } }); it.each(["timeout", "unexpected"] as const)("classifies a manual probe's %s lock failure before starting the provider", async (kind) => { @@ -703,7 +706,7 @@ describe("managed AI connections", () => { } finally { request.mockRestore(); } }); - it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("recovers exchanged OAuth tokens after %s persistence failure without another exchange", async (failure) => { + it.each(["rotation", "secret_metadata", "activity", "commit"] as const)("fails truthfully after %s persistence failure and permits sign-in recovery without a journal", async (failure) => { const owner = `quota-rollback-${failure}`; await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); const original = JSON.stringify({ tokens: { access_token: `expired-${failure}`, refresh_token: "single-use", account_id: owner } }); @@ -723,17 +726,25 @@ describe("managed AI connections", () => { await expect(service.refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow(); expect(request).toHaveBeenCalledTimes(1); expect(await service.credential(row)).toBe(original); - const encrypted = await readFile(file, "utf8"); - expect(encrypted).not.toContain(`fresh-${failure}`); - expect(encrypted).not.toContain(`replacement-${failure}`); + await expect(access(file)).rejects.toMatchObject({ code: "ENOENT" }); await db.execute(sql.raw(`drop trigger quota_save_failure on ${table}; drop function quota_save_failure();`)); triggerDropped = true; - // A new service instance recovers from disk, without an in-memory cache. - const recovered = aiConnectionService(db); - await recovered.refreshQuotaCredential(row, original, new AbortController().signal); - expect(JSON.parse(await recovered.credential(row))).toMatchObject({ tokens: { access_token: `fresh-${failure}`, refresh_token: `replacement-${failure}` } }); + // A new receiver has no replacement to recover. A failed use routes to the + // existing sign-in flow, rather than claiming the failed save succeeded. + const receiver = aiConnectionService(db); + const generation = createHash("sha256").update(original).digest("hex").slice(0, 16); + await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: { + provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id, + mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`, + } }); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization"); + const signedIn = JSON.stringify({ tokens: { access_token: "signed-in", refresh_token: "new-login" } }); + await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", + ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn); + const [current] = await receiver.quotaAccounts(companyId, owner); + expect(current.grant.status).toBe("active"); + expect(await receiver.runtimeCredential(current)).toBe(signedIn); expect(request).toHaveBeenCalledTimes(1); - await expect(access(file)).rejects.toThrow(); expect((await db.select().from(companySecretVersions).where(eq(companySecretVersions.secretId, ref.secretId))).map(version => version.version).sort()).toEqual([1, 2]); } finally { request.mockRestore(); @@ -819,6 +830,112 @@ describe("managed AI connections", () => { } finally { request.mockRestore(); await oldRun.cleanup(); await nextRun?.cleanup(); await journal.clear(); } }); + it("requires sign-in after the receiver process dies with an exchanged but unsaved token", async () => { + const owner = "quota-receiver-process-loss"; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + const original = JSON.stringify({ tokens: { access_token: "expired-process", refresh_token: "single-use-process" } }); + await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original); + const [row] = await service.quotaAccounts(companyId, owner); + const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!; + const held = deferredSignal(), release = deferredSignal(); + const lockKey = 7192031; + const holder = db.transaction(async tx => { + await tx.execute(sql`select pg_advisory_xact_lock(${lockKey})`); + held.resolve(); await release.promise; + }); + await held.promise; + await db.execute(sql.raw(`create function hold_receiver_save() returns trigger language plpgsql as $$ begin + if new.secret_id = '${ref.secretId}'::uuid then perform pg_advisory_xact_lock(${lockKey}); end if; return new; end $$; + create trigger hold_receiver_save before insert on company_secret_versions for each row execute function hold_receiver_save();`)); + let exchanges = 0; + const provider = createServer((req, res) => { + exchanges++; req.resume(); + res.setHeader("Content-Type", "application/json"); + res.end(JSON.stringify({ access_token: "unsaved-process", refresh_token: "replacement-process" })); + }); + provider.listen(0, "127.0.0.1"); await once(provider, "listening"); + const address = provider.address() as { port: number }; + const child = spawn(process.execPath, ["--import", createRequire(import.meta.url).resolve("tsx"), new URL("./helpers/quota-credential-process.ts", import.meta.url).pathname], { + env: { ...process.env, CREDENTIAL_TEST_DATABASE_URL: database.connectionString, + CREDENTIAL_TEST_PROVIDER_URL: `http://127.0.0.1:${address.port}`, CREDENTIAL_TEST_COMPANY_ID: companyId, CREDENTIAL_TEST_OWNER: owner }, + stdio: "ignore", + }); + const exited = once(child, "exit"); + try { + let blocked = false; + const deadline = Date.now() + 25_000; + while (!blocked && Date.now() < deadline && child.exitCode === null) { + const rows = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() + and wait_event_type = 'Lock' and wait_event = 'advisory'`); + blocked = exchanges === 1 && rows.length > 0; + if (!blocked) await new Promise(resolve => setTimeout(resolve, 25)); + } + expect(blocked).toBe(true); + child.kill("SIGKILL"); await exited; + } finally { + if (child.exitCode === null && child.signalCode === null) { child.kill("SIGKILL"); await exited; } + release.resolve(); await holder; + await db.execute(sql`drop trigger hold_receiver_save on company_secret_versions; drop function hold_receiver_save()`); + await new Promise(resolve => provider.close(() => resolve())); + } + expect(exchanges).toBe(1); + expect(await service.credential(row)).toBe(original); + await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" }); + const receiver = aiConnectionService(db); + const generation = createHash("sha256").update(original).digest("hex").slice(0, 16); + await receiver.markAuthenticationFailed({ companyId, runStartedAt: new Date(), attribution: { + provider: "openai", method: "subscription", connectionId: row.connection.id, grantId: row.grant.id, + mode: "responsible_user", responsibleUserId: owner, identity: `${row.grant.id}:${owner}:${generation}`, + } }); + expect((await db.select().from(connectionGrants).where(eq(connectionGrants.id, row.grant.id)))[0].status).toBe("needs_reauthorization"); + const signedIn = JSON.stringify({ tokens: { access_token: "process-sign-in", refresh_token: "process-new-login" } }); + await receiver.save(companyId, owner, { connectionId: row.connection.id, provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, signedIn); + const [current] = await receiver.quotaAccounts(companyId, owner); + expect(await receiver.runtimeCredential(current)).toBe(signedIn); + }, 40_000); + + it("does not exchange a token when a required database transaction fails", async () => { + const owner = "quota-db-prerequisite"; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } }); + await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original); + const [row] = await service.quotaAccounts(companyId, owner); + const request = vi.spyOn(globalThis, "fetch"); + const unavailableUrl = new URL(database.connectionString); + unavailableUrl.port = "1"; + const unavailable = createDb(unavailableUrl.toString()); + try { + await expect(aiConnectionService(unavailable).refreshQuotaCredential(row, original, new AbortController().signal)).rejects.toThrow(); + expect(request).not.toHaveBeenCalled(); + await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" }); + } finally { await unavailable.$client.end(); request.mockRestore(); } + }); + + it.each(["reconnect", "same_value_new_epoch", "revoked"] as const)("lets %s win after a returned token's save rolls back", async change => { + const owner = `quota-delayed-save-${change}`; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + const original = JSON.stringify({ tokens: { access_token: "expired", refresh_token: "single-use" } }); + await service.save(companyId, owner, { provider: "openai", method: "subscription", ownership: "personal", name: owner, loginSessionId: "fixture", agentIds: [], allAgents: true }, original); + const [row] = await service.quotaAccounts(companyId, owner); + const ref = row.grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential")!; + const transaction = db.transaction.bind(db); + const interrupted = vi.spyOn(db, "transaction").mockImplementationOnce(async (work, config) => { + await transaction(async tx => { await work(tx); throw new Error("fixture rollback"); }, config).catch(() => {}); + if (change === "revoked") await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, row.grant.id)); + else await secretService(db).rotate(ref.secretId, { value: change === "reconnect" ? "new-login" : original }); + throw new Error("fixture lost response"); + }); + const request = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ access_token: "late", refresh_token: "late-refresh" })); + try { + const refresh = service.refreshQuotaCredential(row, original, new AbortController().signal); + if (change === "revoked") await expect(refresh).rejects.toThrow("credentials_unavailable"); + else expect(await refresh).toBe(change === "reconnect" ? "new-login" : original); + expect(await service.credential(row)).toBe(change === "reconnect" ? "new-login" : original); + expect(request).toHaveBeenCalledTimes(1); + await expect(access(quotaCredentialRecoveryPath(companyId, row.grant.id))).rejects.toMatchObject({ code: "ENOENT" }); + } finally { interrupted.mockRestore(); request.mockRestore(); } + }); + it.each(["rollback", "lost_commit_reply"] as const)("automatically saves one exchanged token across %s", async (failure) => { const owner = `quota-once-${failure}`; await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); @@ -863,7 +980,10 @@ describe("managed AI connections", () => { triggerInstalled = true; await expect(service.refreshQuotaCredential(row, current, new AbortController().signal)).rejects.toThrow(); expect(await service.credential(row)).toBe(current); - expect(JSON.parse((await journal.read())!.value).tokens.access_token).toBe("C"); + expect(await journal.read()).toBeNull(); + // Only observed records from the earlier release use the legacy reader. + await journal.write({ companyId, grantId: row.grant.id, connectionId: row.connection.id, secretId: ref.secretId, + baseHash: quotaCredentialHash(current), value: JSON.stringify({ tokens: { access_token: "C", refresh_token: "refresh-C" } }) }); await db.execute(sql`drop trigger delayed_quota_commit_failure on company_secret_versions; drop function delayed_quota_commit_failure()`); triggerInstalled = false; const recovered = await service.refreshQuotaCredential(row, original, new AbortController().signal); diff --git a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts index 572e721996a..c610d17ded9 100644 --- a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts +++ b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts @@ -32,8 +32,10 @@ import { activityRoutes } from "../routes/activity.js"; import { agentRoutes } from "../routes/agents.js"; import { authorizationService } from "../services/authorization.js"; import { canActorReadWorkspaceOperation, getHeartbeatRunReadDecisions } from "../services/heartbeat-run-privacy.js"; +import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; -const embeddedPostgresSupport = await getEmbeddedPostgresTestSupport(); +const externalDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_LOG_PRIVACY_TEST_DATABASE_URL"); +const embeddedPostgresSupport = externalDatabaseUrl ? { supported: true } : await getEmbeddedPostgresTestSupport(); const describeEmbeddedPostgres = embeddedPostgresSupport.supported ? describe : describe.skip; describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, () => { @@ -42,8 +44,8 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, const previousPrivacyMode = process.env.PAPERCLIP_ISSUE_PRIVACY_MODE; beforeAll(async () => { - tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-"); - db = createDb(tempDb.connectionString); + if (!externalDatabaseUrl) tempDb = await startEmbeddedPostgresTestDatabase("paperclip-run-privacy-routes-"); + db = createDb(externalDatabaseUrl ?? tempDb!.connectionString); process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = "enforce"; }, 120_000); @@ -66,6 +68,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, if (previousPrivacyMode === undefined) delete process.env.PAPERCLIP_ISSUE_PRIVACY_MODE; else process.env.PAPERCLIP_ISSUE_PRIVACY_MODE = previousPrivacyMode; await tempDb?.cleanup(); + if (externalDatabaseUrl) await db.$client.end({ timeout: 1 }); }); async function seedFixture() { @@ -264,7 +267,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, runnerProfileJson: { instructions: payload }, stdoutExcerpt: payload, stderrExcerpt: payload, }).where(eq(heartbeatRuns.id, f.privateRunId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId)); - const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); + const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array> = [], now = Date.now(); const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { @@ -281,8 +284,9 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, } finally { capture.flush(); await observed.$client.end(); } }); - it("returns 404 for direct private-run content and metadata-only list stubs to a non-member", async () => { + it.each(["local_file", "s3_segments"])("returns 404 for private %s run content and metadata-only list stubs", async (logStore) => { const fixture = await seedFixture(); + await db.update(heartbeatRuns).set({ logStore }).where(eq(heartbeatRuns.id, fixture.privateRunId)); const app = createApp(fixture.companyId, fixture.otherAgentId); for (const path of [ diff --git a/server/src/__tests__/helpers/native-run-log-database.ts b/server/src/__tests__/helpers/native-run-log-database.ts new file mode 100644 index 00000000000..a19b5c2b5ae --- /dev/null +++ b/server/src/__tests__/helpers/native-run-log-database.ts @@ -0,0 +1,22 @@ +import { sql } from "drizzle-orm"; +import type { Db } from "@paperclipai/db"; + +export async function createNativeRunLogTestTables(db: Db) { + await db.execute(sql`create table heartbeat_runs(id uuid primary key, company_id uuid not null, agent_id uuid not null, + status text not null, runtime_mode text not null default 'native', native_session_id text, runner_instance_id text, + native_issue_id uuid, issue_id uuid, context_snapshot jsonb, + log_store text, log_ref text, log_bytes bigint, + log_sha256 text, last_output_seq bigint not null default 0, last_output_at timestamptz, last_output_stream text, + last_output_bytes bigint, next_event_seq bigint not null default 1, controller_lease_expires_at timestamptz, + created_at timestamptz not null default now(), updated_at timestamptz not null default now())`); + await db.execute(sql`create table heartbeat_run_events(id bigserial primary key, company_id uuid not null, + run_id uuid not null, agent_id uuid not null, seq bigint not null, event_type text not null, stream text, + level text, color text, message text, payload jsonb, source_instance_id text, source_event_id text, + source_seq bigint, source_payload_sha256 text, protocol_schema_version int, created_at timestamptz not null default now(), + unique(run_id,seq), unique(run_id,source_event_id), unique(run_id,source_instance_id,source_seq))`); + await db.execute( + sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`, + ); + await db.execute(sql`create table agent_identity_keys(agent_id uuid primary key,company_id uuid,algorithm text,key_id text, + public_key_pem text,private_key_material jsonb,created_at timestamptz default now())`); +} diff --git a/server/src/__tests__/helpers/quota-credential-process.ts b/server/src/__tests__/helpers/quota-credential-process.ts new file mode 100644 index 00000000000..3914dcfe7e5 --- /dev/null +++ b/server/src/__tests__/helpers/quota-credential-process.ts @@ -0,0 +1,19 @@ +// A real receiver process for the credential-loss boundary test. No provider +// credentials leave this process through logs, IPC, or diagnostic exceptions. +import { createDb } from "@paperclipai/db"; +import { aiConnectionService } from "../../services/ai-connections.js"; + +const db = createDb(process.env.CREDENTIAL_TEST_DATABASE_URL!); +const provider = process.env.CREDENTIAL_TEST_PROVIDER_URL!; +const request = globalThis.fetch; +globalThis.fetch = (url, init) => { + if (String(url) !== "https://auth.openai.com/oauth/token") throw new Error("Unexpected fixture network request"); + return request(provider, init); +}; +try { + const service = aiConnectionService(db); + const [row] = await service.quotaAccounts(process.env.CREDENTIAL_TEST_COMPANY_ID!, process.env.CREDENTIAL_TEST_OWNER!); + await service.refreshQuotaCredential(row, await service.credential(row), new AbortController().signal); + process.exitCode = 0; +} catch { process.exitCode = 1; } +finally { await db.$client.end(); } diff --git a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts index 826fe78ec31..68f1c5eda6f 100644 --- a/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts +++ b/server/src/__tests__/issue-agent-mutation-ownership-routes.test.ts @@ -1090,6 +1090,7 @@ describe("agent issue mutation checkout ownership", () => { ]); expect(mockAccessService.decide).toHaveBeenCalledWith(expect.objectContaining({ action: "issue:read" })); expect(mockIssueService.listComments).toHaveBeenCalledWith(issueId, { + companyId, afterCommentId: null, order: "desc", limit: null, diff --git a/server/src/__tests__/issue-scheduled-retry-routes.test.ts b/server/src/__tests__/issue-scheduled-retry-routes.test.ts index 81168a07195..2f196d89289 100644 --- a/server/src/__tests__/issue-scheduled-retry-routes.test.ts +++ b/server/src/__tests__/issue-scheduled-retry-routes.test.ts @@ -580,16 +580,33 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => { .where(inArray(heartbeatRuns.id, work.map(row => row.id))); clock(); const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError: error => { throw error; } }); + const register = coordinator.register.bind(coordinator); + let initial: Awaited> | undefined; + let releaseFirstPass!: () => void; + const firstPassHeld = new Promise(resolve => { releaseFirstPass = resolve; }); + vi.spyOn(coordinator, "register").mockImplementation((queue, task) => register(queue, { + ...task, + run: async signal => { + const result = await task.run(signal); + if (initial) return result; + const cancelled = (await states()).filter(row => row.status === "cancelled"); + if (cancelled.length === 0) return result; + initial = cancelled; + await firstPassHeld; + return result; + }, + })); try { await heartbeatService(db, { runtimeEnv: {} }).registerScheduledRetryCancellationWork(coordinator, () => true).ready; expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(0); const earlierId = "00000000-0000-4000-8000-000000000000"; - await writeIntent({ companyId: a.companyId, retryRunId: earlierId }); work.push({ id: earlierId }); + await writeIntent({ companyId: a.companyId, retryRunId: earlierId }); await vi.advanceTimersByTimeAsync(1000); - await expect.poll(async () => (await states()).filter(row => row.status === "cancelled").length).toBe(20); - const initial = (await states()).filter(row => row.status === "cancelled"); - expect(new Set(initial.map(row => row.companyId)).size).toBe(2); + await expect.poll(() => initial).toBeDefined(); + expect(initial).toHaveLength(20); + expect(new Set(initial!.map(row => row.companyId)).size).toBe(2); + releaseFirstPass(); await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(1000); expect((await states()).filter(row => row.status === "cancelled")).toHaveLength(29); @@ -600,7 +617,7 @@ describeEmbeddedPostgres("issue scheduled retry routes", () => { const reads = vi.spyOn(db, "select"); await vi.advanceTimersByTimeAsync(180_000); expect(reads).not.toHaveBeenCalled(); reads.mockRestore(); - } finally { await coordinator.stop(); } + } finally { releaseFirstPass(); await coordinator.stop(); } }); }); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index bbf94717ff3..fb13ff43e82 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -48,6 +48,7 @@ import { readIssueCommentRunLogText, } from "../services/issues.ts"; import { getRunLogStore } from "../services/run-log-store.js"; +import * as nativeRunLogRecovery from "../services/native-runtime/native-run-log-recovery.js"; import { terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { settleUnrecoverableExecutions } from "../services/execution-recovery-resolution.js"; import { @@ -192,6 +193,35 @@ describeEmbeddedPostgres("issueService run attachment artifacts", () => { }); describe("readIssueCommentRunLogText", () => { + it("does not read after a timed-out native repair", async () => { + const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockImplementationOnce((_db, _store, _ref, _redaction, signal) => new Promise((_resolve, reject) => { + signal!.addEventListener("abort", () => reject(signal!.reason), { once: true }); + })); + const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValue({ content: "too late" }); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const pending = readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType); + try { + await vi.advanceTimersByTimeAsync(3_000); + await expect(pending).resolves.toBe(""); + expect(read).not.toHaveBeenCalled(); + } finally { + await pending; + vi.useRealTimers(); + repair.mockRestore(); + read.mockRestore(); + } + }); + it("keeps published evidence when a concurrent native repair wins ownership", async () => { + const repair = vi.spyOn(nativeRunLogRecovery, "recoverSettledNativeRunLog").mockRejectedValueOnce(new Error("The output journal is owned by another writer")); + const read = vi.spyOn(getRunLogStore(), "read").mockResolvedValueOnce({ content: "published evidence" }); + try { + await expect(readIssueCommentRunLogText({ logStore: "s3_segments", logRef: "company/agent/run", logBytes: null }, {} as ReturnType)).resolves.toBe("published evidence"); + expect(read).toHaveBeenCalledOnce(); + } finally { + repair.mockRestore(); + read.mockRestore(); + } + }); it("cancels timed-out storage reads so later listings can recover", async () => { let active = 0; const cleanups: Array<() => void> = []; diff --git a/server/src/__tests__/local-encrypted-cipher.test.ts b/server/src/__tests__/local-encrypted-cipher.test.ts index 06e2b0de36f..88f04740e67 100644 --- a/server/src/__tests__/local-encrypted-cipher.test.ts +++ b/server/src/__tests__/local-encrypted-cipher.test.ts @@ -1,9 +1,9 @@ import { randomBytes } from "node:crypto"; -import { mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdtempSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createLocalEncryptedCipher, localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; +import { createLocalEncryptedCipher, deriveLocalEncryptedInstanceSecret, localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; describe("operation-scoped local encryption", () => { let root: string; @@ -60,4 +60,16 @@ describe("operation-scoped local encryption", () => { code: "provider_unavailable", operation: "resolveVersion", })); }); + it("derives the same instance secret from an existing key file or environment without creating missing keys", () => { + expect(() => deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false })) + .toThrow("Secrets master key file is missing"); + expect(existsSync(keyFile)).toBe(false); + const key = randomBytes(32).toString("hex"); + writeFileSync(keyFile, key, { mode: 0o600 }); + const fromFile = deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false }); + unlinkSync(keyFile); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", key); + expect(deriveLocalEncryptedInstanceSecret("trace-test", process.env, { allowCreate: false })).toBe(fromFile); + expect(existsSync(keyFile)).toBe(false); + }); }); diff --git a/server/src/__tests__/native-harness-backup-expiration.test.ts b/server/src/__tests__/native-harness-backup-expiration.test.ts index b24c12a80c9..2437d9711b5 100644 --- a/server/src/__tests__/native-harness-backup-expiration.test.ts +++ b/server/src/__tests__/native-harness-backup-expiration.test.ts @@ -645,8 +645,13 @@ describeEmbeddedPostgres("native harness backup expiration", () => { const available = await seedArchive(healthy.root, { name: `${dueAt - 7 * DAY_MS - 1}-${randomUUID()}` }); await startEvidence({ isSessionBusy: scope => scope === busy.scope }); await settled(dueAt); await vi.advanceTimersByTimeAsync(dueAt - Date.now()); - await vi.waitFor(async () => { await vi.advanceTimersByTimeAsync(100); expect(await exists(available)).toBe(false); }); - expect(await exists(held)).toBe(true); expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000); + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + expect(await exists(available)).toBe(false); + expect(idleWorkSnapshot().active).toBe(0); + expect(coordinator!.nextWakeAt()! - Date.now()).toBeGreaterThan(59 * 60_000); + }); + expect(await exists(held)).toBe(true); await quiet(60_000); }); diff --git a/server/src/__tests__/plugin-task-credential-writeback.test.ts b/server/src/__tests__/plugin-task-credential-writeback.test.ts index 8ef383854c9..0603d09f59a 100644 --- a/server/src/__tests__/plugin-task-credential-writeback.test.ts +++ b/server/src/__tests__/plugin-task-credential-writeback.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { mkdtemp, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { and, eq } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -90,6 +90,26 @@ suite("durable plugin task credential handoff", () => { expect(await f.credential()).toBe(auth("fixture-refreshed", 2021)); expect(f.calls.at(-1)).toBe("complete"); }); + it("retains the Runner credential after a database save failure until a fresh receiver commits", async () => { + const f = await fixture(); + await db.execute(sql.raw(`create function reject_runner_credential_save() returns trigger language plpgsql as $$ begin + if new.secret_id = '${f.managed.credentialWriteback!.secretId}'::uuid then raise exception 'fixture receiver save unavailable'; end if; return new; end $$; + create constraint trigger reject_runner_credential_save after insert on company_secret_versions + deferrable initially deferred for each row execute function reject_runner_credential_save();`)); + try { + await expect(f.settle()).rejects.toThrow(); + expect(f.calls).toContain("credentials"); + expect(f.calls).not.toContain("complete"); + expect(await f.credential()).toBe(auth("fixture-original")); + expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "pending" }); + } finally { await db.execute(sql`drop trigger reject_runner_credential_save on company_secret_versions; drop function reject_runner_credential_save()`); } + const receiver = createDb(database.connectionString); + expect(await settleDispatchedEnvironmentTask(receiver, f.workers, { companyId: f.companyId, id: f.leaseId })).toBe(true); + expect(await f.credential()).toBe(auth("fixture-refreshed", 2021)); + expect(f.calls.filter(call => call === "credentials")).toHaveLength(2); + expect(f.calls.at(-1)).toBe("complete"); + expect((await f.lease()).metadata?.runnerTaskCredentialHandoff).toMatchObject({ state: "complete" }); + }); it("does not overwrite an explicit account replacement", async () => { const f = await fixture(); await secretService(db).rotate(f.managed.credentialWriteback!.secretId, { value: auth("fixture-reconnected", 2022) }, { userId: f.owner }); diff --git a/server/src/__tests__/provider-trace-retention-work.test.ts b/server/src/__tests__/provider-trace-retention-work.test.ts new file mode 100644 index 00000000000..2c2d2fcc3ac --- /dev/null +++ b/server/src/__tests__/provider-trace-retention-work.test.ts @@ -0,0 +1,66 @@ +import type { Db } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { afterEach, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerProviderTraceRetentionWork } from "../services/provider-trace-retention-work.js"; + +const state = vi.hoisted(() => ({ + cleanup: vi.fn(async () => []), + next: vi.fn<() => Promise>(), + probe: undefined as undefined | (() => string), + unregister: vi.fn(), +})); +vi.mock("../services/provider-trace-store.js", () => ({ + providerTraceStore: () => ({ cleanupExpired: state.cleanup, nextRetentionDeadline: state.next }), +})); +vi.mock("../services/idle-local-work.js", () => ({ + registerIdleLocalWorkProbe: (probe: () => string) => { + state.probe = probe; + return state.unregister; + }, +})); +let coordinator: ReturnType | undefined; +afterEach(async () => { + await coordinator?.stop(); + vi.useRealTimers(); + vi.clearAllMocks(); +}); + +it("reconstructs trace deadlines, stays quiet between obligations, and discharges the sleep blocker", async () => { + vi.useFakeTimers(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun: () => true, onError }); + state.next.mockResolvedValue(null); + const worker = registerProviderTraceRetentionWork( + owner as unknown as Db, + coordinator, + () => true, + ); + expect(state.probe!()).toBe("unknown"); + await worker.ready; + expect(state.probe!()).toBe("none"); + expect(coordinator.nextWakeAt()).toBeNull(); + const initial = state.cleanup.mock.calls.length; + await vi.advanceTimersByTimeAsync(300_000); + expect(state.cleanup).toHaveBeenCalledTimes(initial); + expect(state.next).toHaveBeenCalledOnce(); + + const deadline = Date.now() + 60_000; + state.next.mockResolvedValue(deadline); + worker.wake(); + await vi.advanceTimersByTimeAsync(0); + expect(state.probe!()).toBe("present"); + expect(coordinator.nextWakeAt()).toBe(deadline); + const scheduled = state.cleanup.mock.calls.length; + await vi.advanceTimersByTimeAsync(59_999); + expect(state.cleanup).toHaveBeenCalledTimes(scheduled); + state.next.mockResolvedValue(null); + await vi.advanceTimersByTimeAsync(1); + expect(state.cleanup).toHaveBeenCalledTimes(scheduled + 1); + expect(state.probe!()).toBe("none"); + expect(coordinator.nextWakeAt()).toBeNull(); + await worker.stop(); + expect(state.unregister).toHaveBeenCalledOnce(); + expect(onError).not.toHaveBeenCalled(); +}); diff --git a/server/src/__tests__/recovery-cutover.test.ts b/server/src/__tests__/recovery-cutover.test.ts new file mode 100644 index 00000000000..ee9aca31a51 --- /dev/null +++ b/server/src/__tests__/recovery-cutover.test.ts @@ -0,0 +1,15 @@ +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { it } from "vitest"; + +const run = promisify(execFile); + +// Keep the dependency-free, actual-CLI acceptance suite usable on a retained +// volume while also exercising the same cases in the normal CI test lane. +it("passes the read-only recovery cutover CLI acceptance suite", async () => { + await run(process.execPath, ["--test", fileURLToPath(new URL("../../../scripts/recovery-cutover.test.mjs", import.meta.url))], { + timeout: 45_000, + maxBuffer: 1024 * 1024, + }); +}, 60_000); diff --git a/server/src/__tests__/run-evidence-migration-work.test.ts b/server/src/__tests__/run-evidence-migration-work.test.ts new file mode 100644 index 00000000000..d14da6c3018 --- /dev/null +++ b/server/src/__tests__/run-evidence-migration-work.test.ts @@ -0,0 +1,168 @@ +import type { Db } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "../../../packages/db/src/work-signals.js"; +import { afterEach, expect, it, vi } from "vitest"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { registerRunEvidenceMigrationWork } from "../services/run-evidence-migration-work.js"; + +const state = vi.hoisted(() => ({ + provider: "s3", + migrate: vi.fn(), + probe: undefined as undefined | (() => string), + unregister: vi.fn(), + info: vi.fn(), + warn: vi.fn(), +})); +vi.mock("../storage/config.js", () => ({ + loadStorageConfig: () => ({ storageProvider: state.provider }), +})); +vi.mock("../services/run-evidence-migration.js", () => ({ migrateRunEvidence: state.migrate })); +vi.mock("../middleware/logger.js", () => ({ logger: { info: state.info, warn: state.warn } })); +vi.mock("../services/idle-local-work.js", () => ({ + registerIdleLocalWorkProbe: (probe: () => string) => { + state.probe = probe; + return state.unregister; + }, +})); +let coordinator: ReturnType | undefined; +afterEach(async () => { + await coordinator?.stop(); + coordinator = undefined; + state.provider = "s3"; + state.probe = undefined; + vi.resetAllMocks(); + vi.useRealTimers(); +}); +function setup(canRun = () => true) { + vi.useFakeTimers(); + const owner = installDatabaseWorkSignals({ execute: vi.fn(async () => []) }); + const onError = vi.fn(); + coordinator = createDeliveryWorkCoordinator({ owner, canRun, canReconcile: () => true, onError }); + const worker = registerRunEvidenceMigrationWork(owner as unknown as Db, coordinator, canRun); + return { worker, onError }; +} +it("does no migration work for local storage", async () => { + state.provider = "local_disk"; + const { worker } = setup(); + expect(worker).toBeNull(); + await vi.advanceTimersByTimeAsync(3_600_000); + expect(state.migrate).not.toHaveBeenCalled(); + expect(state.probe).toBeUndefined(); + expect(coordinator!.nextWakeAt()).toBeNull(); +}); +it("automatically pages both resources, then leaves completed history quiet", async () => { + state.migrate + .mockResolvedValueOnce({ + items: [{ kind: "log", id: "one", state: "migrated" }], + nextRunId: "one", + }) + .mockResolvedValueOnce({ + items: [{ kind: "log", id: "two", state: "migrated" }], + nextRunId: "two", + }) + .mockResolvedValue({ items: [] }); + const { worker, onError } = setup(); + expect(state.probe!()).toBe("unknown"); + await worker!.ready; + expect(state.migrate).toHaveBeenNthCalledWith( + 1, + expect.anything(), + expect.objectContaining({ + apply: true, + limit: 1, + kinds: ["log", "trace"], + }), + ); + expect(state.probe!()).toBe("present"); + await vi.advanceTimersByTimeAsync(2_000); + expect(state.migrate).toHaveBeenNthCalledWith( + 2, + expect.anything(), + expect.objectContaining({ + afterRunId: "one", + kinds: ["log"], + }), + ); + expect(state.migrate).toHaveBeenNthCalledWith( + 3, + expect.anything(), + expect.objectContaining({ + afterRunId: "two", + kinds: ["log"], + }), + ); + expect(state.probe!()).toBe("none"); + expect(coordinator!.nextWakeAt()).toBeNull(); + await vi.advanceTimersByTimeAsync(3_600_000); + expect(state.migrate).toHaveBeenCalledTimes(3); + expect(onError).not.toHaveBeenCalled(); + await worker!.stop(); + expect(state.unregister).toHaveBeenCalledOnce(); +}); +it("keeps blocked history visible and retries it without stranding later pages", async () => { + state.migrate + .mockResolvedValueOnce({ + items: [{ kind: "log", id: "active", state: "blocked", reason: "legacy_writer_not_settled" }], + nextRunId: "active", + }) + .mockResolvedValueOnce({ items: [{ kind: "log", id: "later", state: "migrated" }] }) + .mockResolvedValue({ items: [{ kind: "log", id: "active", state: "migrated" }] }); + const { worker } = setup(); + await worker!.ready; + await vi.advanceTimersByTimeAsync(1_000); + expect(state.warn).toHaveBeenCalledWith( + expect.objectContaining({ id: "active", reason: "legacy_writer_not_settled" }), + expect.any(String), + ); + expect(state.probe!()).toBe("present"); + await vi.advanceTimersByTimeAsync(299_999); + expect(state.migrate).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(1); + expect(state.migrate).toHaveBeenNthCalledWith( + 3, + expect.anything(), + expect.objectContaining({ afterRunId: undefined, kinds: ["log", "trace"] }), + ); + expect(state.probe!()).toBe("none"); + expect(coordinator!.nextWakeAt()).toBeNull(); +}); +it("retains startup work during drain and retries a failed sweep before becoming quiet", async () => { + let admitted = false; + state.migrate + .mockRejectedValueOnce(new Error("storage unavailable")) + .mockResolvedValue({ items: [] }); + const { worker, onError } = setup(() => admitted); + await worker!.ready; + await vi.advanceTimersByTimeAsync(60_000); + expect(state.migrate).not.toHaveBeenCalled(); + expect(state.probe!()).toBe("unknown"); + admitted = true; + await vi.advanceTimersByTimeAsync(30_000); + expect(onError).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(30_000); + expect(state.migrate).toHaveBeenCalledTimes(2); + expect(state.probe!()).toBe("none"); + expect(coordinator!.nextWakeAt()).toBeNull(); +}); +it("aborts owned work on shutdown and reconstructs outstanding history on restart", async () => { + let started: AbortSignal | undefined; + state.migrate.mockImplementationOnce( + (_db, options) => + new Promise((_resolve, reject) => { + started = options.signal; + started!.addEventListener("abort", () => reject(started!.reason), { once: true }); + }), + ); + const { worker } = setup(); + await vi.advanceTimersByTimeAsync(0); + expect(started?.aborted).toBe(false); + await worker!.stop(); + await worker!.ready; + expect(started?.aborted).toBe(true); + expect(coordinator!.nextWakeAt()).toBeNull(); + await coordinator!.stop(); + state.migrate.mockResolvedValue({ items: [] }); + const restarted = setup(); + await restarted.worker!.ready; + expect(state.migrate).toHaveBeenCalledTimes(2); + expect(state.probe!()).toBe("none"); +}); diff --git a/server/src/__tests__/server-startup-feedback-export.test.ts b/server/src/__tests__/server-startup-feedback-export.test.ts index 6942d87e8d3..b645d7437c8 100644 --- a/server/src/__tests__/server-startup-feedback-export.test.ts +++ b/server/src/__tests__/server-startup-feedback-export.test.ts @@ -697,6 +697,12 @@ describe("startServer feedback export wiring", () => { expect(deliveryWorkErrors.filter(error => error.queue === DELIVERY_QUEUES.nativeRunnerEvidenceRetention)).toEqual([]); }); + it.each([false, true])("registers trace retention independently of heartbeat scheduling (%s)", async enabled => { + loadConfigMock.mockReturnValue(buildTestConfig({ heartbeatSchedulerEnabled: enabled })); + await startServer(); await settleStartupHousekeeping(); + expect(deliveryWorkRegisterMock.mock.calls.filter(([queue]) => queue === DELIVERY_QUEUES.providerTraceRetention)).toHaveLength(1); + }); + it.each([false, true])("registers skill cache retention only when enabled (%s)", async enabled => { loadConfigMock.mockReturnValue(buildTestConfig({ skillVersionRetentionEnabled: enabled })); await startServer(); await settleStartupHousekeeping(); diff --git a/server/src/__tests__/task-cold-open.test.ts b/server/src/__tests__/task-cold-open.test.ts index 15fc364a644..3d6aac70b08 100644 --- a/server/src/__tests__/task-cold-open.test.ts +++ b/server/src/__tests__/task-cold-open.test.ts @@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { activityLog, agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db"; +import { activityLog, agents, applyPendingMigrations, closeRegisteredClients, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, issueWorkProducts } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { issueRoutes } from "../routes/issues.js"; import { errorHandler } from "../middleware/index.js"; @@ -11,16 +11,25 @@ import { createHeartbeatRunState } from "../services/heartbeat/run-state.js"; import { activityService } from "../services/activity.js"; import { documentService } from "../services/documents.js"; -const support = await getEmbeddedPostgresTestSupport(); +const externalDatabase = process.env.PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL; +if (externalDatabase) { + const target = new URL(externalDatabase); + if (!["localhost", "127.0.0.1", "[::1]"].includes(target.hostname) || !target.pathname.endsWith("_test")) + throw new Error("PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL must name a disposable loopback database ending in _test"); +} +const support = externalDatabase ? { supported: true } : await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("task cold-open reads", () => { - let fixture: Awaited>; + let fixture: Pick>, "connectionString" | "cleanup">; let db: ReturnType; const companyId = randomUUID(); const issueId = randomUUID(); const agentId = randomUUID(); const runId = randomUUID(); beforeAll(async () => { - fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-"); + if (externalDatabase) { + await applyPendingMigrations(externalDatabase); + fixture = { connectionString: externalDatabase, cleanup: () => closeRegisteredClients(externalDatabase) }; + } else fixture = await startEmbeddedPostgresTestDatabase("task-cold-open-"); db = createDb(fixture.connectionString); await db.insert(companies).values({ id: companyId, name: "Cold open", issuePrefix: `T${companyId.slice(0, 6)}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Writer", role: "engineer", adapterType: "codex_local" }); @@ -79,6 +88,39 @@ const support = await getEmbeddedPostgresTestSupport(); expect(response.body.latestCommentId).toBe(response.body.comments[1].id); expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${issueId}/recent-comments`)).status); }); + it("opens long chats with a contiguous recent window and pages every older message", async () => { + const chatId = randomUUID(); + await db.insert(issues).values({ id: chatId, companyId, title: "Persistent chat", conversationAgentId: agentId, conversationUserId: "local-board", assigneeAgentId: agentId, conversationState: "waiting" }); + const rows = Array.from({ length: 123 }, (_, i) => ({ id: randomUUID(), companyId, issueId: chatId, + authorType: "user" as const, authorUserId: "local-board", body: `Chat ${i}`, createdAt: new Date(1000 + Math.floor(i / 3)) })); + await db.insert(issueComments).values(rows); + const ordered = [...rows].sort((a, b) => +b.createdAt - +a.createdAt || b.id.localeCompare(a.id)); + const response = await request(app()).get(`/api/issues/${chatId}/recent-comments`); + expect(response.status).toBe(200); + expect(response.body.comments.map((row: { id: string }) => row.id)).toEqual(ordered.slice(0, 20).map(row => row.id)); + expect(response.body.oldestCommentId).toBe(ordered[19].id); + expect(response.body.hasMore).toBe(true); + const seen = response.body.comments.map((row: { id: string }) => row.id); + let cursor = response.body.oldestCommentId; + while (cursor) { + const page = await request(app()).get(`/api/issues/${chatId}/comments`).query({ order: "desc", limit: 50, after: cursor }); + expect(page.status).toBe(200); + seen.push(...page.body.map((row: { id: string }) => row.id)); + cursor = page.body.at(-1)?.id; + } + expect(seen).toEqual(ordered.map(row => row.id)); + expect(new Set(seen).size).toBe(123); + expect([403, 404]).toContain((await request(app(randomUUID())).get(`/api/issues/${chatId}/recent-comments`)).status); + }); + it("hydrates only requested runs linked to the company and issue", async () => { + const otherIssue = randomUUID(), otherRun = randomUUID(); + await db.insert(issues).values({ id: otherIssue, companyId, title: "Other task" }); + await db.insert(heartbeatRuns).values({ id: otherRun, companyId, agentId, issueId: otherIssue, status: "succeeded" }); + const activity = activityService(db); + expect((await activity.runsForIssue(companyId, issueId, null, undefined, [runId, otherRun])).map(run => run.runId)).toEqual([runId]); + expect(await activity.runsForIssue(randomUUID(), issueId, null, undefined, [runId])).toEqual([]); + expect(await activity.runsForIssue(companyId, issueId, null, undefined, [])).toEqual([]); + }); it.each(["user", null] as const)("preserves initial and latest human instructions without a description (authorType=%s)", async (authorType) => { const bareIssueId = randomUUID(); await db.insert(issues).values({ id: bareIssueId, companyId, title: "No description" }); diff --git a/server/src/config.ts b/server/src/config.ts index e5af468eedb..c1156068fa8 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -16,13 +16,11 @@ import { BIND_MODES, DEPLOYMENT_EXPOSURES, SECRET_PROVIDERS, - STORAGE_PROVIDERS, type BindMode, type AuthBaseUrlMode, type DeploymentExposure, type DeploymentMode, type SecretProvider, - type StorageProvider, inferBindModeFromHost, resolveRuntimeBind, validateConfiguredBindMode, @@ -31,9 +29,9 @@ import { resolveDefaultBackupDir, resolveDefaultEmbeddedPostgresDir, resolveDefaultSecretsKeyFilePath, - resolveDefaultStorageDir, resolveHomeAwarePath, } from "./home-paths.js"; +import { loadStorageConfig, type StorageConfig } from "./storage/config.js"; if (!isCloudManagedInstance()) { const PAPERCLIP_ENV_FILE_PATH = resolvePaperclipEnvPath(); @@ -63,7 +61,7 @@ const TAILSCALE_DETECT_TIMEOUT_MS = 3000; type DatabaseMode = "embedded-postgres" | "postgres"; -export interface Config { +export interface Config extends StorageConfig { deploymentMode: DeploymentMode; deploymentExposure: DeploymentExposure; bind: BindMode; @@ -116,13 +114,6 @@ export interface Config { secretsProvider: SecretProvider; secretsStrictMode: boolean; secretsMasterKeyFilePath: string; - storageProvider: StorageProvider; - storageLocalDiskBaseDir: string; - storageS3Bucket: string; - storageS3Region: string; - storageS3Endpoint: string | undefined; - storageS3Prefix: string; - storageS3ForcePathStyle: boolean; feedbackExportBackendUrl: string | undefined; feedbackExportBackendToken: string | undefined; heartbeatSchedulerEnabled: boolean; @@ -162,7 +153,6 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile() : undefined; const fileDatabaseBackup = fileConfig?.database.backup; const fileSecrets = fileConfig?.secrets; - const fileStorage = fileConfig?.storage; const providerFromEnvRaw = process.env.PAPERCLIP_SECRETS_PROVIDER; const providerFromEnv = @@ -172,25 +162,7 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile() const providerFromFile = fileSecrets?.provider; const secretsProvider: SecretProvider = providerFromEnv ?? providerFromFile ?? "local_encrypted"; - const storageProviderFromEnvRaw = process.env.PAPERCLIP_STORAGE_PROVIDER; - const storageProviderFromEnv = - storageProviderFromEnvRaw && STORAGE_PROVIDERS.includes(storageProviderFromEnvRaw as StorageProvider) - ? (storageProviderFromEnvRaw as StorageProvider) - : null; - const storageProvider: StorageProvider = storageProviderFromEnv ?? fileStorage?.provider ?? "local_disk"; - const storageLocalDiskBaseDir = resolveHomeAwarePath( - process.env.PAPERCLIP_STORAGE_LOCAL_DIR ?? - fileStorage?.localDisk?.baseDir ?? - resolveDefaultStorageDir(), - ); - const storageS3Bucket = process.env.PAPERCLIP_STORAGE_S3_BUCKET ?? fileStorage?.s3?.bucket ?? "paperclip"; - const storageS3Region = process.env.PAPERCLIP_STORAGE_S3_REGION ?? fileStorage?.s3?.region ?? "us-east-1"; - const storageS3Endpoint = process.env.PAPERCLIP_STORAGE_S3_ENDPOINT ?? fileStorage?.s3?.endpoint ?? undefined; - const storageS3Prefix = process.env.PAPERCLIP_STORAGE_S3_PREFIX ?? fileStorage?.s3?.prefix ?? ""; - const storageS3ForcePathStyle = - process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE !== undefined - ? process.env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE === "true" - : (fileStorage?.s3?.forcePathStyle ?? false); + const storage = loadStorageConfig(fileConfig); const feedbackExportBackendUrl = process.env.PAPERCLIP_FEEDBACK_EXPORT_BACKEND_URL?.trim() || process.env.PAPERCLIP_TELEMETRY_BACKEND_URL?.trim() || @@ -447,13 +419,7 @@ export function loadConfig(fileConfig: PaperclipConfig | null = readConfigFile() fileSecrets?.localEncrypted.keyFilePath ?? resolveDefaultSecretsKeyFilePath(), ), - storageProvider, - storageLocalDiskBaseDir, - storageS3Bucket, - storageS3Region, - storageS3Endpoint, - storageS3Prefix, - storageS3ForcePathStyle, + ...storage, feedbackExportBackendUrl, feedbackExportBackendToken, heartbeatSchedulerEnabled: process.env.HEARTBEAT_SCHEDULER_ENABLED !== "false", diff --git a/server/src/index.ts b/server/src/index.ts index 9ba0e2c02d2..ae229f03518 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -1,3 +1,5 @@ +import { registerProviderTraceRetentionWork } from "./services/provider-trace-retention-work.js"; +import { registerRunEvidenceMigrationWork } from "./services/run-evidence-migration-work.js"; import { registerSkillVersionRetentionWork } from "./services/skill-version-retention-work.js"; import { registerDatabaseBackupWork } from "./services/database-backup-work.js"; import { registerToolConnectionHealthWork } from "./services/tool-connection-health-work.js"; @@ -1365,6 +1367,13 @@ async function startServerWithDatabaseTeardown( })); }); server.on("close", () => { void taskDrainExpiryWork.stop(); }); + const runEvidenceMigrationWork = registerRunEvidenceMigrationWork(db, deliveryWork, + () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby(), { storageProvider: config.storageProvider }); + if (runEvidenceMigrationWork) trackHeartbeatSchedulerWork(runEvidenceMigrationWork.ready); + const providerTraceRetentionWork = registerProviderTraceRetentionWork(db, deliveryWork, + () => !heartbeatSchedulerStopped && !isIdleTaskDrainActive() && !isWarmStandby()); + trackHeartbeatSchedulerWork(providerTraceRetentionWork.ready); + server.on("close", providerTraceRetentionWork.stop); const executionSchedule = executionControlSchedule(db); const workspaceRecoveryRuntime = environmentRuntimeService(db, { pluginWorkerManager }); const nativeRunnerEvidenceRetentionWorker = config.nativeHarnessBackupExpirationEnabled @@ -2236,6 +2245,8 @@ async function startServerWithDatabaseTeardown( unsubscribeNativeWorkspaceFinalization?.(); unsubscribeRetainedNativeSessionCleanup?.(); await deliveryWork.stop(); + await runEvidenceMigrationWork?.stop(); + await providerTraceRetentionWork.stop(); await skillVersionRetentionWorker?.dispose(); await workspaceRecoveryWork.stop(); unregisterTerminalWorkspaceProbe?.(); diff --git a/server/src/routes/activity.ts b/server/src/routes/activity.ts index fd46f3d1208..7a9b187dbab 100644 --- a/server/src/routes/activity.ts +++ b/server/src/routes/activity.ts @@ -373,6 +373,14 @@ export function activityRoutes(db: Db) { if (!issue) return; if (!(await assertIssueReadAllowed(req, res, issue))) return; const viewerUserId = req.actor.type === "board" ? getActorInfo(req).actorId : null; + if (req.query.runIds !== undefined) { + const parsed = z.array(z.string().uuid()).min(1).max(100).safeParse( + typeof req.query.runIds === "string" ? req.query.runIds.split(",") : null, + ); + if (!parsed.success) throw badRequest("runIds must contain 1 to 100 UUIDs"); + res.json(await svc.runsForIssue(issue.companyId, issue.id, viewerUserId, undefined, parsed.data)); + return; + } const result = await (req.query.view === "recent" ? svc.runsForIssue(issue.companyId, issue.id, viewerUserId, 2) : svc.runsForIssue(issue.companyId, issue.id, viewerUserId)); diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index b168d12beaf..cf86556ce5e 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -16449,12 +16449,23 @@ export function issueRoutes( router.get("/issues/:id/recent-comments", async (req, res) => { const issue = await getAccessibleResource(req, res, getIssueById(req, req.params.id as string), "Issue not found"); if (!issue || !(await assertIssueReadAllowed(req, res, issue))) return; + if (issue.conversationAgentId) { + const recent = await svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 21 }); + const comments = recent.slice(0, 20); + res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, { + comments, + latestCommentId: comments[0]?.id ?? null, + oldestCommentId: comments.at(-1)?.id ?? null, + hasMore: recent.length > 20, + })); + return; + } const [recent, human, firstInstruction] = await Promise.all([ - svc.listComments(issue.id, { order: "desc", limit: 5 }), - svc.listComments(issue.id, { order: "desc", limit: 1, humanOnly: true }), + svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 5 }), + svc.listComments(issue.id, { companyId: issue.companyId, order: "desc", limit: 1, humanOnly: true }), issue.description?.trim() ? Promise.resolve([]) - : svc.listComments(issue.id, { order: "asc", limit: 1, humanOnly: true }), + : svc.listComments(issue.id, { companyId: issue.companyId, order: "asc", limit: 1, humanOnly: true }), ]); const comments = [...new Map([...firstInstruction, ...human, ...recent.slice(0, 4)].map((comment) => [comment.id, comment])).values()]; res.json(await runRedactions.redactForIssue(issue.companyId, issue.id, { @@ -16494,7 +16505,8 @@ export function issueRoutes( limitRaw && Number.isFinite(limitRaw) && limitRaw > 0 ? Math.min(Math.floor(limitRaw), MAX_ISSUE_COMMENT_LIMIT) : null; - const comments = await svc.listComments(id, { + const comments = await svc.listComments(issue.id, { + companyId: issue.companyId, afterCommentId, order, limit, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 7b10759ed26..ff110a52e18 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -4599,7 +4599,7 @@ registry.registerPath({ path: "/api/issues/{id}/recent-comments", tags: ["issues"], summary: "Read a bounded recent task exchange", - description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.", + description: "Returns four recent comments, the latest human comment, and the first human instruction when the task has no description. Agent chats instead return the latest 20 contiguous comments and oldestCommentId for backward cursor paging. Includes latestCommentId and hasMore. Uses the same authorization and redaction as full comment reads.", request: { params: z.object({ id: z.string() }) }, responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, }); @@ -6091,8 +6091,8 @@ registry.registerPath({ path: "/api/issues/{id}/runs", tags: ["activity"], summary: "List runs for an issue", - request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries.") }) }, - responses: { 200: r.ok(), 401: r.unauthorized }, + request: { params: z.object({ id: z.string() }), query: z.object({ view: z.literal("recent").optional().describe("At most two recent run summaries."), runIds: z.string().optional().describe("Comma-separated list of 1 to 100 run UUIDs. Returns only those runs linked to this accessible issue.") }) }, + responses: { 200: r.ok(), 400: r.badRequest, 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, }); registry.registerPath({ diff --git a/server/src/secrets/local-encrypted-provider.ts b/server/src/secrets/local-encrypted-provider.ts index ea0611bd06f..f0162fb56e8 100644 --- a/server/src/secrets/local-encrypted-provider.ts +++ b/server/src/secrets/local-encrypted-provider.ts @@ -94,8 +94,8 @@ function loadOrCreateMasterKey(allowCreate = true, env: LocalEncryptedKeyEnv = p } /** Derive a purpose-specific instance secret without exposing or reusing the vault key. */ -export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env): string { - return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(true, env), Buffer.alloc(0), +export function deriveLocalEncryptedInstanceSecret(purpose: string, env: LocalEncryptedKeyEnv = process.env, options: { allowCreate?: boolean } = {}): string { + return Buffer.from(hkdfSync("sha256", loadOrCreateMasterKey(options.allowCreate ?? true, env), Buffer.alloc(0), `paperclip:instance-secret:v1\0${purpose}`, 32)).toString("base64url"); } diff --git a/server/src/services/activity.ts b/server/src/services/activity.ts index b256bf358ff..bac15a5239f 100644 --- a/server/src/services/activity.ts +++ b/server/src/services/activity.ts @@ -428,7 +428,8 @@ export function activityService(db: Db) { return limit === undefined ? query : query.limit(limit); }, - runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number) => { + runsForIssue: async (companyId: string, issueId: string, viewerUserId: string | null = null, limit?: number, selectedRunIds?: string[]) => { + if (selectedRunIds && !selectedRunIds.length) return []; scheduleRunLivenessBackfill(companyId, issueId); const runsQuery = db .select({ @@ -479,6 +480,7 @@ export function activityService(db: Db) { .where( and( eq(heartbeatRuns.companyId, companyId), + ...(selectedRunIds ? [inArray(heartbeatRuns.id, selectedRunIds)] : []), or( eq(heartbeatRuns.issueId, issueId), sql`exists ( diff --git a/server/src/services/agent-identity.ts b/server/src/services/agent-identity.ts index 462c0393931..42615880789 100644 --- a/server/src/services/agent-identity.ts +++ b/server/src/services/agent-identity.ts @@ -5,6 +5,7 @@ import type { AgentPublicIdentity } from "@paperclipai/shared"; import { notFound } from "../errors.js"; import { localEncryptedProvider } from "../secrets/local-encrypted-provider.js"; import { persistActivity } from "./activity-log.js"; +import { createAgentIdentityRedactor } from "./agent-identity-redaction.js"; const publicColumns = { algorithm: agentIdentityKeys.algorithm, @@ -17,6 +18,16 @@ export function agentIdentityService(db: Db) { const ownership = (companyId: string, agentId: string) => and( eq(agentIdentityKeys.companyId, companyId), eq(agentIdentityKeys.agentId, agentId), ); + async function resolveIdentity(row: typeof agentIdentityKeys.$inferSelect) { + const privateKeyPem = await localEncryptedProvider.resolveVersion({ material: row.privateKeyMaterial, externalRef: null }); + const privateKey = createPrivateKey(privateKeyPem); + const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" }); + const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" }); + if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" || + !derived.equals(stored) || fingerprint(stored) !== row.keyId) + throw new Error("Agent identity public and private keys do not match"); + return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem }; + } return { // Reading an identity must never provision it or decrypt private material. @@ -24,6 +35,10 @@ export function agentIdentityService(db: Db) { const [row] = await db.select(publicColumns).from(agentIdentityKeys).where(ownership(companyId, agentId)); return row ? { ...row, createdAt: row.createdAt.toISOString() } : null; }, + async createExistingLogRedactor(companyId: string, agentId: string) { + const [row] = await db.select().from(agentIdentityKeys).where(ownership(companyId, agentId)); + return createAgentIdentityRedactor(row ? (await resolveIdentity(row)).privateKeyPem : undefined); + }, async ensureAgentIdentity(companyId: string, agentId: string) { return db.transaction(async (tx) => { @@ -49,18 +64,8 @@ export function agentIdentityService(db: Db) { details: { algorithm: "Ed25519", keyId }, }); } - const privateKeyPem = await localEncryptedProvider.resolveVersion({ - material: row.privateKeyMaterial, externalRef: null, - }); // Fail closed on corruption or a wrong master key; never replace a key. - const privateKey = createPrivateKey(privateKeyPem); - const derived = createPublicKey(privateKey).export({ type: "spki", format: "der" }); - const stored = createPublicKey(row.publicKeyPem).export({ type: "spki", format: "der" }); - if (row.algorithm !== "Ed25519" || privateKey.asymmetricKeyType !== "ed25519" || - !derived.equals(stored) || fingerprint(stored) !== row.keyId) { - throw new Error("Agent identity public and private keys do not match"); - } - return { keyId: row.keyId, publicKeyPem: row.publicKeyPem, privateKeyPem }; + return resolveIdentity(row); }); }, }; diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index c03b40e3ced..d76f9606c9e 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -1105,7 +1105,7 @@ export function aiConnectionService(db: Db) { async function refreshQuotaCredential(row: Parameters[0], failedValue: string, signal: AbortSignal) { return withAccountHomeSecretMutationLock(undefined, row.connection.companyId, async () => { const recovery = quotaCredentialRecovery(row.connection.companyId, row.grant.id); - let savingReplacement = false; + let replacement: { value: string; secretId: string; version: number; epoch: number; base: string; grantUpdatedAt: number; versionSelector: "latest" | number } | undefined; const save = () => db.transaction(async (tx) => { // Do not wait on a transaction that might itself be waiting for the file // lock. All database locks must be available before consuming OAuth tokens. @@ -1117,11 +1117,20 @@ export function aiConnectionService(db: Db) { if (!grant || grant.status !== "active") throw new Error("credentials_unavailable"); const ref = grant.credentialSecretRefs.find(ref => ref.configPath === "ai.credential"); if (!ref) throw new Error("credentials_unavailable"); - await tx.select({ id: companySecrets.id }).from(companySecrets).where(and( + const [secret] = await tx.select({ id: companySecrets.id, version: companySecrets.latestVersion, epoch: companySecrets.aiSessionEpoch }).from(companySecrets).where(and( eq(companySecrets.id, ref.secretId), eq(companySecrets.companyId, companyId), )).for("update", { noWait: true }); + if (!secret) throw new Error("credentials_unavailable"); const current = await aiConnectionService(tx as unknown as Db).credential({ connection: row.connection, grant }); - signal.throwIfAborted(); + if (replacement) { + // A lost commit response is reconciled from the authoritative secret. + // Reconnect, revocation and a competing saved generation always win. + if (current === replacement.value) return current; + if (ref.secretId !== replacement.secretId || secret.version !== replacement.version + || secret.epoch !== replacement.epoch || current !== replacement.base + || grant.updatedAt.getTime() !== replacement.grantUpdatedAt + || (ref.versionSelector ?? "latest") !== replacement.versionSelector) return current; + } else signal.throwIfAborted(); let pending = await recovery.read(); if (pending && (pending.secretId !== ref.secretId || pending.connectionId !== row.connection.id || pending.baseHash !== quotaCredentialHash(current))) { // Reconnect may reuse this grant. Once the current authorized token @@ -1141,10 +1150,9 @@ export function aiConnectionService(db: Db) { or(sql`${heartbeatRuns.contextSnapshot}->'aiConnection'->>'grantId' = ${grant.id}`, sql`${agents.runtimeConfig}->'aiConnection'->>'provider' = 'openai'`, eq(agents.adapterType, "codex_local")), )).limit(1); - if (active && !pending) throw new Error("provider_unavailable"); - let refreshed = pending?.value; + if (active && !pending && !replacement) throw new Error("provider_unavailable"); + let refreshed = replacement?.value ?? pending?.value; if (!refreshed) { - await recovery.prepare(); const auth = JSON.parse(current); const refreshToken = auth.tokens?.refresh_token; if (typeof refreshToken !== "string" || !refreshToken) throw new Error("authentication_required"); @@ -1169,12 +1177,12 @@ export function aiConnectionService(db: Db) { ...(typeof tokens.refresh_token === "string" && tokens.refresh_token ? { refresh_token: tokens.refresh_token } : {}), ...(typeof tokens.id_token === "string" && tokens.id_token ? { id_token: tokens.id_token } : {}), }, last_refresh: new Date().toISOString() }); - // This fsynced encrypted journal survives transaction rollback and host - // restart. Never exchange the consumed token again after a failed save. - await recovery.write({ companyId, grantId: grant.id, connectionId: row.connection.id, - secretId: ref.secretId, baseHash: quotaCredentialHash(current), value: refreshed }); } - savingReplacement = true; + // Only this logical refresh owns the replacement. Save retries must not + // consume another single-use token. Process loss before commit can require + // signing in again in v1; no secondary credential vault is created. + replacement ??= { value: refreshed, secretId: ref.secretId, version: secret.version, epoch: secret.epoch, base: current, + grantUpdatedAt: grant.updatedAt.getTime(), versionSelector: ref.versionSelector ?? "latest" }; // Persist the rotated token even if the quota deadline expires after the // successful exchange; losing it would strand all subsequent executions. await secretService(tx).rotate(ref.secretId, { value: refreshed, preserveAiSessionEpoch: true }, { userId: grant.subjectUserId }); @@ -1191,9 +1199,9 @@ export function aiConnectionService(db: Db) { await recovery.clear().catch((error) => logger.warn({ err: error, grantId: row.grant.id }, "Saved quota credential recovery cleanup pending")); return value; } catch (error) { - if (!savingReplacement || attempt >= 2) throw error; - // Saving retries have no provider side effect. Persistent failures - // retain the journal for the next quota poll, including after restart. + if (!replacement || attempt >= 2) throw error; + // Retry only the returned value. Required database reads still fail + // closed; exhaustion is an error, never a successful refresh. } } }); diff --git a/server/src/services/customer-success-inspection.ts b/server/src/services/customer-success-inspection.ts index c4a5504ed97..a73d1673dcd 100644 --- a/server/src/services/customer-success-inspection.ts +++ b/server/src/services/customer-success-inspection.ts @@ -20,7 +20,7 @@ import { import { deriveBundleState } from "./agent-instructions.js"; import { readInstructionBytes } from "./agent-instruction-files.js"; import { skillPackageStore } from "./skill-package-store.js"; -import { getRunLogStore } from "./run-log-store.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; // This is a reviewed catalog of readers, not a proxy for existing GET routes. // Select credentials out of queries; never load encrypted identity records. @@ -311,9 +311,9 @@ export async function readCustomerSuccessResource( } if (q.operation === "runs.log") { const run = await ownRow(tx, schema.heartbeatRuns, q.companyId!, q.resourceId!); - if (!run.logRef || run.logStore !== "local_file") return unavailable("run_log_unavailable"); + if (!run.logRef || !isRunLogStoreType(run.logStore)) return unavailable("run_log_unavailable"); const result = await getRunLogStore().read( - { store: "local_file", logRef: String(run.logRef) }, + { store: run.logStore, logRef: String(run.logRef) }, { offset: q.range?.start ?? q.offset ?? 0, limitBytes: q.range ? q.range.end - q.range.start + 1 : 256000, diff --git a/server/src/services/delivery-work-notifications.ts b/server/src/services/delivery-work-notifications.ts index 67cfa6173df..9195f1fcb12 100644 --- a/server/src/services/delivery-work-notifications.ts +++ b/server/src/services/delivery-work-notifications.ts @@ -2,6 +2,8 @@ import { signalDatabaseWork, subscribeDatabaseWork } from "@paperclipai/db"; /** Topics for existing durable queues; register intent before their writes. */ export const DELIVERY_QUEUES = { + runEvidenceMigration: "run-evidence-migration", + providerTraceRetention: "provider-trace-retention", nativeRunnerEvidenceRetention: "native-runner-evidence-retention", skillVersionRetention: "skill-version-retention", databaseBackups: "database-backups", diff --git a/server/src/services/execution-preparation-timing.test.ts b/server/src/services/execution-preparation-timing.test.ts index c267d1ee68d..7b1703ae7a9 100644 --- a/server/src/services/execution-preparation-timing.test.ts +++ b/server/src/services/execution-preparation-timing.test.ts @@ -1,6 +1,6 @@ import { getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; -import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming } from "./execution-preparation-timing.js"; +import { createRunPreparationTiming, measureRunPhase, recordRepositoryObjectCacheDetail, withRunExecutionTiming, withRunWorkspaceFinalizationTiming, withRunResponsePresentationTiming } from "./execution-preparation-timing.js"; import { ROOT_CONTEXT, trace, SpanStatusCode, type Context } from "@opentelemetry/api"; import { createHash } from "node:crypto"; @@ -45,6 +45,7 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async try { await expect(createRunPreparationTiming("run", "native.controller")("backend_setup", () => 42)).resolves.toBe(42); await expect(withRunWorkspaceFinalizationTiming("run", () => 42)).resolves.toBe(42); + await expect(withRunResponsePresentationTiming("run", () => 42)).resolves.toBe(42); for (const phase of ["execution.usage_startup", "execution.mcp_handoff", "execution.dispatch_admission", "execution.budget_reservation", "skills.inventory", "skills.inventory.wait", "skills.cache.acquire", "skills.version.materialize", "skills.consume", "workspace_sync.seed_read_copy", "workspace_sync.seed_read_hash", "workspace_sync.seed_read_touch"] as const) { @@ -69,7 +70,10 @@ it.each([undefined, "", " "])("does not consult tracing for endpoint %s", async } }); -it("parents concurrent deferred copyback phases to their own finalization roots and preserves failure", async () => { +it.each([ + [withRunWorkspaceFinalizationTiming, "task.workspace_finalization", "workspace_sync.restore"], + [withRunResponsePresentationTiming, "task.response_presentation", "response.transaction"], +] as const)("parents concurrent deferred phases to their own %s root and preserves failure", async (withTiming, rootName, phase) => { const spans: Array<{ name: string; attributes: Record; context: Context; span: { end: ReturnType; setStatus: ReturnType } }> = []; handles.startActiveSpan.mockImplementation((name, options, context, callback) => { @@ -77,16 +81,16 @@ it("parents concurrent deferred copyback phases to their own finalization roots spans.push({ name, attributes: options.attributes, context, span }); return callback(span); }); - await Promise.all(["first", "second"].map(runId => withRunWorkspaceFinalizationTiming(runId, async () => { + await Promise.all(["first", "second"].map(runId => withTiming(runId, async () => { await Promise.resolve(); - await measureRunPhase("workspace_sync.restore", async () => {}); + await measureRunPhase(phase, async () => {}); }))); for (const runId of ["first", "second"]) { const hash = createHash("sha256").update(runId).digest("hex").slice(0, 12); const owned = spans.filter(span => span.attributes["paperclip.task.run.run_id"] === hash); expect(owned).toHaveLength(2); - const root = owned.find(span => span.name === "task.workspace_finalization")!; - const child = owned.find(span => span.name === "workspace_sync.restore")!; + const root = owned.find(span => span.name === rootName)!; + const child = owned.find(span => span.name === phase)!; expect(root.context).toBe(ROOT_CONTEXT); expect(trace.getSpan(child.context)).toBe(root.span); expect(root.span.end).toHaveBeenCalledOnce(); @@ -94,7 +98,7 @@ it("parents concurrent deferred copyback phases to their own finalization roots } const original = new Error("private finalization failure"); const work = vi.fn(() => { throw original; }); - await expect(withRunWorkspaceFinalizationTiming("failed", work)).rejects.toBe(original); + await expect(withTiming("failed", work)).rejects.toBe(original); expect(work).toHaveBeenCalledOnce(); expect(spans.at(-1)!.span.setStatus).toHaveBeenCalledWith({ code: SpanStatusCode.ERROR }); expect(spans.at(-1)!.span.end).toHaveBeenCalledOnce(); diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index 9041b76f8f3..9ee7f8a189e 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -199,6 +199,14 @@ export function withRunWorkspaceFinalizationTiming(runId: string, work: () => return measure(runId, "task.workspace_finalization", work, true); } +export function withRunResponsePresentationTiming(runId: string, work: () => T | Promise): Promise { + const owned = ownership.getStore(); + if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) { + return (async () => work())(); + } + return measure(runId, "task.response_presentation", work, true); +} + export async function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise { const owned = ownership.getStore(); if (!endpointEnabled() || !owned || owned.ended) return work(); diff --git a/server/src/services/feedback.ts b/server/src/services/feedback.ts index 1f9ceb78721..e3e4d70b766 100644 --- a/server/src/services/feedback.ts +++ b/server/src/services/feedback.ts @@ -47,7 +47,8 @@ import { sanitizeFeedbackValue, sha256Digest, } from "./feedback-redaction.js"; -import { getRunLogStore } from "./run-log-store.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; +import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js"; import { getOperatorSettingDefaults } from "./setting-defaults.js"; const FEEDBACK_SCHEMA_VERSION = "paperclip-feedback-envelope-v2"; @@ -318,17 +319,18 @@ async function findMatchingFile( return search(rootDir, 0); } -async function readFullRunLog(run: { +async function readFullRunLog(db: Db, run: { logStore: string | null; logRef: string | null; }) { - if (run.logStore !== "local_file" || !run.logRef) return null; + if (!isRunLogStoreType(run.logStore) || !run.logRef) return null; const store = getRunLogStore(); + if (run.logStore === "s3_segments") await recoverSettledNativeRunLog(db, store, run.logRef).catch(() => undefined); let offset = 0; let combined = ""; while (true) { - const result = await store.read({ store: "local_file", logRef: run.logRef }, { + const result = await store.read({ store: run.logStore, logRef: run.logRef }, { offset, limitBytes: 512_000, }).catch(() => null); @@ -1526,7 +1528,7 @@ async function buildFeedbackTraceBundleFromRow( .from(heartbeatRunEvents) .where(eq(heartbeatRunEvents.runId, run.id)) .orderBy(asc(heartbeatRunEvents.seq)); - const logText = await readFullRunLog(run); + const logText = await readFullRunLog(db, run); const logEntries = parseRunLogEntries(logText); const stdoutText = logEntries .filter((entry) => entry.stream === "stdout") diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index de0f082c8eb..670e4a2755b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -25,7 +25,10 @@ import { createHeartbeatPreparationTiming } from "./heartbeat/preparation-timing import { createHeartbeatInstructionCopyRelease, createHeartbeatInstructionWorkingCopy } from "./heartbeat/instruction-working-copy.js"; import { prepareHeartbeatRuntimeServiceIntents, startHeartbeatRuntimeServices, recordHeartbeatAdapterRuntimeServices } from "./heartbeat/runtime-services.js"; import { bindHeartbeatRunnerHomeMode, prepareHeartbeatExecutionConfiguration } from "./heartbeat/execution-configuration.js"; +import { createRunLogWriterAuthority } from "./heartbeat/run-log-authority.js"; import { createHeartbeatRunOutput } from "./heartbeat/run-output.js"; +import { readRecoverableRunLog } from "./native-runtime/native-run-log-recovery.js"; +import { observabilitySecretValues } from "@paperclipai/adapter-utils/agent-observability"; import { prepareHeartbeatSession, resolveHeartbeatAdapterSession } from "./heartbeat/session-preparation.js"; import { prepareHeartbeatTaskContext } from "./heartbeat/task-context.js"; import { selectHeartbeatEnvironment } from "./heartbeat/environment-selection.js"; @@ -370,7 +373,7 @@ import { type NativeSessionBackend, } from "../vendor/paperclip-runner/index.js"; -import { getRunLogStore } from "./run-log-store.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; import { providerTraceStore, } from "./provider-trace-store.js"; @@ -2075,6 +2078,7 @@ export function heartbeatService( } try { providerTraceCapture = await traceStore.prepare({ + assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner), runId: run.id, companyId: run.companyId, provider: @@ -3116,6 +3120,8 @@ export function heartbeatService( const runOutput = createHeartbeatRunOutput(db, { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions, + assertWriterAuthority: createRunLogWriterAuthority(db, run, runOptions.nativeLeaseOwner), + runtimeLogSecretValues: observabilitySecretValues(observability), }); const goalCheckpointSession: { current: { @@ -3201,7 +3207,7 @@ export function heartbeatService( message: "run started", }); - const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent } = + const { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onPersistedNativeLog, onAdapterMeta, onAdapterEvent } = await timePreparation("run_output", () => runOutput.begin(currentRun)); if (runScopedMentionedSkillKeys.length > 0) { await onLog( @@ -3435,6 +3441,7 @@ export function heartbeatService( identityRedactor, appendIdentityRedactedLog, onLog, + onPersistedNativeLog, onAdapterEvent, onAdapterMeta, }, @@ -4055,6 +4062,9 @@ export function heartbeatService( companyId: string; logStore: string | null; logRef: string | null; + runtimeMode?: string | null; + status?: string; + logBytes?: number | null; }, opts?: { offset?: number; limitBytes?: number; alignStartToRecord?: boolean; signal?: AbortSignal }, ) => { @@ -4067,12 +4077,14 @@ export function heartbeatService( if (!run) throw notFound("Heartbeat run not found"); if (!run.logStore || !run.logRef) throw notFound("Run log not found"); - const result = await runLogStore.read( + if (!isRunLogStoreType(run.logStore)) throw notFound("Run log backend not supported"); + const result = await readRecoverableRunLog(db, runLogStore, { - store: run.logStore as "local_file", + store: run.logStore, logRef: run.logRef, }, opts, + run, ); return { diff --git a/server/src/services/heartbeat/run-completion.test.ts b/server/src/services/heartbeat/run-completion.test.ts index 1dc15b06e9c..62ce16df460 100644 --- a/server/src/services/heartbeat/run-completion.test.ts +++ b/server/src/services/heartbeat/run-completion.test.ts @@ -1,3 +1,5 @@ +import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js"; +import type { PrpEvent } from "../../vendor/paperclip-runner/index.js"; import { randomUUID } from "node:crypto"; import type { Db } from "@paperclipai/db"; import { agents, agentWakeupRequests, caseHandoffs, cases, companies, costEvents, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; @@ -158,6 +160,57 @@ describe.skipIf(!support.supported && !process.env.PAPERCLIP_TEST_DATABASE_URL)( expect(f.deps.finalizeAgentStatus).toHaveBeenCalledWith(f.agent.id, "succeeded", null, expect.any(Object)); }); + it("settles the accepted native exact rate-card price without replacing it with zero", async () => { + const f = await fixture({ runtimeMode: "native" }); + const runnerInstanceId = randomUUID(); + const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId, + provider: { kind: "codex", model: "gpt-6.1-sol" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } }); + const event = (sourceSeq: number, eventType: string, payload: Record): PrpEvent => ({ + sourceKind: "runner", sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `rate-card-${sourceSeq}`, + turnId: "price-turn", eventType, emittedAt: new Date().toISOString(), payload, + } as PrpEvent); + await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true, + runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } })); + await delivery.capture(event(2, "turn.completed", {})); + const accepted = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id }); + expect(accepted?.costStatus).toBe("estimated"); + const exact = accepted!.costUsdExact; + f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...accepted, usageComplete: accepted!.complete }; + await f.completion.completeRun(f.input); + const saved = await read(f.run.id); + expect(saved).toMatchObject({ status: "succeeded", costAccountingPending: false, costAccountedAt: expect.any(Date), + usageJson: { costUsdExact: exact, costStatus: "estimated" } }); + const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)); + expect(ledger).toHaveLength(1); + expect(ledger[0].costCents).toBe(100.2); + }); + + it("retains newer accepted native usage when completion consumed an older receipt", async () => { + const f = await fixture({ runtimeMode: "native", usageJson: { sessionRotated: true, configFreshness: { version: 0 } } }); + const runnerInstanceId = randomUUID(); + const delivery = await createNativeUsageDelivery(db, { companyId: f.run.companyId, runId: f.run.id, runnerInstanceId, + provider: { kind: "codex", model: "test-model" }, billing: { provider: "openai", biller: "openai", billingType: "metered_api" } }); + const event = (sourceSeq: number, eventType: string, inputTokens = 100): PrpEvent => ({ + sourceInstanceId: runnerInstanceId, runId: f.run.id, sourceSeq, sourceEventId: `racing-${sourceSeq}`, turnId: "turn-1", + emittedAt: new Date().toISOString(), eventType, payload: eventType === "usage.reported" + ? { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens, outputTokens: 10, providerCostUsd: inputTokens / 1000 } } : {}, + } as PrpEvent); + await delivery.capture(event(1, "usage.reported")); + await delivery.capture(event(2, "turn.completed")); + const acceptedA = await usageReceipts.readAcceptedRunUsageCheckpoint(db, { companyId: f.run.companyId, runId: f.run.id }); + await delivery.capture(event(3, "usage.reported", 120)); + f.input.adapterResult = { exitCode: 0, signal: null, timedOut: false, ...acceptedA, usageComplete: acceptedA!.complete }; + await f.completion.completeRun(f.input); + expect(await read(f.run.id)).toMatchObject({ status: "succeeded", costAccountingPending: true, costAccountedAt: null, + usageJson: { inputTokens: 120, costUsd: 0.12, accountingReceiptSequence: 3, accountingReceiptReady: false, sessionRotated: false, configFreshness: { version: 1 } } }); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0); + await delivery.capture(event(4, "turn.completed", 120)); + expect(await (await import("../run-cost-accounting.js")).accountRunCost(db, f.run.id)).toBe(true); + const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)); + expect(ledger).toHaveLength(1); + expect(ledger[0].costCents).toBe(12); + }); + it.each([ { timedOut: true, expected: "timed_out", errorCode: "timeout" }, { signal: "SIGTERM", expected: "failed", errorCode: "adapter_failed" }, diff --git a/server/src/services/heartbeat/run-completion.ts b/server/src/services/heartbeat/run-completion.ts index 39bb4a509ab..c82816f384b 100644 --- a/server/src/services/heartbeat/run-completion.ts +++ b/server/src/services/heartbeat/run-completion.ts @@ -1,3 +1,4 @@ +import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; import { notifyChatRunMilestones } from "../chat-run-milestone-notifications.js"; import { measureRunPhase } from "../execution-preparation-timing.js"; import { PROJECT_ACCESS_DENIED, ProjectAccessDeniedError } from "../project-access-denial.js"; @@ -686,7 +687,7 @@ export function createHeartbeatRunCompletion(db: Db, dependencies: HeartbeatRunC cancellationResultJson(persistedRunWrite.run, status, finalRunPatch.resultJson, runErrorCode, runErrorMessage) ?? null, ), - usageJson: { ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) }, + usageJson: preserveAcceptedRunUsageProjectionSql({ ...parseObject(persistedRunWrite.run.usageJson), ...parseObject(finalRunPatch.usageJson) }), finishedAt: persistedRunWrite.run.finishedAt ?? finalRunPatch.finishedAt, updatedAt: new Date(), diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 1922364615a..00a27c1faf2 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; import { notifyNativeEnvironmentSettlement } from "../native-environment-settlement-notifications.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyExternalConversationSettlement } from "../external-conversation-settlement-notifications.js"; @@ -1202,6 +1203,7 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .set({ status, ...patch, + ...(patch?.usageJson ? { usageJson: preserveAcceptedRunUsageProjectionSql(patch.usageJson) } : {}), executionStatusDeliveryId: randomUUID(), updatedAt: new Date(), }) diff --git a/server/src/services/heartbeat/run-log-authority.ts b/server/src/services/heartbeat/run-log-authority.ts new file mode 100644 index 00000000000..705035cf617 --- /dev/null +++ b/server/src/services/heartbeat/run-log-authority.ts @@ -0,0 +1,65 @@ +import { and, eq } from "drizzle-orm"; +import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { legacyControllerBootId } from "../legacy-controller-lease.js"; +import { currentNativeControllerIdentity } from "../native-runtime/native-restart-recovery.js"; + +export function createRunLogWriterAuthority( + db: Db, + admitted: typeof heartbeatRuns.$inferSelect, + nativeLeaseOwner?: string, +) { + return async () => { + const [run] = await db + .select({ + id: heartbeatRuns.id, + runtimeMode: heartbeatRuns.runtimeMode, + controllerBootId: heartbeatRuns.controllerBootId, + leaseExpiresAt: heartbeatRuns.controllerLeaseExpiresAt, + logStore: heartbeatRuns.logStore, + logRef: heartbeatRuns.logRef, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, admitted.id), + eq(heartbeatRuns.companyId, admitted.companyId), + eq(heartbeatRuns.agentId, admitted.agentId), + eq(heartbeatRuns.status, "running"), + ), + ); + if (!run) throw new Error("run_log_writer_authority_lost"); + if (run.runtimeMode !== "native") { + if ( + run.controllerBootId !== legacyControllerBootId || + !run.leaseExpiresAt || + run.leaseExpiresAt <= new Date() + ) + throw new Error("run_log_writer_authority_lost"); + return; + } + const [owner] = await db + .select({ + leaseOwner: nativeRunFinalizations.leaseOwner, + bootId: nativeRunFinalizations.controllerBootId, + expiresAt: nativeRunFinalizations.leaseExpiresAt, + }) + .from(nativeRunFinalizations) + .where( + and( + eq(nativeRunFinalizations.runId, admitted.id), + eq(nativeRunFinalizations.companyId, admitted.companyId), + ), + ); + + if (!owner && !run.logStore && !run.logRef && !admitted.logStore && !admitted.logRef) return; + const identity = await currentNativeControllerIdentity(); + if ( + !owner || + owner.bootId !== identity.bootId || + !owner.expiresAt || + owner.expiresAt <= new Date() || + (nativeLeaseOwner !== undefined && owner.leaseOwner !== nativeLeaseOwner) + ) + throw new Error("run_log_writer_authority_lost"); + }; +} diff --git a/server/src/services/heartbeat/run-output.test.ts b/server/src/services/heartbeat/run-output.test.ts index bcfb7c3c2bc..f457f11f55c 100644 --- a/server/src/services/heartbeat/run-output.test.ts +++ b/server/src/services/heartbeat/run-output.test.ts @@ -7,6 +7,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createDurableRunLogStore } from "../run-log-store.js"; import type { StorageProvider } from "../../storage/types.js"; import { createHeartbeatRunOutput, isNewRunLog } from "./run-output.js"; +import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { formatNativeRunLogEvent } from "../native-runtime/native-run-log-export.js"; +import { generateKeyPairSync } from "node:crypto"; vi.mock("../live-events.js", () => ({ publishLiveEvent: vi.fn() })); @@ -45,15 +48,24 @@ function createRunRowDb(row: Partial) { return { update: () => ({ set: (values: Partial) => ({ - where: async () => { Object.assign(row, values); }, + where: async () => { + Object.assign(row, values); + }, }), }), } as unknown as Db; } const newRunRow = { - id: "run-1", companyId: "company-1", agentId: "agent-1", status: "running", - logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0, lastOutputAt: null, + id: "run-1", + companyId: "company-1", + agentId: "agent-1", + status: "running", + logStore: null, + logRef: null, + logBytes: null, + lastOutputSeq: 0, + lastOutputAt: null, } as Partial; async function beginAttempt(store: ReturnType, row: Partial) { @@ -81,11 +93,26 @@ afterEach(async () => { describe("heartbeat run output log recovery", () => { it("treats only a run row with no recorded log or output as new", () => { - expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe(true); - expect(isNewRunLog({ logStore: "local_file", logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false); - expect(isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 })).toBe(false); - expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe(false); - expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe(false); + expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 0 })).toBe( + true, + ); + expect( + isNewRunLog({ + logStore: "local_file", + logRef: "c/a/r.ndjson", + logBytes: null, + lastOutputSeq: 0, + }), + ).toBe(false); + expect( + isNewRunLog({ logStore: null, logRef: "c/a/r.ndjson", logBytes: null, lastOutputSeq: 0 }), + ).toBe(false); + expect(isNewRunLog({ logStore: null, logRef: null, logBytes: null, lastOutputSeq: 3 })).toBe( + false, + ); + expect(isNewRunLog({ logStore: null, logRef: null, logBytes: 120, lastOutputSeq: 0 })).toBe( + false, + ); }); it("makes no remote log recovery request for a new run", async () => { @@ -116,8 +143,121 @@ describe("heartbeat run output log recovery", () => { await second.onLog("stdout", "attempt two\n"); await restartedStore.finalize(second.output.handle!); - const lines = objects.get(row.logRef!)!.toString().trim().split("\n").map((line) => JSON.parse(line)); + const lines = objects + .get(row.logRef!)! + .toString() + .trim() + .split("\n") + .map((line) => JSON.parse(line)); expect(lines.map((line) => line.chunk)).toEqual(["attempt one\n", "attempt two\n"]); expect(lines.map((line) => line.seq)).toEqual([1, 2]); }); }); + +describe("object-backed diagnostic recording", () => { + it("applies identity redaction to a native key-bearing event before S3 admission", async () => { + const privateKeyPem = generateKeyPairSync("ed25519") + .privateKey.export({ type: "pkcs8", format: "pem" }) + .toString(); + const row = { ...newRunRow, runtimeMode: "native" } as Run; + const db = createRunRowDb(row) as Db & { select: () => unknown }; + db.select = (() => ({ + from: () => ({ where: () => ({ orderBy: () => ({ limit: async () => [] }) }) }), + })) as typeof db.select; + const chunks: string[] = []; + const output = createHeartbeatRunOutput(db, { + run: row, + runId: row.id, + issueId: null, + secretKeys: new Set(), + identityRedactor: createAgentIdentityRedactor(privateKeyPem), + runLogStore: { + begin: async () => ({ + store: "s3_segments", + logRef: "company/agent/run", + initialSourceCursor: 0, + }), + append: async () => 0, + enqueue: async (_handle, event) => { + chunks.push(event.chunk); + return { committed: Promise.resolve(100) }; + }, + flush: async () => undefined, + }, + appendRunEvent: vi.fn(), + getCurrentUserRedactionOptions: async () => ({ enabled: false }), + }); + const handlers = await output.begin(row); + await handlers.onPersistedNativeLog!( + 1, + formatNativeRunLogEvent({ + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + runId: row.id, + sourceEventId: "source:1", + sourceInstanceId: "source", + sourceKind: "runner", + sourceSeq: 1, + normalizedSessionId: "session", + eventType: "harness.diagnostic", + priority: 1, + emittedAt: new Date().toISOString(), + payload: { text: privateKeyPem }, + }), + ); + expect(chunks).toHaveLength(1); + expect(chunks[0]).not.toContain(privateKeyPem.split("\n")[1]!); + expect(chunks[0]).toContain("REDACTED"); + }); + it("admits a burst without serial network waits, publishes after receipts, and coalesces progress", async () => { + const { publishLiveEvent } = await import("../live-events.js"); + vi.mocked(publishLiveEvent).mockClear(); + const row = { ...newRunRow }; + const writes: Array> = []; + const db = { + update: () => ({ + set: (values: Partial) => ({ + where: async () => { + writes.push(values); + Object.assign(row, values); + }, + }), + }), + } as unknown as Db; + const commit: Array<() => void> = []; + const recorder = createHeartbeatRunOutput(db, { + run: row as Run, + runId: row.id!, + issueId: null, + runLogStore: { + begin: async () => ({ + store: "s3_segments", + logRef: "company/agent/run", + initialBytes: 0, + initialSequence: 10, + }), + append: () => new Promise((resolve) => commit.push(() => resolve(100))), + }, + identityRedactor: { chunk: (_stream, chunk) => chunk, redact: (value) => value }, + secretKeys: new Set(), + appendRunEvent: vi.fn(), + getCurrentUserRedactionOptions: async () => ({ enabled: false }), + }); + const { onLog } = await recorder.begin(row as Run); + for (let i = 0; i < 100; i++) await onLog("stdout", `line ${i}\n`); + expect(commit).toHaveLength(100); + expect(publishLiveEvent).not.toHaveBeenCalled(); + for (const resolve of commit) resolve(); + await vi.waitFor(() => + expect( + vi + .mocked(publishLiveEvent) + .mock.calls.filter(([event]) => event.type === "heartbeat.run.log"), + ).toHaveLength(100), + ); + await recorder.flushOutputProgress({ force: true }); + expect(row.lastOutputSeq).toBe(110); + expect(row.lastOutputBytes).toBe(10000); + expect(writes.filter((write) => write.lastOutputSeq !== undefined)).toHaveLength(2); + }); +}); diff --git a/server/src/services/heartbeat/run-output.ts b/server/src/services/heartbeat/run-output.ts index 62d929938ff..71a606a53a4 100644 --- a/server/src/services/heartbeat/run-output.ts +++ b/server/src/services/heartbeat/run-output.ts @@ -5,9 +5,15 @@ import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../l import type { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; import { touchHeartbeatRunRuntimeStatus } from "../heartbeat-run-runtime-status.js"; import { publishLiveEvent } from "../live-events.js"; -import type { RunLogHandle, RunLogStore } from "../run-log-store.js"; +import { isRunLogStoreType, type RunLogHandle, type RunLogStore } from "../run-log-store.js"; import { appendExcerpt, compactRunLogChunk } from "./run-log.js"; -import { isHeartbeatRunRuntimeStatusActive, publishHeartbeatRunRuntimeProgress, type createHeartbeatLifecycle } from "./run-lifecycle.js"; +import { createNativeRunLogExporter } from "../native-runtime/native-run-log-export.js"; +import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js"; +import { + isHeartbeatRunRuntimeStatusActive, + publishHeartbeatRunRuntimeProgress, + type createHeartbeatLifecycle, +} from "./run-lifecycle.js"; const MAX_LIVE_LOG_CHUNK_BYTES = 8 * 1024; export const ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS = 60 * 1000; @@ -19,11 +25,13 @@ export interface HeartbeatRunOutputInput { run: Run; runId: string; issueId: string | null; - runLogStore: Pick; + runLogStore: Pick; identityRedactor: Pick, "chunk" | "redact">; secretKeys: Set; appendRunEvent: ReturnType["appendRunEvent"]; getCurrentUserRedactionOptions: () => Promise; + assertWriterAuthority?: () => Promise; + runtimeLogSecretValues?: readonly string[]; } /** @@ -35,15 +43,26 @@ export interface HeartbeatRunOutputInput { export function isNewRunLog( run: Pick, ): boolean { - return run.logStore == null - && run.logRef == null - && run.logBytes == null - && Number(run.lastOutputSeq ?? 0) === 0; + return ( + run.logStore == null && + run.logRef == null && + run.logBytes == null && + Number(run.lastOutputSeq ?? 0) === 0 + ); } /** One recorder per execution attempt; the log store itself stays service-scoped. */ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) { - const { run, runId, issueId, runLogStore, identityRedactor, secretKeys, appendRunEvent, getCurrentUserRedactionOptions } = input; + const { + run, + runId, + issueId, + runLogStore, + identityRedactor, + secretKeys, + appendRunEvent, + getCurrentUserRedactionOptions, + } = input; let handle: RunLogHandle | null = null; let stdoutExcerpt = ""; let stderrExcerpt = ""; @@ -59,7 +78,14 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) } | null; } = { pending: null }; let persistedLogBytes = Number(run.logBytes ?? 0); + let progressWrite: Promise | null = null; + let recordingFailureReported = false; const flushOutputProgress = async (opts?: { force?: boolean }) => { + if (progressWrite) { + await progressWrite; + if (opts?.force) return flushOutputProgress(opts); + return; + } const pendingOutputProgress = outputProgressState.pending; if (!pendingOutputProgress) return; const shouldFlush = @@ -68,7 +94,7 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) pendingOutputProgress.at.getTime() - lastOutputFlushAt.getTime() >= ACTIVE_RUN_OUTPUT_PROGRESS_FLUSH_INTERVAL_MS; if (!shouldFlush) return; - await db + progressWrite = db .update(heartbeatRuns) .set({ lastOutputAt: pendingOutputProgress.at, @@ -77,111 +103,181 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) lastOutputBytes: pendingOutputProgress.bytes, updatedAt: new Date(), }) - .where(eq(heartbeatRuns.id, run.id)); - lastOutputFlushAt = pendingOutputProgress.at; - outputProgressState.pending = null; + .where(eq(heartbeatRuns.id, run.id)) + .then(() => { + lastOutputFlushAt = pendingOutputProgress.at; + if (outputProgressState.pending === pendingOutputProgress) + outputProgressState.pending = null; + }) + .finally(() => { + progressWrite = null; + }); + await progressWrite; }; // Bind handlers to the run row persisted at execution start. Progress writes // above use only the immutable run ID from the initial row. async function begin(currentRun: Run) { const run = currentRun; + if (run.logStore != null && !isRunLogStoreType(run.logStore)) + throw new Error("Run log backend is not supported"); handle = await runLogStore.begin({ companyId: run.companyId, agentId: run.agentId, runId, newRun: isNewRunLog(run), + assertWriterAuthority: input.assertWriterAuthority, + previousStore: run.logStore ?? undefined, }); + outputSeq = handle.initialSequence ?? outputSeq; + persistedLogBytes = handle.initialBytes ?? persistedLogBytes; await db .update(heartbeatRuns) .set({ logStore: handle.store, logRef: handle.logRef, + ...(handle.store === "s3_segments" ? { logBytes: null, logSha256: null } : {}), updatedAt: new Date(), }) .where(eq(heartbeatRuns.id, runId)); - const currentUserRedactionOptions = - await getCurrentUserRedactionOptions(); - const appendIdentityRedactedLog = async (stream: "stdout" | "stderr", chunk: string) => { + const currentUserRedactionOptions = await getCurrentUserRedactionOptions(); + if ( + handle.store === "s3_segments" && + run.runtimeMode === "native" && + input.runtimeLogSecretValues?.length + ) { + const registry = createRunSecretRedactionRegistry(db); + for (const value of new Set(input.runtimeLogSecretValues)) + await registry.register(run.companyId, runId, value); + } + const reportRecordingFailure = async () => { + if (recordingFailureReported) return; + recordingFailureReported = true; + await appendRunEvent(currentRun, { + eventType: "lifecycle", + stream: "system", + level: "warn", + message: "Run log persistence is incomplete", + payload: { reason: "object_log_persistence_failed" }, + }).catch(() => undefined); + }; + const appendIdentityRedactedLog = async ( + stream: "stdout" | "stderr", + chunk: string, + waitForCommit = true, + sourceCursor?: number, + ) => { const sanitizedChunk = compactRunLogChunk( redactCurrentUserText(chunk, currentUserRedactionOptions), ); - if (stream === "stdout") - stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk); - if (stream === "stderr") - stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); + if (stream === "stdout") stdoutExcerpt = appendExcerpt(stdoutExcerpt, sanitizedChunk); + if (stream === "stderr") stderrExcerpt = appendExcerpt(stderrExcerpt, sanitizedChunk); const ts = new Date().toISOString(); outputSeq += 1; const chunkSeq = outputSeq; - let appendedBytes = 0; + let committed: Promise = Promise.resolve(0); if (handle) { - appendedBytes = await runLogStore.append(handle, { + const event = { stream, chunk: sanitizedChunk, ts, seq: chunkSeq, - }); + sourceCursor, + }; + committed = runLogStore.enqueue + ? (await runLogStore.enqueue(handle, event)).committed + : runLogStore.append(handle, event); + } + const recorded = committed.then(async (appendedBytes) => { persistedLogBytes += appendedBytes; - } - outputProgressState.pending = { - at: new Date(ts), - seq: chunkSeq, - stream, - bytes: persistedLogBytes, - }; - await flushOutputProgress(); - - // Streamed CLI output is real run activity: keep the in-memory - // runtime status ("Working... / X ago") fresh between structured - // events so sandbox runs with mid-run log streaming never show a - // minutes-stale timestamp. Throttled to avoid churning the live - // event stream on every 250ms tail chunk. - const logActivityAt = new Date(ts); - if ( - isHeartbeatRunRuntimeStatusActive(run.status) && - logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >= - ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS - ) { - lastLogRuntimeStatusTouchMs = logActivityAt.getTime(); - const touchedStatus = touchHeartbeatRunRuntimeStatus({ - companyId: run.companyId, - issueId, - agentId: run.agentId, - runId: run.id, - at: logActivityAt, - }); - if (touchedStatus) - publishHeartbeatRunRuntimeProgress(touchedStatus); - } - - const payloadChunk = - sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES - ? sanitizedChunk.slice( - sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES, - ) - : sanitizedChunk; - - publishLiveEvent({ - companyId: run.companyId, - type: "heartbeat.run.log", - payload: { - runId: run.id, - agentId: run.agentId, - issueId, - ts, + outputProgressState.pending = { + at: new Date(ts), seq: chunkSeq, stream, - chunk: payloadChunk, - logRef: handle?.logRef, - logStore: handle?.store, - truncated: payloadChunk.length !== sanitizedChunk.length, - }, + bytes: persistedLogBytes, + }; + await flushOutputProgress(); + + // Streamed CLI output is real run activity: keep the in-memory + // runtime status ("Working... / X ago") fresh between structured + // events so sandbox runs with mid-run log streaming never show a + // minutes-stale timestamp. Throttled to avoid churning the live + // event stream on every 250ms tail chunk. + const logActivityAt = new Date(ts); + if ( + isHeartbeatRunRuntimeStatusActive(run.status) && + logActivityAt.getTime() - lastLogRuntimeStatusTouchMs >= + ACTIVE_RUN_LOG_RUNTIME_STATUS_REFRESH_INTERVAL_MS + ) { + lastLogRuntimeStatusTouchMs = logActivityAt.getTime(); + const touchedStatus = touchHeartbeatRunRuntimeStatus({ + companyId: run.companyId, + issueId, + agentId: run.agentId, + runId: run.id, + at: logActivityAt, + }); + if (touchedStatus) publishHeartbeatRunRuntimeProgress(touchedStatus); + } + + const payloadChunk = + sanitizedChunk.length > MAX_LIVE_LOG_CHUNK_BYTES + ? sanitizedChunk.slice(sanitizedChunk.length - MAX_LIVE_LOG_CHUNK_BYTES) + : sanitizedChunk; + + publishLiveEvent({ + companyId: run.companyId, + type: "heartbeat.run.log", + payload: { + runId: run.id, + agentId: run.agentId, + issueId, + ts, + seq: chunkSeq, + stream, + chunk: payloadChunk, + logRef: handle?.logRef, + logStore: handle?.store, + truncated: payloadChunk.length !== sanitizedChunk.length, + }, + }); }); + if (waitForCommit) await recorded; + else void recorded.catch(reportRecordingFailure); }; - const onLog = (stream: "stdout" | "stderr", chunk: string) => - appendIdentityRedactedLog(stream, identityRedactor.chunk(stream, chunk)); + const onLog = (stream: "stdout" | "stderr", chunk: string) => { + const external = handle?.store === "s3_segments"; + const recorded = appendIdentityRedactedLog( + stream, + identityRedactor.chunk(stream, chunk), + !external, + ); + return external ? recorded.catch(reportRecordingFailure) : recorded; + }; + const nativeExporter = + handle.store === "s3_segments" && run.runtimeMode === "native" + ? createNativeRunLogExporter({ + db, + companyId: run.companyId, + runId, + afterCursor: handle.initialSourceCursor ?? 0, + enqueue: (cursor, chunk) => + appendIdentityRedactedLog( + "stdout", + identityRedactor.chunk("stdout", chunk), + false, + cursor, + ), + flush: () => runLogStore.flush!(handle!), + }) + : undefined; + if (nativeExporter) await nativeExporter.recover(); + const onPersistedNativeLog = nativeExporter + ? (cursor: number, chunk: string) => + nativeExporter.record(cursor, chunk).catch(reportRecordingFailure) + : undefined; const onAdapterMeta = async (meta: AdapterInvocationMeta) => { meta = identityRedactor.redact(meta); if (meta.env && secretKeys.size > 0) { @@ -214,16 +310,29 @@ export function createHeartbeatRunOutput(db: Db, input: HeartbeatRunOutputInput) }); }; - return { currentUserRedactionOptions, appendIdentityRedactedLog, onLog, onAdapterMeta, onAdapterEvent }; + return { + currentUserRedactionOptions, + appendIdentityRedactedLog, + onLog, + onPersistedNativeLog, + onAdapterMeta, + onAdapterEvent, + }; } return { begin, - get handle() { return handle; }, + get handle() { + return handle; + }, outputProgressState, flushOutputProgress, // Finalization awaits I/O and can receive more output before persisting excerpts. - get stdoutExcerpt() { return stdoutExcerpt; }, - get stderrExcerpt() { return stderrExcerpt; }, + get stdoutExcerpt() { + return stdoutExcerpt; + }, + get stderrExcerpt() { + return stderrExcerpt; + }, }; } diff --git a/server/src/services/heartbeat/run-state.ts b/server/src/services/heartbeat/run-state.ts index af4d682291e..c2030afa7d1 100644 --- a/server/src/services/heartbeat/run-state.ts +++ b/server/src/services/heartbeat/run-state.ts @@ -355,6 +355,9 @@ const heartbeatRunLogAccessColumns = { ...heartbeatRunReadAccessColumns, logStore: heartbeatRuns.logStore, logRef: heartbeatRuns.logRef, + runtimeMode: heartbeatRuns.runtimeMode, + status: heartbeatRuns.status, + logBytes: heartbeatRuns.logBytes, } as const; const heartbeatRunIssueControlColumns = { diff --git a/server/src/services/heartbeat/runtime-execution.test.ts b/server/src/services/heartbeat/runtime-execution.test.ts index f40166ebd45..4f5b9c349dc 100644 --- a/server/src/services/heartbeat/runtime-execution.test.ts +++ b/server/src/services/heartbeat/runtime-execution.test.ts @@ -1,8 +1,10 @@ +import { createNativeUsageDelivery } from "../native-runtime/native-usage-delivery.js"; +import type { PrpEvent } from "../../vendor/paperclip-runner/index.js"; import { randomUUID } from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; +import { agents, budgetPolicies, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, executionWorkspaces, nativeRunFinalizations, runUsageReceipts, workspaceOperations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { trace } from "@opentelemetry/api"; @@ -213,6 +215,38 @@ describe.skipIf(!support.supported)("heartbeat runtime execution boundary", () = expectCleanup(input); }); + it("returns the accepted exact native rate-card price and preserves its live budget hold", async () => { + const { input, run } = await fixture(true); + input.workspace.executionTarget = { kind: "remote", transport: "sandbox", remoteCwd: home }; + input.runtime.nativeExecution!.provider = { kind: "codex", model: "gpt-6.1-sol" }; + await db.insert(budgetPolicies).values({ companyId: run.companyId, scopeType: "agent", scopeId: input.agent.id, + metric: "billed_cents", windowKind: "lifetime", amount: 100, notifyEnabled: false }); + let exact: string | undefined; + vi.mocked(executePaperclipNativeSession).mockImplementationOnce(async native => { + expect(native.onUsage).toBeUndefined(); + const delivery = await createNativeUsageDelivery(db, { companyId: run.companyId, runId: run.id, + runnerInstanceId: native.runnerInstanceId, provider: native.execution.provider, + billing: { provider: "openai", biller: "openai", billingType: "metered_api" } }); + const event = (sourceSeq: number, eventType: string, payload: Record): PrpEvent => ({ + sourceKind: "runner", sourceInstanceId: native.runnerInstanceId, runId: run.id, sourceSeq, sourceEventId: `priced-${sourceSeq}`, + turnId: "priced-turn", eventType, emittedAt: new Date().toISOString(), payload, + } as PrpEvent); + await delivery.capture(event(1, "usage.reported", { runDeltaBasis: "run", runDeltaAvailable: true, + runDelta: { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 0 } })); + await delivery.capture(event(2, "turn.completed", {})); + const saved = (await read(run.id)).usageJson!; + exact = saved.costUsdExact as string; + expect(Number(exact)).toBe(1.002); + expect(saved.budgetExposure).toMatchObject({ costCents: "100.2000000" }); + expect((await db.select().from(agents).where(eq(agents.id, input.agent.id)))[0].lifecycleHolds).toContain("budget"); + return { exitCode: 0, signal: null, timedOut: false, usageComplete: true, costUsd: null, model: "gpt-6.1-sol" }; + }); + const result = await executeHeartbeatRuntime(db, input); + if (!result.dispatched) throw new Error("expected native dispatch"); + expect(result.adapterResult).toMatchObject({ costUsdExact: exact, costStatus: "estimated", usageComplete: true, + pricingProvenance: { source: "rate_card" }, usage: { inputTokens: 1000, outputTokens: 100000 } }); + }); + it.each(["expected", "wrong"])("finalizes a plugin worktree through its owner (branch: %s)", async branch => { const { input } = await fixture(true); const [workspace] = await db.insert(executionWorkspaces).values({ companyId: input.agent.companyId, diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 00430730ba3..b5b64abc948 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1,3 +1,4 @@ +import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js"; import { DELIVERY_QUEUES } from "../delivery-work-notifications.js"; import { writeExecutionControl } from "../execution-control-notifications.js"; import { inspectPluginWorkspaceForFinalization } from "../native-runtime/native-plugin-workspace-finalization.js"; @@ -33,7 +34,7 @@ import { NativeWorkspaceFinalizationOwnershipLostError, type NativeWorkspaceFinalizationOwnership, } from "../native-runtime/native-workspace-finalization-ownership.js"; -import { createRunUsageRecorder } from "../usage-receipts.js"; +import { createRunUsageRecorder, readAcceptedRunUsageCheckpoint } from "../usage-receipts.js"; import { applyWorkspaceRestoreFailure } from "@paperclipai/adapter-utils/workspace-restore-result"; import { hasWorkspaceRestoreFailure } from "@paperclipai/shared"; import { isConversation } from "../agent-conversations.js"; @@ -172,6 +173,7 @@ export interface HeartbeatRuntimeExecutionInput { >; }; output: Pick & { + onPersistedNativeLog?: (cursor: number, chunk: string) => Promise; onAdapterEvent: NonNullable; onAdapterMeta: AdapterExecutionContext["onMeta"]; identityRedactor: ReturnType; @@ -495,9 +497,13 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe adapterFinalizeOutcome = status; }; - const usageRecorder = await measureRunPhase("execution.usage_startup", () => + const producerOwnsUsage = usesNativeUsageDelivery({ + runnerd: nativeRuntimeResolution.kind === "native" && agent.adapterType === "paperclip_runner", + remote: executionTarget?.kind === "remote", provider: nativeExecution?.provider, + }); + const usageRecorder = producerOwnsUsage ? undefined : await measureRunPhase("execution.usage_startup", () => createRunUsageRecorder(db, { companyId: run.companyId, runId: run.id, adapterType: agent.adapterType })); - input.effects.onUsageCaptureReady(usageRecorder.persistFailure); + input.effects.onUsageCaptureReady(usageRecorder?.persistFailure ?? (async () => {})); let adapterResult: AdapterExecutionResult; const runGoalControlRequestId = readNonEmptyString( context.goalControlRequestId, @@ -629,10 +635,11 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe goalCheckpointSession.current = { params, displayId }; }, onLog, + onPersistedNativeLog: input.output.onPersistedNativeLog, onEvent: onAdapterEvent, instructionWorkingCopy: nativeInstructionWorkingCopy(), - onUsage: async receipt => { await usageRecorder.capture(receipt); }, + onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined, preparationSpans: nativeRunnerPreparationSpans, // Bootstrap with executable/home discovery while keeping // configured provider values and the server-selected @@ -834,7 +841,7 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe onLog, onMeta: onAdapterMeta, onEvent: onAdapterEvent, - onUsage: async receipt => { await usageRecorder.capture(receipt); }, + onUsage: usageRecorder ? async receipt => { await usageRecorder.capture(receipt); } : undefined, onExecutionPhase: executionControl.phases.enter, startupTraceContext: getStartupTraceContext(), onRuntimeProgress: async (progress) => { @@ -913,8 +920,14 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe if (instructionSave) adapterResult.resultJson = { ...adapterResult.resultJson, instructionSave }; if (parseObject(adapterResult.executionRecovery).providerWorkStarted !== false) { - const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult)); - adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete }; + if (usageRecorder) { + const captured = await measureRunPhase("execution.usage", () => usageRecorder.complete(adapterResult)); + adapterResult = { ...adapterResult, ...captured, usageComplete: captured.complete }; + } else { + const accepted = await measureRunPhase("execution.usage", () => readAcceptedRunUsageCheckpoint(db, { companyId: run.companyId, runId: run.id })); + adapterResult = { ...adapterResult, ...accepted, usageComplete: accepted?.complete ?? false, + ...(!accepted ? { costStatus: "unpriced" as const } : {}) }; + } } else { // Stop may already own the terminal result. Preserve its metadata // while durably recording the proof needed to release admission. diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index e066949c630..42c65bb5d02 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -189,7 +189,8 @@ import { resolveIssueGoalId, resolveNextIssueGoalId, } from "./issue-goal-fallback.js"; -import { getRunLogStore } from "./run-log-store.js"; +import { getRunLogStore, isRunLogStoreType } from "./run-log-store.js"; +import { recoverSettledNativeRunLog } from "./native-runtime/native-run-log-recovery.js"; import { getDefaultCompanyGoal } from "./goals.js"; import { assertAssignableAgent } from "./agent-assignability.js"; import { @@ -6712,13 +6713,14 @@ export async function readIssueCommentRunLogText(run: { logStore: string | null; logRef: string | null; logBytes: number | null; -}) { - if (run.logStore !== "local_file" || !run.logRef) return ""; +}, db?: Db) { + if (!isRunLogStoreType(run.logStore) || !run.logRef) return ""; // A timed-out finalization leaves size unknown even when earlier entries // exist. Read those logs within the same byte budget as a known-size log. if (run.logBytes !== null && (!Number.isFinite(run.logBytes) || run.logBytes <= 0)) return ""; const logRef = run.logRef; + const storeType = run.logStore; const store = getRunLogStore(); let offset = 0; let content = ""; @@ -6727,6 +6729,8 @@ export async function readIssueCommentRunLogText(run: { let readTimer: NodeJS.Timeout | undefined; const readChunks = async () => { + if (db && storeType === "s3_segments") await recoverSettledNativeRunLog(db, store, logRef, undefined, controller.signal) + .catch(() => { controller.signal.throwIfAborted(); }); while (nextOffset !== undefined) { controller.signal.throwIfAborted(); const remainingBytes = @@ -6734,7 +6738,7 @@ export async function readIssueCommentRunLogText(run: { Buffer.byteLength(content, "utf8"); if (remainingBytes <= 0) break; const chunk = await store.read( - { store: "local_file", logRef }, + { store: storeType, logRef }, { offset, limitBytes: Math.min(ISSUE_COMMENT_RUN_LOG_DERIVATION_CHUNK_BYTES, remainingBytes), @@ -7259,7 +7263,7 @@ export function issueService(db: Db) { ); await Promise.all( batch.map(async (run) => { - logByRunId.set(run.runId, await readIssueCommentRunLogText(run)); + logByRunId.set(run.runId, await readIssueCommentRunLogText(run, db)); }), ); } @@ -12587,6 +12591,7 @@ export function issueService(db: Db) { order?: "asc" | "desc"; limit?: number | null; humanOnly?: boolean; + companyId?: string; }, ) => { const order = opts?.order === "asc" ? "asc" : "desc"; @@ -12597,6 +12602,7 @@ export function issueService(db: Db) { : null; const conditions = [eq(issueComments.issueId, issueId)]; + if (opts?.companyId) conditions.push(eq(issueComments.companyId, opts.companyId)); if (opts?.humanOnly) conditions.push(or( eq(issueComments.authorType, "user"), and(isNull(issueComments.authorType), isNotNull(issueComments.authorUserId)), diff --git a/server/src/services/native-runtime/__fixtures__/usage-receiver.ts b/server/src/services/native-runtime/__fixtures__/usage-receiver.ts new file mode 100644 index 00000000000..8b5a2805397 --- /dev/null +++ b/server/src/services/native-runtime/__fixtures__/usage-receiver.ts @@ -0,0 +1,45 @@ +import { resolve } from "node:path"; +import { createDb } from "@paperclipai/db"; +import { DurablePrpControlPlane, decodeDurablePrpCheckpoint, type DurablePrpCheckpoint } from "../../../vendor/paperclip-runner/index.js"; +import { createNativeUsageDelivery } from "../native-usage-delivery.js"; + +type Start = { + kind: "start"; requestId: number; connectionString: string; directory: string; port?: number; + identity: ConstructorParameters[0]["identity"]; + input: Parameters[1]; + checkpoint?: [string, string][]; +}; +let core: DurablePrpControlPlane | undefined; +let checkpoint: DurablePrpCheckpoint | undefined; +process.on("message", async (message: Start | { kind: "command"; requestId: number; type: string; commandId: string } | { kind: "stop"; requestId: number }) => { + try { + if (message.kind === "start") { + const delivery = await createNativeUsageDelivery(createDb(message.connectionString), message.input); + if (message.checkpoint) decodeDurablePrpCheckpoint(new Map(message.checkpoint)).restoreSessionDirectory(message.directory); + core = new DurablePrpControlPlane({ stateDirectory: resolve(message.directory, "control-plane"), identity: message.identity, + expectedRunnerVersion: "0.3.0", expectedRunnerDigest: `sha256:${"c".repeat(64)}`, + onCommittedEvent: async event => { + try { await delivery.capture(event); } + catch (error) { process.send?.({ kind: "capture-failed", cause: error instanceof Error ? error.message : String(error) }); throw error; } + }, + }); + core.setCheckpointPersistence(async saved => { checkpoint = saved; }); + await core.start(message.port); + const ticket = message.checkpoint ? undefined : core.issueBootstrapTicket(); + await core.flushCheckpoint(); + process.send?.({ requestId: message.requestId, connectUrl: core.connectUrl, ticket }); + } else if (message.kind === "command") { + core!.queueCommand(message.type, {}, message.commandId, true); + process.send?.({ requestId: message.requestId }); + } else { + await core?.stop(); + await core?.drainPendingConnectionProcessing(); + await core?.flushCheckpoint(); + process.send?.({ requestId: message.requestId, checkpoint: [...checkpoint!.records] }, () => process.exit(0)); + } + } catch (error) { + process.send?.({ requestId: message.requestId, error: error instanceof Error ? error.message : String(error) }); + } +}); + +process.send?.({ kind: "ready" }); diff --git a/server/src/services/native-runtime/native-github-access.test.ts b/server/src/services/native-runtime/native-github-access.test.ts index 444715a1b14..1f8b07172b1 100644 --- a/server/src/services/native-runtime/native-github-access.test.ts +++ b/server/src/services/native-runtime/native-github-access.test.ts @@ -3,6 +3,7 @@ import { spawn, execFile } from "node:child_process"; import { mkdtemp, mkdir, readFile, rm, writeFile, access } from "node:fs/promises"; import { promisify } from "node:util"; import type { CommandManagedDuplexChannel, CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import type { DuplexObservabilityRecorder } from "@paperclipai/adapter-utils/duplex-observability"; import { startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target"; import { tmpdir } from "node:os"; import path from "node:path"; @@ -39,7 +40,7 @@ function localDuplexChannel(command: readonly string[], cwd: string): CommandMan async function broker(resolveCredentials = vi.fn(async (binding: ReturnType) => ({ status: "available", env: { GH_TOKEN: `fixture-${binding.runId}` }, })), remote = false, bridgeUnavailable = false, onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise, -duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) { +duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean; recorder?: DuplexObservabilityRecorder }) { const root = await mkdtemp(path.join(tmpdir(), "native-github-reuse-")); cleanups.push(() => rm(root, { recursive: true, force: true })); const bin = path.join(root, "real-bin"); @@ -61,7 +62,7 @@ duplex?: { enabled?: boolean; capability: boolean; unavailable?: boolean }) { }); const target = remote ? { kind: "remote" as const, transport: "sandbox" as const, providerKey: "test", remoteCwd: root, runner: { execute, ...(duplex ? { openDuplexChannel } : {}) }, streamRunLogs: false, - ...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, effectiveCapabilities: { + ...(duplex ? { enableSandboxDuplexBridge: duplex.enabled, duplexObservabilityRecorder: duplex.recorder, effectiveCapabilities: { reusableLeases: true, nativeSyncIn: false, nativeSyncOut: false, persistentProcessSessions: false, independentControlCommands: true, incrementalSessionOutput: false, concurrentSyncOperations: false, duplexCommandStream: duplex.capability, runnerWebSocketIngress: false, @@ -122,9 +123,13 @@ it.each([false, true])("resolves managed GitHub credentials from OpenCode shell expect(childEnvironment.githubTokenPresent).toBe(false); }, 30000); it("uses the selected duplex transport across credential rotation without file polling", async () => { - const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true }); + const recorder = { recordSpan: vi.fn(), incrementCounter: vi.fn(), emitEvent: vi.fn() }; + const b = await broker(undefined, true, false, undefined, { enabled: true, capability: true, recorder }); expect(b.bridgeModes).toEqual(["http2_v1"]); expect(b.openDuplexChannel).toHaveBeenCalledOnce(); + expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.channel_open", dimensions: { + provider: "other", transport: "http2", outcome: "ok", + } })); const commandsAfterStartup = b.execute.mock.calls.length; expect((await request(b)).status).toBe(403); const releaseA = b.activate(run("a")); @@ -135,6 +140,8 @@ it("uses the selected duplex transport across credential rotation without file p releaseA(); expect(await (await request(b)).json()).toMatchObject({ env: { GH_TOKEN: "fixture-b" } }); expect(b.resolveCredentials.mock.calls.map(([binding]) => binding.runId)).toEqual(["a", "b"]); + expect(recorder.recordSpan).toHaveBeenCalledWith(expect.objectContaining({ name: "sandbox.duplex.request" })); + expect(recorder.emitEvent).not.toHaveBeenCalledWith(expect.objectContaining({ dimensions: expect.objectContaining({ transport: "file" }) })); expect(b.execute.mock.calls).toHaveLength(commandsAfterStartup); releaseB(); await b.stop(); diff --git a/server/src/services/native-runtime/native-github-access.ts b/server/src/services/native-runtime/native-github-access.ts index 87233394538..870dbbbbac9 100644 --- a/server/src/services/native-runtime/native-github-access.ts +++ b/server/src/services/native-runtime/native-github-access.ts @@ -3,6 +3,7 @@ import { createServer } from "node:http"; import { cleanupGitHubOperationLaunchers, adapterExecutionTargetEnablesSandboxDuplexBridge, + adapterExecutionTargetDuplexObservabilityRecorder, githubOperationLauncherDirectory, prepareGitHubOperationLaunchers, startAdapterExecutionTargetPaperclipBridge, @@ -107,6 +108,7 @@ export async function createNativeGitHubAccess(input: { hostApiToken: token, hostApiUrl: url, enableSandboxDuplexBridge: duplexRequested, + duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(input.target), onLog: input.onLog, }); if (input.target?.kind === "remote" && !bridge) { diff --git a/server/src/services/native-runtime/native-response-presentation-work.ts b/server/src/services/native-runtime/native-response-presentation-work.ts index aac11b54e3f..cad63e0f53c 100644 --- a/server/src/services/native-runtime/native-response-presentation-work.ts +++ b/server/src/services/native-runtime/native-response-presentation-work.ts @@ -7,6 +7,7 @@ import type { NativeResponsePresentationScope } from "../native-response-present import { onCommittedChatResponseAuthorityRegistered } from "../durable-chat-wakeup.js"; import { registerScopedDeadlineWork } from "../scoped-deadline-work.js"; import { recoverCommittedNativeChatResponse } from "./native-run-finalizer.js"; +import { measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js"; export function registerNativeResponsePresentationWork(db: Db, coordinator: ReturnType, canRun: () => boolean) { @@ -20,27 +21,35 @@ export function nativeResponsePresentationWork(db: Db) { const query = nativeResponsePresentationCandidates(scope, after, runId).as("pending_native_response"); return db.select().from(query).orderBy(asc(query.id)); }; - async function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise { + function load(scope?: NativeResponsePresentationScope, signal?: AbortSignal, canRun = () => true): Promise { + const work = () => loadWithinTrace(scope, signal, canRun); + return scope?.runId ? withRunResponsePresentationTiming(scope.runId, work) : work(); + } + async function loadWithinTrace(scope: NativeResponsePresentationScope | undefined, signal: AbortSignal | undefined, + canRun: () => boolean): Promise { const entries: Entry[] = []; let after: string | undefined; while (canRun() && !signal?.aborted) { - const rows = await candidates(scope, after); + const rows = await measureRunPhase("response.resolve", () => candidates(scope, after)); for (const row of rows) entries.push({ id: row.id, companyId: row.companyId, issueId: row.issueId, agentId: row.agentId, dueAt: row.nextAttemptAt?.getTime() ?? 0 }); if (rows.length < 100) return entries; after = rows[rows.length - 1].id; } throw new Error("Native response presentation discovery stopped"); } - async function check(entry: Entry, signal?: AbortSignal): Promise { + function check(entry: Entry, signal?: AbortSignal): Promise { + return withRunResponsePresentationTiming(entry.id, () => checkWithinTrace(entry, signal)); + } + async function checkWithinTrace(entry: Entry, signal?: AbortSignal): Promise { if (signal?.aborted) throw new Error("Native response presentation recovery stopped"); const scope = { companyId: entry.companyId, issueId: entry.issueId }; - const [current] = await candidates(scope, undefined, entry.id); + const [current] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id)); if (!current) return null; if (current.nextAttemptAt && current.nextAttemptAt.getTime() > Date.now()) return { ...entry, dueAt: current.nextAttemptAt.getTime() }; if (signal?.aborted) throw new Error("Native response presentation recovery stopped"); const recoveryStartedAt = Date.now(); await recoverCommittedNativeChatResponse(db, { ...scope, runId: entry.id, onlyWhenDue: true }); - const [remaining] = await candidates(scope, undefined, entry.id); + const [remaining] = await measureRunPhase("response.resolve", () => candidates(scope, undefined, entry.id)); return remaining?.nextAttemptAt && remaining.nextAttemptAt.getTime() > recoveryStartedAt ? { ...entry, dueAt: remaining.nextAttemptAt.getTime() } : null; } diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index ba2017eb9a8..202d2b4eac3 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -67,6 +67,7 @@ import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { issueService } from "../issues.js"; import { publishChatPublicationCommitSignal } from "../chat-publication-reconciliation.js"; import { nativeSha256 } from "./canonical.js"; +import { beginRunPhase, measureRunPhase, withRunResponsePresentationTiming } from "../execution-preparation-timing.js"; import { readNativeBoardResponseWaitOrigin, readNativeBoardResponseWaitSource, @@ -736,11 +737,20 @@ export async function repairCommittedNativeChatResponse( export async function recoverCommittedNativeChatResponse( db: Db, input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean }, +): Promise { + return withRunResponsePresentationTiming(input.runId, () => recoverCommittedNativeChatResponseInScope(db, input)); +} + +async function recoverCommittedNativeChatResponseInScope( + db: Db, + input: { companyId: string; issueId: string; runId: string; onlyWhenDue?: boolean }, ): Promise { let agentId: string | null = null; let materialized = false; + const entered = beginRunPhase("response.transaction.entry"); try { - materialized = await db.transaction(async (transaction) => { + materialized = await measureRunPhase("response.transaction", () => db.transaction(async (transaction) => { + entered(); const tx = transaction as unknown as Db; @@ -1022,10 +1032,12 @@ export async function recoverCommittedNativeChatResponse( await projectCommittedRun({ db: tx, run, coordinator }); agentId = run.agentId; return true; - }); + })); } catch (error) { if (error instanceof CommittedChatResponseAuthorizationError) return false; throw error; + } finally { + entered(); } if (materialized && agentId) publishChatPublicationCommitSignal({ diff --git a/server/src/services/native-runtime/native-run-log-export.integration.test.ts b/server/src/services/native-runtime/native-run-log-export.integration.test.ts new file mode 100644 index 00000000000..582bab13407 --- /dev/null +++ b/server/src/services/native-runtime/native-run-log-export.integration.test.ts @@ -0,0 +1,474 @@ +import { randomUUID, createHash, generateKeyPairSync } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { + createDb, + heartbeatRunEvents, + heartbeatRuns, + agentIdentityKeys, + type Db, +} from "@paperclipai/db"; +import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectsCommand, + ListObjectsV2Command, +} from "@aws-sdk/client-s3"; +import { createS3StorageProvider } from "../../storage/s3-provider.js"; +import { createObjectRunLogStore } from "../object-run-log-store.js"; +import { createNativeRunLogExporter, formatNativeRunLogEvent } from "./native-run-log-export.js"; +import { recoverSettledNativeRunLog, readRecoverableRunLog } from "./native-run-log-recovery.js"; +import { PaperclipControlPlanePort } from "./paperclip-control-plane-port.js"; +import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; +import { createNativeRunLogTestTables } from "../../__tests__/helpers/native-run-log-database.js"; +import type { PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import type { StorageProvider } from "../../storage/types.js"; +import { appendNativeCredentialUpdate } from "./native-credential-updates.js"; +import { appendRetainedNativeCleanupEvent } from "./native-session-executor.js"; +import { idleWorkSnapshot } from "../task-admission.js"; +import { appendHeartbeatRunEvent } from "../heartbeat-run-events.js"; +import { localEncryptedProvider } from "../../secrets/local-encrypted-provider.js"; +import { createHeartbeatRunOutput } from "../heartbeat/run-output.js"; +import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { ObjectJournalConflictError } from "../object-byte-journal.js"; + +const url = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL"); +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +describe.skipIf(!url || !endpoint)("native run log source recovery", () => { + const schema = `native_log_${randomUUID().replaceAll("-", "")}`, + bucket = `pc-native-log-${randomUUID()}`; + let db: Db, + client: S3Client, + bucketCreated = false; + beforeAll(async () => { + db = createDb(url!, { maxConnections: 1, idleTimeoutSeconds: 0 }); + await db.execute(sql.raw(`create schema ${schema}`)); + await db.execute(sql.raw(`set search_path=${schema}`)); + await createNativeRunLogTestTables(db); + client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + bucketCreated = true; + }); + afterEach(() => vi.unstubAllEnvs()); + afterAll(async () => { + if (client && bucketCreated) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((item) => ({ Key: item.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + } + if (db) { + await db.execute(sql.raw(`drop schema ${schema} cascade`)); + await db.$client.end(); + } + }); + async function fixture() { + const binding = { + companyId: randomUUID(), + runId: randomUUID(), + agentId: randomUUID(), + sessionId: randomUUID(), + issueId: randomUUID(), + sourceInstanceId: randomUUID(), + controlPlaneSourceInstanceId: randomUUID(), + completionContractId: randomUUID(), + completionContractSha256: "test", + }; + await db.execute( + sql`insert into heartbeat_runs(id,company_id,agent_id,status) values(${binding.runId},${binding.companyId},${binding.agentId},'running')`, + ); + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + const store = (storage: StorageProvider = provider) => + createObjectRunLogStore({ + provider: storage, + keyPrefix: "logs", + flushMs: 30_000, + assertWriterAuthority: async (input) => { + await input.assertWriterAuthority?.(); + }, + }); + const begin = { companyId: binding.companyId, agentId: binding.agentId, runId: binding.runId }; + const port = new PaperclipControlPlanePort(db, binding); + const event = (seq: number): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + sourceInstanceId: binding.sourceInstanceId, + sourceEventId: `${binding.sourceInstanceId}:${seq}`, + sourceSeq: seq, + sourceKind: "runner", + runId: binding.runId, + normalizedSessionId: binding.sessionId, + eventType: "item.delta", + priority: 2, + emittedAt: new Date().toISOString(), + payload: { text: `step ${seq} 再開` }, + }); + return { binding, store, begin, port, event, provider }; + } + it("replays acknowledged native records into a new writer, repairs gaps, and ignores late duplicates", async () => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin); + await db.execute( + sql`update heartbeat_runs set log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + const events = [f.event(1), f.event(2), f.event(3)]; + const cursors = await Promise.all(events.slice(0, 2).map((event) => f.port.appendEvent(event))); + const receipt = await a.enqueue!(h, { + stream: "stdout", + chunk: formatNativeRunLogEvent(events[0]!), + ts: "then", + seq: 1, + sourceCursor: cursors[0]!.cursor, + }); + await a.flush!(h); + await receipt.committed; + const b = f.store(), + successor = await b.begin(f.begin); + let seq = successor.initialSequence!; + const exporter = createNativeRunLogExporter({ + db, + companyId: f.binding.companyId, + runId: f.binding.runId, + afterCursor: successor.initialSourceCursor!, + enqueue: async (sourceCursor, chunk) => { + const queued = await b.enqueue!(successor, { + stream: "stdout", + chunk, + ts: "now", + seq: ++seq, + sourceCursor, + }); + void queued.committed.catch(() => undefined); + }, + flush: () => b.flush!(successor), + }); + const third = await f.port.appendEvent(events[2]!); + await exporter.record(third.cursor, formatNativeRunLogEvent(events[2]!)); + await exporter.record(cursors[0]!.cursor, formatNativeRunLogEvent(events[0]!)); + await b.finalize(successor); + const entries = (await b.read(successor)).content + .trim() + .split("\n") + .map((line) => JSON.parse(JSON.parse(line).chunk).event); + expect(entries.map((event) => event.sourceSeq)).toEqual([1, 2, 3]); + expect( + await db + .select() + .from(heartbeatRunEvents) + .where(eq(heartbeatRunEvents.runId, f.binding.runId)), + ).toHaveLength(3); + }); + it("repairs a settled run on demand after its controller lease expires, without extending source retention", async () => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin); + await f.port.appendEvent(f.event(1)); + await db.execute( + sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + await db.execute( + sql`insert into native_run_finalizations values(${f.binding.runId},${f.binding.companyId},now()+interval '1 minute')`, + ); + const b = f.store(); + await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })); + expect((await b.read(h)).content).toBe(""); + await db.execute( + sql`update native_run_finalizations set lease_expires_at=now()-interval '1 second' where run_id=${f.binding.runId}`, + ); + await Promise.all( + [1, 2].map(() => + recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })), + ), + ); + expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1); + const [run] = await db + .select({ bytes: heartbeatRuns.logBytes }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.binding.runId)); + expect(run!.bytes).toBeGreaterThan(0); + }); + it.each(["hash", "payload", "identity"])( + "rejects a corrupt canonical source %s without marking the partial archive complete", + async (kind) => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin); + await f.port.appendEvent(f.event(1)); + if (kind === "hash") + await db.execute( + sql`update heartbeat_run_events set source_payload_sha256='wrong' where run_id=${f.binding.runId}`, + ); + else if (kind === "payload") + await db.execute( + sql`update heartbeat_run_events set payload='{}'::jsonb where run_id=${f.binding.runId}`, + ); + else + await db.execute( + sql`update heartbeat_run_events set source_instance_id='wrong' where run_id=${f.binding.runId}`, + ); + await db.execute( + sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + await expect( + recoverSettledNativeRunLog(db, f.store(), h.logRef, async () => ({ enabled: false })), + ).rejects.toThrow("integrity"); + const [run] = await db + .select({ bytes: heartbeatRuns.logBytes }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.binding.runId)); + expect(run!.bytes).toBeNull(); + }, + ); + it("ignores credential and retained cleanup projections when replaying canonical log sources", async () => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin); + await f.port.appendEvent(f.event(1)); + const credential = (sourceSeq: number): PrpEvent => ({ + ...f.event(sourceSeq), + schema: "paperclip.prp.event.v3", + schemaVersion: 3, + priority: 0, + eventType: "auth.credentials.updated", + payload: { + schema: "paperclip.auth.credentials.updated.v1", + provider: "codex", + bindingId: "test-auth", + revision: 1, + detection: "credential_store", + }, + }); + await f.port.appendEvent(credential(2)); + await db.execute(sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef}, + native_session_id=${f.binding.sessionId}, runner_instance_id=${f.binding.sourceInstanceId} where id=${f.binding.runId}`); + await appendNativeCredentialUpdate(db, { + companyId: f.binding.companyId, + agentId: f.binding.agentId, + normalizedSessionId: f.binding.sessionId, + runnerInstanceId: f.binding.sourceInstanceId, + event: credential(20), + }); + await appendRetainedNativeCleanupEvent(db, { + companyId: f.binding.companyId, + agentId: f.binding.agentId, + runId: f.binding.runId, + nativeSessionId: f.binding.sessionId, + runnerInstanceId: f.binding.sourceInstanceId, + requestId: "test-cleanup", + event: f.event(21), + }); + await appendHeartbeatRunEvent(db, { + companyId: f.binding.companyId, + agentId: f.binding.agentId, + runId: f.binding.runId, + eventType: "workspace.diff.recorded", + payload: { complete: true }, + nativeSource: { + sourceInstanceId: "provider-trace-reproject:test", + sourceEventId: "provider-trace-reproject:test:change", + sourceSeq: 1, + protocolSchemaVersion: 1, + canonicalPayload: { complete: true }, + }, + }); + const b = f.store(); + await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })); + const entries = (await b.read(h)).content.trim().split("\n"); + expect(entries).toHaveLength(2); + expect(JSON.parse(JSON.parse(entries[0]!).chunk).event.sourceSeq).toBe(1); + expect(JSON.parse(JSON.parse(entries[1]!).chunk).event.eventType).toBe( + "auth.credentials.updated", + ); + const [run] = await db + .select({ bytes: heartbeatRuns.logBytes }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.binding.runId)); + expect(run!.bytes).toBeGreaterThan(0); + }); + it("aborts in-flight archive writes when its last reader cancels, then resumes without loss", async () => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin), + baseline = idleWorkSnapshot().active; + await f.port.appendEvent(f.event(1)); + await db.execute( + sql`update heartbeat_runs set status='succeeded', log_store='s3_segments', log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + let entered = false, + aborted = false; + const blocked: StorageProvider = { + ...f.provider, + async putObjectConditional(input) { + if (input.objectKey.includes("/objects/")) { + entered = true; + await new Promise((_resolve, reject) => + input.signal!.addEventListener( + "abort", + () => { + aborted = true; + reject(input.signal!.reason); + }, + { once: true }, + ), + ); + } + return f.provider.putObjectConditional!(input); + }, + }; + const controller = new AbortController(); + const repairing = recoverSettledNativeRunLog( + db, + f.store(blocked), + h.logRef, + async () => ({ enabled: false }), + controller.signal, + ); + const rejected = expect(repairing).rejects.toThrow("reader cancelled"); + const lastReader = new AbortController(); + const joined = recoverSettledNativeRunLog( + db, + f.store(blocked), + h.logRef, + async () => ({ enabled: false }), + lastReader.signal, + ); + const joinedRejected = expect(joined).rejects.toThrow("last reader cancelled"); + await vi.waitFor(() => expect(entered).toBe(true)); + controller.abort(new Error("reader cancelled")); + await rejected; + expect(aborted).toBe(false); + lastReader.abort(new Error("last reader cancelled")); + await joinedRejected; + await vi.waitFor(() => { + expect(aborted).toBe(true); + expect(idleWorkSnapshot().active).toBe(baseline); + }); + const b = f.store(); + await vi.waitFor(async () => { + await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })); + }); + expect((await b.read(h)).content.trim().split("\n")).toHaveLength(1); + }); + it("redacts the existing agent identity and registered secrets before publishing recovered native output", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "e".repeat(64)); + const f = await fixture(); + const keys = generateKeyPairSync("ed25519"), + privateKeyPem = keys.privateKey.export({ type: "pkcs8", format: "pem" }).toString(); + const publicKeyPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const keyId = `sha256:${createHash("sha256") + .update(keys.publicKey.export({ type: "spki", format: "der" })) + .digest("base64url")}`; + const encrypted = await localEncryptedProvider.createSecret({ value: privateKeyPem }); + await db.insert(agentIdentityKeys).values({ + agentId: f.binding.agentId, + companyId: f.binding.companyId, + algorithm: "Ed25519", + keyId, + publicKeyPem, + privateKeyMaterial: encrypted.material, + }); + const secret = "registered-native-fixture-secret"; + const redactor = createAgentIdentityRedactor(privateKeyPem); + redactor.values.push(secret); + const run = { + id: f.binding.runId, + companyId: f.binding.companyId, + agentId: f.binding.agentId, + status: "running", + runtimeMode: "native", + logStore: null, + logRef: null, + logBytes: null, + lastOutputSeq: 0, + lastOutputAt: null, + } as typeof heartbeatRuns.$inferSelect; + const output = createHeartbeatRunOutput(db, { + run, + runId: run.id, + issueId: null, + runLogStore: f.store(), + identityRedactor: redactor, + secretKeys: new Set(), + runtimeLogSecretValues: [secret], + appendRunEvent: async () => undefined, + getCurrentUserRedactionOptions: async () => ({ enabled: false }), + }); + await output.begin(run); + const h = output.handle!; + await f.port.appendEvent({ ...f.event(1), payload: { text: `${privateKeyPem}\n${secret}` } }); + await db.execute( + sql`update heartbeat_runs set status='succeeded', log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + const b = f.store(); + await recoverSettledNativeRunLog(db, b, h.logRef, async () => ({ enabled: false })); + const content = (await b.read(h)).content; + expect(content).not.toContain(privateKeyPem.split("\n")[1]!); + expect(content).not.toContain(secret); + expect(content).toContain("REDACTED"); + const [snapshot] = await db + .select({ context: heartbeatRuns.contextSnapshot }) + .from(heartbeatRuns) + .where(eq(heartbeatRuns.id, f.binding.runId)); + expect(JSON.stringify(snapshot!.context)).not.toContain(privateKeyPem.split("\n")[1]!); + }); + it("serves the published snapshot when another writer wins a repair claim", async () => { + const f = await fixture(), + a = f.store(), + h = await a.begin(f.begin); + const event = f.event(1), + receipt = await f.port.appendEvent(event); + const queued = await a.enqueue!(h, { + stream: "stdout", + chunk: formatNativeRunLogEvent(event), + ts: "then", + seq: 1, + sourceCursor: receipt.cursor, + }); + await a.flush!(h); + await queued.committed; + await f.port.appendEvent(f.event(2)); + await db.execute( + sql`update heartbeat_runs set status='succeeded',log_store='s3_segments',log_ref=${h.logRef} where id=${f.binding.runId}`, + ); + let enter!: () => void, release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }), + blocked = new Promise((resolve) => { + release = resolve; + }); + const store = { + ...f.store(), + async begin() { + enter(); + await blocked; + throw new ObjectJournalConflictError(); + }, + }; + const repairing = recoverSettledNativeRunLog(db, store, h.logRef, async () => ({ + enabled: false, + })); + const rejected = expect(repairing).rejects.toBeInstanceOf(ObjectJournalConflictError); + await entered; + const read = readRecoverableRunLog(db, store, h); + release(); + await rejected; + expect((await read).content.trim().split("\n")).toHaveLength(1); + await a.finalize(h); + }); +}); diff --git a/server/src/services/native-runtime/native-run-log-export.ts b/server/src/services/native-runtime/native-run-log-export.ts new file mode 100644 index 00000000000..d8693c56d64 --- /dev/null +++ b/server/src/services/native-runtime/native-run-log-export.ts @@ -0,0 +1,210 @@ +import { and, asc, desc, eq, gt, isNotNull, lte, not, or, sql } from "drizzle-orm"; +import { StringDecoder } from "node:string_decoder"; +import { heartbeatRunEvents, type Db } from "@paperclipai/db"; +import { validatePrpEvent, type PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { nativeSha256 } from "./canonical.js"; + +export function formatNativeRunLogEvent(event: PrpEvent) { + return `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`; +} +export function nativeRunLogSourceWhere(companyId: string, runId: string) { + return and( + eq(heartbeatRunEvents.companyId, companyId), + eq(heartbeatRunEvents.runId, runId), + isNotNull(heartbeatRunEvents.sourceEventId), + not( + sql`coalesce((${or( + and( + eq(heartbeatRunEvents.eventType, "auth.credentials.updated"), + sql`${heartbeatRunEvents.sourceEventId} like 'credentials:%'`, + sql`${heartbeatRunEvents.sourceInstanceId} like '%:credentials'`, + ), + and( + eq(heartbeatRunEvents.eventType, "native.cleanup.event"), + sql`${heartbeatRunEvents.sourceEventId} like 'cleanup:%'`, + sql`${heartbeatRunEvents.sourceInstanceId} like '%:cleanup:%'`, + ), + and( + eq(heartbeatRunEvents.eventType, "workspace.diff.recorded"), + sql`${heartbeatRunEvents.sourceEventId} like 'provider-trace-reproject:%'`, + sql`${heartbeatRunEvents.sourceInstanceId} like 'provider-trace-reproject:%'`, + ), + )}), false)`, + ), + ); +} +async function sourcePage( + db: Db, + companyId: string, + runId: string, + after: number, + through?: number, + signal?: AbortSignal, +) { + signal?.throwIfAborted(); + const rows = await db + .select({ + seq: heartbeatRunEvents.seq, + payload: heartbeatRunEvents.payload, + sourceInstanceId: heartbeatRunEvents.sourceInstanceId, + sourceEventId: heartbeatRunEvents.sourceEventId, + sourceSeq: heartbeatRunEvents.sourceSeq, + sha256: heartbeatRunEvents.sourcePayloadSha256, + }) + .from(heartbeatRunEvents) + .where( + and( + nativeRunLogSourceWhere(companyId, runId), + gt(heartbeatRunEvents.seq, after), + through === undefined ? undefined : lte(heartbeatRunEvents.seq, through), + ), + ) + .orderBy(asc(heartbeatRunEvents.seq)) + .limit(16); + signal?.throwIfAborted(); + return rows.map((row) => { + const validated = validatePrpEvent(row.payload?.prpEvent); + if ( + !validated.ok || + validated.event.runId !== runId || + validated.event.sourceInstanceId !== row.sourceInstanceId || + validated.event.sourceEventId !== row.sourceEventId || + validated.event.sourceSeq !== row.sourceSeq || + nativeSha256(validated.event) !== row.sha256 + ) + throw new Error("native_run_log_source_integrity_error"); + return { cursor: row.seq, event: validated.event }; + }); +} + +export function createNativeRunLogCoverageVerifier(db: Db, companyId: string, runId: string) { + const decoder = new StringDecoder("utf8"); + let text = "", + cursor = 0; + let page: Awaited> = []; + const unproven = () => new Error("native_log_source_coverage_unproven"); + async function next() { + if (!page.length) page = await sourcePage(db, companyId, runId, cursor); + return page[0]; + } + async function line(value: string) { + if (!value.trim()) return; + if (Buffer.byteLength(value) > 1024 * 1024) throw unproven(); + let chunk: unknown; + try { + chunk = JSON.parse(value).chunk; + } catch { + throw unproven(); + } + if (typeof chunk !== "string" || !chunk.startsWith('{"type":"paperclip.prp.event"')) return; + let event: PrpEvent; + try { + const parsed = validatePrpEvent(JSON.parse(chunk).event); + if (!parsed.ok) throw unproven(); + event = parsed.event; + } catch { + throw unproven(); + } + const expected = await next(); + if ( + !expected || + event.runId !== runId || + event.sourceInstanceId !== expected.event.sourceInstanceId || + event.sourceEventId !== expected.event.sourceEventId || + event.sourceSeq !== expected.event.sourceSeq || + event.normalizedSessionId !== expected.event.normalizedSessionId || + event.schemaVersion !== expected.event.schemaVersion || + event.eventType !== expected.event.eventType + ) + throw unproven(); + cursor = expected.cursor; + page.shift(); + } + return { + async write(bytes: Buffer) { + text += decoder.write(bytes); + let boundary: number; + while ((boundary = text.indexOf("\n")) !== -1) { + await line(text.slice(0, boundary)); + text = text.slice(boundary + 1); + } + if (Buffer.byteLength(text) > 1024 * 1024) throw unproven(); + }, + async finish() { + text += decoder.end(); + if (text) await line(text); + if (await next()) throw unproven(); + return cursor; + }, + }; +} + +export function createNativeRunLogExporter(input: { + db: Db; + companyId: string; + runId: string; + afterCursor: number; + enqueue: (cursor: number, chunk: string) => Promise; + flush: () => Promise; + signal?: AbortSignal; +}) { + let cursor = input.afterCursor; + let queue = Promise.resolve(); + let queuedBytes = 0, + queuedRecords = 0; + const sourceWhere = nativeRunLogSourceWhere(input.companyId, input.runId); + async function replay(through?: number) { + input.signal?.throwIfAborted(); + if (through === undefined) { + const [last] = await input.db + .select({ seq: heartbeatRunEvents.seq }) + .from(heartbeatRunEvents) + .where(sourceWhere) + .orderBy(desc(heartbeatRunEvents.seq)) + .limit(1); + through = last?.seq ?? cursor; + } + while (cursor < through) { + const rows = await sourcePage( + input.db, + input.companyId, + input.runId, + cursor, + through, + input.signal, + ); + if (!rows.length) break; + for (const row of rows) { + input.signal?.throwIfAborted(); + await input.enqueue(row.cursor, formatNativeRunLogEvent(row.event)); + cursor = row.cursor; + } + await input.flush(); + } + } + return { + recover: () => replay(), + async record(sourceCursor: number, chunk: string) { + if (!Number.isSafeInteger(sourceCursor) || sourceCursor < 1) + throw new Error("invalid_native_run_log_cursor"); + const bytes = Buffer.byteLength(chunk); + if (queuedRecords >= 32 || queuedBytes + bytes > 8 * 1024 * 1024) + throw new Error("native_run_log_admission_capacity_exceeded"); + queuedRecords++; + queuedBytes += bytes; + const work = queue + .then(async () => { + if (sourceCursor <= cursor) return; + if (sourceCursor > cursor + 1) await replay(sourceCursor - 1); + await input.enqueue(sourceCursor, chunk); + cursor = sourceCursor; + }) + .finally(() => { + queuedRecords--; + queuedBytes -= bytes; + }); + queue = work.catch(() => undefined); + await work; + }, + }; +} diff --git a/server/src/services/native-runtime/native-run-log-recovery.ts b/server/src/services/native-runtime/native-run-log-recovery.ts new file mode 100644 index 00000000000..b3fd0c36aaf --- /dev/null +++ b/server/src/services/native-runtime/native-run-log-recovery.ts @@ -0,0 +1,223 @@ +import { and, eq, gt, inArray, isNull, sql } from "drizzle-orm"; +import { + heartbeatRuns, + heartbeatRunEvents, + nativeRunFinalizations, + type Db, +} from "@paperclipai/db"; +import { redactCurrentUserText, type CurrentUserRedactionOptions } from "../../log-redaction.js"; +import type { RunLogHandle, RunLogReadOptions, RunLogStore } from "../run-log-store.js"; +import { instanceSettingsService } from "../instance-settings.js"; +import { compactRunLogChunk } from "../heartbeat/run-log.js"; +import { beginIdleTrackedWork } from "../task-admission.js"; +import { agentIdentityService } from "../agent-identity.js"; +import { createRunSecretRedactionRegistry } from "../run-secret-redaction.js"; +import { createAgentIdentityRedactor } from "../agent-identity-redaction.js"; +import { logger } from "../../middleware/logger.js"; +import { createNativeRunLogExporter, nativeRunLogSourceWhere } from "./native-run-log-export.js"; + +interface Recovery { + work: Promise; + controller: AbortController; + readers: number; +} +const recoveries = new WeakMap>(); +function awaitRecovery(recovery: Recovery, signal?: AbortSignal) { + signal?.throwIfAborted(); + recovery.readers++; + return new Promise((resolve, reject) => { + let finished = false; + const finish = () => { + if (finished) return false; + finished = true; + signal?.removeEventListener("abort", abort); + recovery.readers--; + return true; + }; + const abort = () => { + if (!finish()) return; + if (!recovery.readers) recovery.controller.abort(signal!.reason); + reject(signal!.reason); + }; + signal?.addEventListener("abort", abort, { once: true }); + recovery.work.then( + () => { + if (finish()) resolve(); + }, + (error) => { + if (finish()) reject(error); + }, + ); + }); +} + +export async function readRecoverableRunLog( + db: Db, + store: RunLogStore, + handle: RunLogHandle, + options?: RunLogReadOptions, + known?: { runtimeMode?: string | null; status?: string; logBytes?: number | null }, +) { + if ( + handle.store === "s3_segments" && + known?.runtimeMode !== "legacy" && + known?.status !== "running" && + (known?.logBytes === undefined || known.logBytes === null) + ) + await recoverSettledNativeRunLog(db, store, handle.logRef, undefined, options?.signal).catch( + () => { + options?.signal?.throwIfAborted(); + logger.warn( + { runId: handle.logRef.split("/").at(-1) }, + "Native run log repair is incomplete; serving the published prefix", + ); + }, + ); + return store.read(handle, options); +} + +export async function recoverSettledNativeRunLog( + db: Db, + store: RunLogStore, + logRef: string, + getRedactionOptions: () => Promise = async () => ({ + enabled: (await instanceSettingsService(db).getGeneral()).censorUsernameInLogs, + }), + requestSignal?: AbortSignal, +) { + requestSignal?.throwIfAborted(); + const binding = /^([a-f0-9-]+)\/([a-f0-9-]+)\/([a-f0-9-]+)$/.exec(logRef); + if (!binding) throw new Error("invalid_native_run_log_reference"); + let pending = recoveries.get(db); + if (!pending) { + pending = new Map(); + recoveries.set(db, pending); + } + const existing = pending.get(logRef); + if (existing) return awaitRecovery(existing, requestSignal); + const controller = new AbortController(); + const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(30_000)]); + const finishWork = beginIdleTrackedWork(); + const runId = binding[3]!, + companyId = binding[1]!; + const eligible = () => + and( + eq(heartbeatRuns.id, runId), + eq(heartbeatRuns.companyId, companyId), + eq(heartbeatRuns.agentId, binding[2]!), + eq(heartbeatRuns.logStore, "s3_segments"), + eq(heartbeatRuns.logRef, logRef), + eq(heartbeatRuns.runtimeMode, "native"), + inArray(heartbeatRuns.status, ["succeeded", "failed", "cancelled", "timed_out"]), + isNull(heartbeatRuns.logBytes), + sql`not exists (select 1 from ${nativeRunFinalizations} where ${and( + eq(nativeRunFinalizations.runId, runId), + eq(nativeRunFinalizations.companyId, companyId), + gt(nativeRunFinalizations.leaseExpiresAt, new Date()), + )})`, + ); + const recover = (async () => { + signal.throwIfAborted(); + const [run] = await db.select({ id: heartbeatRuns.id }).from(heartbeatRuns).where(eligible()); + signal.throwIfAborted(); + if (!run) return; + const [redaction, identityRedactor, secrets] = await Promise.all([ + getRedactionOptions(), + agentIdentityService(db).createExistingLogRedactor(companyId, binding[2]!), + createRunSecretRedactionRegistry(db).redactionValuesForRun(companyId, runId), + ]); + for (const value of secrets) + identityRedactor.values.push(...createAgentIdentityRedactor(value).values); + identityRedactor.values.sort((left, right) => right.length - left.length); + const handle = await store.begin({ + companyId, + agentId: binding[2]!, + runId, + previousStore: "s3_segments", + signal, + assertWriterAuthority: async () => { + signal.throwIfAborted(); + const [current] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where(eligible()); + if (!current) throw new Error("native_run_log_recovery_authority_lost"); + }, + }); + let sequence = handle.initialSequence ?? 0; + let lastSourceCursor = handle.initialSourceCursor ?? 0; + const receipts: Promise[] = []; + let failure: unknown; + const exporter = createNativeRunLogExporter({ + db, + companyId, + runId, + afterCursor: handle.initialSourceCursor ?? 0, + signal, + enqueue: async (sourceCursor, chunk) => { + const receipt = await store.enqueue!(handle, { + stream: "stdout", + ts: new Date().toISOString(), + seq: ++sequence, + sourceCursor, + chunk: compactRunLogChunk( + redactCurrentUserText(identityRedactor.chunk("stdout", chunk), redaction), + ), + }); + receipts.push( + receipt.committed.catch((error) => { + failure ??= error; + return 0; + }), + ); + lastSourceCursor = sourceCursor; + }, + flush: async () => { + await store.flush!(handle); + await Promise.all(receipts.splice(0)); + if (failure) throw failure; + }, + }); + try { + await exporter.recover(); + } catch (error) { + failure ??= error; + } finally { + const summary = await store.finalize(handle, { computeDigest: false }); + if (summary.bytes === null) failure ??= new Error("native_run_log_recovery_incomplete"); + if (!failure && summary.bytes !== null) { + signal.throwIfAborted(); + const updated = await db + .update(heartbeatRuns) + .set({ + logBytes: summary.bytes, + logSha256: summary.sha256, + lastOutputSeq: sequence, + lastOutputBytes: summary.bytes, + }) + .where( + and( + eligible(), + sql`not exists (select 1 from ${heartbeatRunEvents} where ${and( + nativeRunLogSourceWhere(companyId, runId), + gt(heartbeatRunEvents.seq, lastSourceCursor), + )})`, + ), + ) + .returning({ id: heartbeatRuns.id }); + if (!updated.length) throw new Error("native_run_log_recovery_source_changed"); + } + } + if (failure) throw failure; + })(); + const recovery: Recovery = { + controller, + readers: 0, + work: recover.finally(() => { + pending.delete(logRef); + finishWork(); + }), + }; + pending.set(logRef, recovery); + await awaitRecovery(recovery, requestSignal); +} diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b7775c11fda..30d94805b98 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -110,6 +110,10 @@ vi.mock("../usage-receipts.js", async (importOriginal) => ({ createRunUsageRecorder: usageRecorder.create, })); +vi.mock("./native-usage-delivery.js", () => ({ + createNativeUsageDelivery: vi.fn(async () => ({ capture: async () => undefined })), +})); + const githubAccess = vi.hoisted(() => ({ activate: vi.fn((_binding: { runId: string }) => vi.fn()), stop: vi.fn(async () => undefined), @@ -140,6 +144,7 @@ type BackendFactoryOptions = { }; type RunnerTransportOptions = { + onAccountingEvent?: (event: PrpEvent) => Promise; environment?: NodeJS.ProcessEnv; acpxRuntimeDirectory?: string; adoptExistingRunner?: { pid: number; isAlive: () => Promise | boolean }; @@ -249,8 +254,8 @@ vi.mock("./paperclip-control-plane-port.js", async importOriginal => { return { ...original, PaperclipControlPlanePort: class extends original.PaperclipControlPlanePort { constructor(...args: ConstructorParameters) { super(...args); - accountingEvents.committed = args[2]?.onCommittedEvent; - accountingEvents.duplicate = args[2]?.onDuplicateEvent; + accountingEvents.committed = args[2]?.onCommittedEvent ? event => args[2]!.onCommittedEvent!(event, 1) : undefined; + accountingEvents.duplicate = args[2]?.onDuplicateEvent ? event => args[2]!.onDuplicateEvent!(event, 1) : undefined; } } }; }); @@ -7154,6 +7159,36 @@ describe("native terminal-turn accounting", () => { expect(onUsage).toHaveBeenLastCalledWith(expect.objectContaining({ complete: true })); expect(usageRecorder.create).not.toHaveBeenCalled(); }); + it("binds retained warm transport usage to the successor run authority", async () => { + const directory = await mkdtemp(join(tmpdir(), "native-warm-usage-")); + const previous = process.env.PAPERCLIP_RUNNER_STATE_DIR; + process.env.PAPERCLIP_RUNNER_STATE_DIR = directory; + const first = { ...execution, binding: { ...execution.binding, runId: "first-warm-usage", executionWorkspaceId: "warm-usage-workspace" }, + session: { ...execution.session, normalizedSessionId: "warm-usage-session", lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60000 } } } as NativeExecutionInputV1; + const second = { ...first, binding: { ...first.binding, runId: "second-warm-usage" } }; + const firstCapture = vi.fn(async (_event: PrpEvent) => {}), secondCapture = vi.fn(async (_event: PrpEvent) => {}); + try { + await createRunnerdBackend({ db: leaseDb(first), execution: first, runnerInstanceId: "warm-producer", onAccountingEvent: firstCapture }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const retainedHook = state.createTransport.mock.calls.at(-1)![0].onAccountingEvent!; + const firstReport = { eventType: "usage.reported", runId: first.binding.runId, sourceInstanceId: "warm-producer" } as PrpEvent; + await retainedHook(firstReport); + expect(firstCapture).toHaveBeenCalledExactlyOnceWith(firstReport); + await rm(state.createTransport.mock.calls.at(-1)![0].stateDirectory!, { recursive: true, force: true }); + await createRunnerdBackend({ db: leaseDb(second), execution: second, runnerInstanceId: "warm-producer", onAccountingEvent: secondCapture }); + await expect(retainedHook({ ...firstReport, eventType: "run.attached" })).resolves.toBeUndefined(); + expect(secondCapture).not.toHaveBeenCalled(); + const secondReport = { ...firstReport, runId: second.binding.runId }; + await retainedHook(secondReport); + expect(secondCapture).toHaveBeenCalledExactlyOnceWith(secondReport); + expect(firstCapture).toHaveBeenCalledTimes(1); + await expect(retainedHook(firstReport)).rejects.toThrow("native_usage_run_binding_mismatch"); + } finally { + if (previous === undefined) delete process.env.PAPERCLIP_RUNNER_STATE_DIR; else process.env.PAPERCLIP_RUNNER_STATE_DIR = previous; + await rm(directory, { recursive: true, force: true }); + } + }); + it("uses the canonical recorder for partial and completed usage when no onUsage callback is supplied", async () => { const updates: Array<{ table: unknown; values: Record }> = []; state.execute.mockReset().mockImplementationOnce(async () => { @@ -12341,7 +12376,14 @@ describe("runnerd provider runtime wiring", () => { } }); - it.each(["github", "assigned"] as const)("cleans run-owned access after overlapping executor failure: %s", async failure => { + it.each((["codex", "opencode"] as const).flatMap(provider => + (["github", "assigned"] as const).map(failure => ({ provider, failure })), + ))("cleans run-owned access after overlapping $provider executor failure: $failure", async ({ provider, failure }) => { + const current = { + ...execution, + provider: provider === "codex" ? execution.provider : { kind: "opencode", model: null }, + session: { ...execution.session, driverKind: provider === "codex" ? "codex_app_server" : "opencode_server" }, + } as NativeExecutionInputV1; const issueService = vi.spyOn(issueServiceModule, "issueService").mockReturnValue({ update: vi.fn(async () => ({ status: "blocked", statusVersion: 1 })), } as unknown as ReturnType); @@ -12367,7 +12409,7 @@ describe("runnerd provider runtime wiring", () => { state.createBackend.mockClear(); let settled = false; const running = executePaperclipNativeSession({ - db: leaseDb(execution), execution, runnerInstanceId: "runner-overlap-cleanup", + db: leaseDb(current), execution: current, runnerInstanceId: "runner-overlap-cleanup", useRunnerd: true, managedGitHub: true, runnerEnvironment: { PAPERCLIP_NATIVE_MCP_NAME: "paperclip-assigned", @@ -12393,7 +12435,7 @@ describe("runnerd provider runtime wiring", () => { expect(state.execute).not.toHaveBeenCalled(); expect(state.createBackend).not.toHaveBeenCalled(); if (failure === "assigned") { - expect(activate).toHaveBeenCalledWith(execution.binding); + expect(activate).toHaveBeenCalledWith(current.binding); expect(deactivate).toHaveBeenCalledOnce(); expect(stop).toHaveBeenCalledOnce(); } diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index bddb90efb6d..66b77f875a8 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -1,4 +1,8 @@ +import { createNativeUsageDelivery } from "./native-usage-delivery.js"; +import { createNativeTurnAccounting, nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage, usesNativeUsageDelivery, isNativeUsageAccountingEvent } from "./native-usage-accounting.js"; +export { nativeUsageCostUsd, nativeUsageBiller, normalizeNativeUsage } from "./native-usage-accounting.js"; import { measureRunnerOperation, recordRunnerTimingReceipt } from "./runner-operation-timing.js"; +import { formatNativeRunLogEvent } from "./native-run-log-export.js"; import { notifyExternalAgentSettlement } from "../../modules/external-agents/index.js"; import { notifyNativeResultCommit } from "../native-result-commit-notifications.js"; import { releaseRetainedNativeSessionCleanup } from "../retained-native-session-cleanup-notifications.js"; @@ -107,7 +111,7 @@ import type { AdapterExecutionResult, AdapterRuntimeEvent, } from "../../adapters/index.js"; -import { usdToUnits, type NativeFinalizationResult } from "@paperclipai/shared"; +import type { NativeFinalizationResult } from "@paperclipai/shared"; import type { DurablePrpControlPlane, HarnessRuntimeRequestResolution, @@ -1752,12 +1756,13 @@ export async function synchronizeCompletedProviderPlan(input: { }; } -class SessionToolAuthorityEpoch { +class SessionAuthorityEpoch { readonly runId: string; #authority: PaperclipRunnerToolAuthority; #revoked = false; - constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace) { + constructor(runId: string, authority: PaperclipRunnerToolAuthority, private readonly toolTrace?: NativeToolTrace, + private readonly onAccountingEvent?: (event: PrpEvent) => Promise) { this.runId = runId; this.#authority = authority; } @@ -1773,6 +1778,14 @@ class SessionToolAuthorityEpoch { } } + async acceptAccountingEvent(event: PrpEvent) { + if (!isNativeUsageAccountingEvent(event)) return; + this.#assertCurrent(); + if (event.runId !== this.runId) throw new Error("native_usage_run_binding_mismatch"); + await this.onAccountingEvent?.(event); + this.#assertCurrent(); + } + definitions() { this.#assertCurrent(); return this.#authority.definitions(); @@ -1793,7 +1806,7 @@ class SessionToolAuthorityEpoch { } } -const sessionToolAuthorityEpochs = new Map(); +const sessionToolAuthorityEpochs = new Map(); const initializingSessionToolAuthorities = new Set(); const executingRunnerdSessionScopes = new Map(); const executingNativeOwnerScopes = new Map(); @@ -7820,6 +7833,7 @@ export async function executePaperclipNativeSession(input: { /** Internal, run-owned file inspection lifetime; never supplied by tool arguments. */ chatAttachmentReadScope?: NativeChatAttachmentReadScope; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + onPersistedNativeLog?: (cursor: number, chunk: string) => Promise; onEvent?: (event: AdapterRuntimeEvent) => Promise; /** Only this run's registered private instruction entry. * Probe at terminal; persist only after owned shutdown. */ @@ -8559,7 +8573,15 @@ async function executePaperclipNativeSessionWithinScope( for (const row of history) turnAccounting.observe(record(row.payload).prpEvent as PrpEvent); } let fallbackUsageRecorder: ReturnType | undefined; + const producerOwnsUsage = usesNativeUsageDelivery({ runnerd: input.useRunnerd === true, + remote: input.runnerExecutionTarget?.kind === "remote", provider: input.execution.provider }); + const producerUsage = producerOwnsUsage ? await createNativeUsageDelivery(input.db, { + companyId: input.execution.binding.companyId, runId: input.execution.binding.runId, + runnerInstanceId: effectiveRunnerInstanceId, provider: input.execution.provider, + billing: resolveNativeBilling(input.execution.provider, input.runnerEnvironment, input.billingIdentity), + }) : undefined; const persistAccountingUsage = async (usage: Record | null, complete: boolean) => { + if (producerOwnsUsage) return; const accountingUsage = normalizeNativeUsage(usage, { inputIncludesCacheReads: input.execution.provider.kind === "codex" }); const accountingCost = accountingUsage ? nativeUsageCostUsd(usage, input.execution.provider) : undefined; complete = complete && accountingUsage !== undefined; @@ -8653,7 +8675,7 @@ async function executePaperclipNativeSessionWithinScope( attributes:{eventKind:event.eventType}}); } }, - onCommittedEvent: async (event) => { + onCommittedEvent: async (event, cursor) => { if (event.eventType === "run.result.proposed") { const at = Date.now(); await trace.record({name:"completion.tool_accepted",parentName:"native.session.execute",startedAtMs:at,endedAtMs:at}); @@ -8806,7 +8828,8 @@ async function executePaperclipNativeSessionWithinScope( }); trace.activate(taskSettleScope); } - if (input.onLog) + if (input.onPersistedNativeLog) await input.onPersistedNativeLog(cursor, formatNativeRunLogEvent(event)); + else if (input.onLog) await input.onLog( "stdout", `${JSON.stringify({ type: "paperclip.prp.event", event })}\n`, @@ -8885,12 +8908,10 @@ async function executePaperclipNativeSessionWithinScope( }, ); }, - onDuplicateEvent: async (event) => { + onDuplicateEvent: async (event, cursor) => { + await input.onPersistedNativeLog?.(cursor, formatNativeRunLogEvent(event)); if (!firstTokenRecorded) observeDeliverableText(event); await observeAccountingEvent(event); - // A crash can happen after the event commit but before its callback - // finishes. Recover only idempotent durable projections here; activity, - // publication, logging, trace, and metric effects remain committed-only. await liveQuestions.observe(event); await projectSessionGoalEvent(event); providerUsageLimitObserved ||= nativeProviderUsageLimitFromEvent(event); @@ -9165,7 +9186,8 @@ async function executePaperclipNativeSessionWithinScope( return input.runnerEnvironment; }; const overlapGitHubPreparation = needsCoreGitHubAccess(input) && input.useRunnerd && input.backend === undefined - && input.runnerExecutionTarget?.kind === "remote" && input.execution.provider.kind === "codex" + && input.runnerExecutionTarget?.kind === "remote" + && (input.execution.provider.kind === "codex" || input.execution.provider.kind === "opencode") && input.execution.schema !== "paperclip.native-execution-input.v6" && input.execution.schema !== "paperclip.native-execution-input.v8"; if (!overlapGitHubPreparation) await prepareRunnerEnvironment(); @@ -9178,6 +9200,7 @@ async function executePaperclipNativeSessionWithinScope( input.useRunnerd && input.backend === undefined ? await timeController("backend_setup", () => createRunnerdBackend({ ...input, + onAccountingEvent: producerUsage?.capture, controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? { bootId: controller.bootId, runnerArtifact: restoredController ? { version: restoredController.runnerVersion, digest: restoredController.runnerDigest } : undefined, @@ -9217,7 +9240,7 @@ async function executePaperclipNativeSessionWithinScope( persistWarmNativeCheckpoint(remoteWarmRecovery.previousExecution, nativeSessionConfigDigest(remoteWarmRecovery.previousExecution, "remote"), await recovered.session.snapshot()); persistedWarmSession = loadWarmNativeCheckpoint(input.execution, warmConfigDigest!, "remote"); - runnerdBackend = await createRunnerdBackend({ ...input, runnerInstanceId: effectiveRunnerInstanceId, + runnerdBackend = await createRunnerdBackend({ ...input, onAccountingEvent: producerUsage?.capture, runnerInstanceId: effectiveRunnerInstanceId, controllerPersistence: input.runnerExecutionTarget?.kind === "remote" ? { bootId: controller.bootId } : undefined, durableEnvironmentLeaseId: durableRunnerBinding?.environmentLeaseId, trace, toolTrace }); } @@ -10261,151 +10284,6 @@ async function executePaperclipNativeSessionWithinScope( return adapterResult; } -function numericUsageField( - usage: Record | null, - keys: string[], -): number | undefined { - if (!usage) return undefined; - for (const key of keys) { - const value = usage[key]; - if (typeof value === "number" && Number.isFinite(value) && value >= 0) - return value; - } - return undefined; -} - -function nativeUsageMeasurement(usage: Record) { - const nestedUsage = record(usage.usage); - // An explicit partial delta must not fall through to a full session total. - if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta); - if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta); - const candidates = [ - record(usage.runDelta), - record(nestedUsage.runDelta), - record(usage.total), - record(nestedUsage.total), - record(usage.cumulative), - record(nestedUsage.cumulative), - nestedUsage, - usage, - ]; - return ( - candidates.find( - (candidate) => - numericUsageField(candidate, [ - "inputTokens", - "input", - "promptTokens", - "outputTokens", - "output", - "completionTokens", - ]) !== undefined, - ) ?? usage - ); -} - -export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string { - if (provider.kind === "acpx") { - if (provider.agent === "cursor") return "cursor"; - if (provider.agent === "copilot") return "github"; - if (provider.agent === "pi") return "openrouter"; - } - return "openai"; -} - -function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) { - const turns = new Map | null; terminal: boolean }>(); - const sequences = new Map(); - let currentTurn: string | undefined; - const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]; - const snapshot = () => { - if (turns.size === 0) return { usage: null, turnId: undefined, complete: false }; - let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0; - let cost = 0n, costKnown = false, costIncomplete = false, complete = true; - for (const [turnId, turn] of turns) { - const normalized = normalizeNativeUsage(turn.usage); - complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined; - inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0); - outputTokens += normalized?.outputTokens ?? 0; - cacheReadTokens += normalized?.cachedInputTokens ?? 0; - cacheWriteTokens += normalized?.cacheWriteTokens ?? 0; - const price = nativeUsageCostUsd(turn.usage, provider); - if (price === undefined) costIncomplete = true; - else { cost += usdToUnits(price); costKnown = true; } - } - // A free empty attempt contributes no price evidence for later paid work. - // Keep an incomplete zero subtotal absent so complete aggregate tokens can - // still receive an estimate. Preserve positive known spend and true zero totals. - const retainCost = costKnown && (!costIncomplete || cost > 0n); - const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, - ...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) }; - const current = currentTurn ? turns.get(currentTurn) : undefined; - return { usage: { runDelta, accountingCostIncomplete: costIncomplete, - ...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) }, - turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete }; - }; - return { - observe(event: PrpEvent) { - if (!event) return false; - const payload = record(event.payload); - const terminal = terminalTypes.includes(event.eventType); - if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false; - // Historical replay can revisit an earlier report after restoration. - // Re-persist its aggregate, but never rewind or count that report twice. - if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) { - if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true; - sequences.set(event.sourceInstanceId, event.sourceSeq); - } - const key = event.turnId ?? "unattributed"; - const turn = turns.get(key) ?? { usage: null, terminal: false }; - if (event.eventType === "turn.started") { - currentTurn = event.turnId; - turn.usage = null; - turn.terminal = false; - } else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId; - if (payload.kind === "usage") { - const usage = record(payload.usage); - turn.usage = Object.hasOwn(usage, "runDelta") - ? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null; - } - if (terminal) turn.terminal = true; - turns.set(key, turn); - return true; - }, - snapshot, - finish(turnId: string | null) { - const turn = turnId ? turns.get(turnId) : undefined; - if (turn) turn.terminal = true; - return snapshot(); - }, - }; -} - -export function nativeUsageCostUsd( - usage: Record | null, - provider?: NativeExecutionInput["provider"], -) { - // The ACP normalization contract fills absent per-turn cost with zero and - // reports actual cost cumulatively. Until it carries an authoritative run - // delta with provenance, neither value is a candidate's billed USD receipt. - if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined; - if (!usage) return undefined; - const measurement = nativeUsageMeasurement(usage); - const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta"); - const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]) - ?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])); - // PRP currently defaults an absent providerCostUsd to zero. That zero is - // not proof of a free request when the runtime observed token consumption. - if (direct !== undefined && direct > 0) return direct; - const normalized = normalizeNativeUsage(usage); - if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0; - const cost = record(hasRunDelta ? measurement.cost : usage.cost); - const currency = - typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; - if (currency !== "USD") return undefined; - return numericUsageField(cost, ["amount", "total"]); -} - /** Billing identity follows the selected runtime and resolved credentials. * Unknown authentication remains unknown; a provider name does not prove billing mode. */ export function resolveNativeBilling(provider: NativeExecutionInput["provider"], env: NodeJS.ProcessEnv = {}, identity?: Pick): Pick { @@ -10438,33 +10316,6 @@ export function resolveNativeBilling(provider: NativeExecutionInput["provider"], return { provider: "openai", biller: billingType === "metered_api" ? inferOpenAiCompatibleBiller(env, "openai") : "unknown", billingType }; } -export function normalizeNativeUsage(usage: Record | null, options: { inputIncludesCacheReads?: boolean } = {}) { - if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined; - const measurement = nativeUsageMeasurement(usage); - const cache = record(measurement.cache); - const count = (source: Record, keys: string[]) => { - for (const key of keys) { - if (source[key] === undefined) continue; - const value = source[key]; - return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN; - } - return undefined; - }; - const input = count(measurement, ["inputTokens", "input", "promptTokens"]); - const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]); - const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]); - const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]); - if (input === undefined || outputTokens === undefined - || ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined; - const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0) - + (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0); - if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined; - return { - inputTokens, outputTokens, - ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }), - ...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }), - }; -} function processEnvironment( environment: NodeJS.ProcessEnv, @@ -11850,6 +11701,7 @@ export async function createRunnerdBackend(input: { casesEnabled?: boolean; db: Db; controllerPersistence?: { bootId: string; runnerArtifact?: { version: string; digest: string } }; + onAccountingEvent?: (event: PrpEvent) => Promise; execution: NativeExecutionInput; runnerInstanceId: string; chatAttachmentReadScope?: NativeChatAttachmentReadScope; @@ -12165,13 +12017,14 @@ async function createRunnerdBackendWithinSessionClaim( syncIssueExternalObjects: input.syncIssueExternalObjects, enqueueWakeup: input.enqueueWakeup, }); - const authorityEpoch = new SessionToolAuthorityEpoch( + const authorityEpoch = new SessionAuthorityEpoch( input.execution.binding.runId, authority, input.toolTrace, + input.onAccountingEvent, ); let dynamicTools: Awaited< - ReturnType + ReturnType >; try { dynamicTools = await timeController("tool_authority", () => authorityEpoch.definitions()); @@ -12273,7 +12126,7 @@ async function createRunnerdBackendWithinSessionClaim( authority.binding.readRemoteTaskFile = file => owner.readWorkspaceFile(file.contentRef, file.byteSize); dynamicTools = await authorityEpoch.definitions(); } - const executeTool = (call: Parameters[0]) => { + const executeTool = (call: Parameters[0]) => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_tool_authority_unavailable"); return current.execute(call); @@ -12368,7 +12221,7 @@ async function createRunnerdBackendWithinSessionClaim( }; const backendOptions = { onSpawn: input.onSpawn, dynamicTools, - dynamicToolHandler: async (call: Parameters[0]) => { + dynamicToolHandler: async (call: Parameters[0]) => { await assertExternalNativeRunAuthority(input.db, externalExecution); return authorityEpoch.execute(call); }, completionFeedback: async (result: PrpStructuredRunResult) => { @@ -13709,7 +13562,7 @@ async function createRunnerdBackendWithinSessionClaim( markSpawned?.(); } const executeCurrentToolAuthority = ( - call: Parameters[0], + call: Parameters[0], ) => { const current = sessionToolAuthorityEpochs.get(sessionScopeId); if (!current) throw new Error("native_session_tool_authority_unavailable"); @@ -13757,6 +13610,11 @@ async function createRunnerdBackendWithinSessionClaim( onSpawn: input.onSpawn, ...(input.execution.provider.kind === "codex" ? { disableCodexWebSearch: "webSearch" in input.execution.provider && input.execution.provider.webSearch === "disabled" } : {}), homeMode: preconfiguredHome ? "preconfigured" : "isolated", + onAccountingEvent: input.onAccountingEvent ? async event => { + const current = sessionToolAuthorityEpochs.get(sessionScopeId); + if (!current) throw new Error("native_session_authority_unavailable"); + await current.acceptAccountingEvent(event); + } : undefined, onCredentialsUpdated: (event) => appendNativeCredentialUpdate(input.db, { companyId: input.execution.binding.companyId, agentId: input.execution.binding.agentId, diff --git a/server/src/services/native-runtime/native-usage-accounting.test.ts b/server/src/services/native-runtime/native-usage-accounting.test.ts new file mode 100644 index 00000000000..7e40c08fd3f --- /dev/null +++ b/server/src/services/native-runtime/native-usage-accounting.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from "vitest"; +import type { PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage, usesNativeUsageDelivery } from "./native-usage-accounting.js"; + +const provider = { kind: "opencode" as const, model: "test-model" }; +const event = (sourceSeq: number, eventType: string, turnId = "turn-1", payload: Record = {}) => ({ + sourceInstanceId: "producer", sourceSeq, eventType, turnId, payload, +} as PrpEvent); + +describe("native turn accounting", () => { + it("selects producer delivery only for supported remote Runner execution", () => { + expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider })).toBe(true); + expect(usesNativeUsageDelivery({ runnerd: false, remote: true, provider })).toBe(false); + expect(usesNativeUsageDelivery({ runnerd: true, remote: false, provider })).toBe(false); + expect(usesNativeUsageDelivery({ runnerd: true, remote: true })).toBe(false); + expect(usesNativeUsageDelivery({ runnerd: true, remote: true, provider: { kind: "muse", model: null } as never })).toBe(false); + }); + + it("reconstructs cumulative turn usage without counting repeated producer events twice", () => { + const accounting = createNativeTurnAccounting(provider); + const usage = { kind: "usage", usage: { runDeltaComplete: true, runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: 0.25 } } }; + const history = [event(1, "turn.started"), event(2, "item.completed", "turn-1", usage), event(3, "turn.completed"), + event(4, "turn.started", "turn-2"), event(5, "item.completed", "turn-2", usage), event(6, "turn.completed", "turn-2")]; + history.forEach(value => accounting.observe(value)); + history.forEach(value => accounting.observe(value)); + const snapshot = accounting.snapshot(); + expect(snapshot.complete).toBe(true); + expect(normalizeNativeUsage(snapshot.usage)).toEqual({ inputTokens: 240, outputTokens: 20, cachedInputTokens: 0, cacheWriteTokens: 0 }); + expect(nativeUsageCostUsd(snapshot.usage, provider)).toBe(0.5); + }); + + it("keeps an incomplete provider report pending instead of trusting placeholder zeroes", () => { + const accounting = createNativeTurnAccounting(provider); + accounting.observe(event(1, "turn.started")); + accounting.observe(event(2, "item.completed", "turn-1", { kind: "usage", usage: { + runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0, providerCostUsd: 0 }, + } })); + accounting.observe(event(3, "turn.completed")); + expect(accounting.snapshot().complete).toBe(false); + expect(accounting.snapshot().usage?.accountingCostIncomplete).toBe(true); + expect(normalizeNativeUsage({ runDeltaComplete: false, runDelta: { inputTokens: 0, outputTokens: 0 } })).toBeUndefined(); + }); +}); diff --git a/server/src/services/native-runtime/native-usage-accounting.ts b/server/src/services/native-runtime/native-usage-accounting.ts new file mode 100644 index 00000000000..9bbeb41ba93 --- /dev/null +++ b/server/src/services/native-runtime/native-usage-accounting.ts @@ -0,0 +1,187 @@ +import { usdToUnits } from "@paperclipai/shared"; +import type { NativeExecutionInput, PrpEvent } from "../../vendor/paperclip-runner/index.js"; +const record = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; + +function numericUsageField( + usage: Record | null, + keys: string[], +): number | undefined { + if (!usage) return undefined; + for (const key of keys) { + const value = usage[key]; + if (typeof value === "number" && Number.isFinite(value) && value >= 0) + return value; + } + return undefined; +} + +function nativeUsageMeasurement(usage: Record) { + const nestedUsage = record(usage.usage); + // An explicit partial delta must not fall through to a full session total. + if (Object.hasOwn(usage, "runDelta")) return record(usage.runDelta); + if (Object.hasOwn(nestedUsage, "runDelta")) return record(nestedUsage.runDelta); + const candidates = [ + record(usage.runDelta), + record(nestedUsage.runDelta), + record(usage.total), + record(nestedUsage.total), + record(usage.cumulative), + record(nestedUsage.cumulative), + nestedUsage, + usage, + ]; + return ( + candidates.find( + (candidate) => + numericUsageField(candidate, [ + "inputTokens", + "input", + "promptTokens", + "outputTokens", + "output", + "completionTokens", + ]) !== undefined, + ) ?? usage + ); +} + +export function nativeUsageBiller(provider: NativeExecutionInput["provider"]): string { + if (provider.kind === "acpx") { + if (provider.agent === "cursor") return "cursor"; + if (provider.agent === "copilot") return "github"; + if (provider.agent === "pi") return "openrouter"; + } + return "openai"; +} + +export function createNativeTurnAccounting(provider: NativeExecutionInput["provider"]) { + const turns = new Map | null; terminal: boolean }>(); + const sequences = new Map(); + let currentTurn: string | undefined; + const terminalTypes = ["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]; + const snapshot = () => { + if (turns.size === 0) return { usage: null, turnId: undefined, complete: false }; + let inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0; + let cost = 0n, costKnown = false, costIncomplete = false, complete = true; + for (const [turnId, turn] of turns) { + const normalized = normalizeNativeUsage(turn.usage); + complete &&= turnId !== "unattributed" && turn.terminal && normalized !== undefined; + inputTokens += (normalized?.inputTokens ?? 0) - (normalized?.cacheWriteTokens ?? 0); + outputTokens += normalized?.outputTokens ?? 0; + cacheReadTokens += normalized?.cachedInputTokens ?? 0; + cacheWriteTokens += normalized?.cacheWriteTokens ?? 0; + const price = nativeUsageCostUsd(turn.usage, provider); + if (price === undefined) costIncomplete = true; + else { cost += usdToUnits(price); costKnown = true; } + } + // A free empty attempt contributes no price evidence for later paid work. + // Keep an incomplete zero subtotal absent so complete aggregate tokens can + // still receive an estimate. Preserve positive known spend and true zero totals. + const retainCost = costKnown && (!costIncomplete || cost > 0n); + const runDelta = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens, + ...(retainCost ? { providerCostUsd: Number(cost) / 1_000_000_000 } : {}) }; + const current = currentTurn ? turns.get(currentTurn) : undefined; + return { usage: { runDelta, accountingCostIncomplete: costIncomplete, + ...(retainCost ? { accountingCostUsdExact: `${cost / 1_000_000_000n}.${String(cost % 1_000_000_000n).padStart(9, "0")}` } : {}) }, + turnId: current && normalizeNativeUsage(current.usage) ? currentTurn : undefined, complete }; + }; + return { + observe(event: PrpEvent) { + if (!event) return false; + const payload = record(event.payload); + const terminal = terminalTypes.includes(event.eventType); + if (payload.kind !== "usage" && event.eventType !== "turn.started" && !terminal) return false; + // Historical replay can revisit an earlier report after restoration. + // Re-persist its aggregate, but never rewind or count that report twice. + if (event.sourceInstanceId && Number.isSafeInteger(event.sourceSeq)) { + if (event.sourceSeq <= (sequences.get(event.sourceInstanceId) ?? -1)) return true; + sequences.set(event.sourceInstanceId, event.sourceSeq); + } + const key = event.turnId ?? "unattributed"; + const turn = turns.get(key) ?? { usage: null, terminal: false }; + if (event.eventType === "turn.started") { + currentTurn = event.turnId; + turn.usage = null; + turn.terminal = false; + } else if (payload.kind === "usage" && (!currentTurn || !turns.has(key))) currentTurn = event.turnId; + if (payload.kind === "usage") { + const usage = record(payload.usage); + turn.usage = Object.hasOwn(usage, "runDelta") + ? { runDelta: record(usage.runDelta), runDeltaComplete: usage.runDeltaComplete } : null; + } + if (terminal) turn.terminal = true; + turns.set(key, turn); + return true; + }, + snapshot, + finish(turnId: string | null) { + const turn = turnId ? turns.get(turnId) : undefined; + if (turn) turn.terminal = true; + return snapshot(); + }, + }; +} + +export function nativeUsageCostUsd( + usage: Record | null, + provider?: NativeExecutionInput["provider"], +) { + // The ACP normalization contract fills absent per-turn cost with zero and + // reports actual cost cumulatively. Until it carries an authoritative run + // delta with provenance, neither value is a candidate's billed USD receipt. + if (provider?.kind === "acpx" && ["cursor", "copilot", "pi"].includes(provider.agent)) return undefined; + if (!usage) return undefined; + const measurement = nativeUsageMeasurement(usage); + const hasRunDelta = Object.hasOwn(usage, "runDelta") || Object.hasOwn(record(usage.usage), "runDelta"); + const direct = numericUsageField(measurement, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"]) + ?? (hasRunDelta ? undefined : numericUsageField(usage, ["cacheAdjustedCostUsd", "providerCostUsd", "costUsd"])); + // PRP currently defaults an absent providerCostUsd to zero. That zero is + // not proof of a free request when the runtime observed token consumption. + if (direct !== undefined && direct > 0) return direct; + const normalized = normalizeNativeUsage(usage); + if (direct === 0 && normalized && Object.values(normalized).every(value => value === 0)) return 0; + const cost = record(hasRunDelta ? measurement.cost : usage.cost); + const currency = + typeof cost.currency === "string" ? cost.currency.toUpperCase() : "USD"; + if (currency !== "USD") return undefined; + return numericUsageField(cost, ["amount", "total"]); +} + +export function normalizeNativeUsage(usage: Record | null, options: { inputIncludesCacheReads?: boolean } = {}) { + if (!usage || usage.runDeltaComplete === false || record(usage.usage).runDeltaComplete === false) return undefined; + const measurement = nativeUsageMeasurement(usage); + const cache = record(measurement.cache); + const count = (source: Record, keys: string[]) => { + for (const key of keys) { + if (source[key] === undefined) continue; + const value = source[key]; + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : NaN; + } + return undefined; + }; + const input = count(measurement, ["inputTokens", "input", "promptTokens"]); + const outputTokens = count(measurement, ["outputTokens", "output", "completionTokens"]); + const cachedInputTokens = count(measurement, ["cachedInputTokens", "cacheReadInputTokens", "cacheReadTokens", "cachedReadTokens"]) ?? count(cache, ["read"]); + const cacheWriteTokens = count(measurement, ["cacheWriteTokens", "cacheCreationInputTokens", "cachedWriteTokens"]) ?? count(cache, ["write"]); + if (input === undefined || outputTokens === undefined + || ![input, outputTokens, cachedInputTokens ?? 0, cacheWriteTokens ?? 0].every(Number.isSafeInteger)) return undefined; + const inputTokens = input - (options.inputIncludesCacheReads ? cachedInputTokens ?? 0 : 0) + + (options.inputIncludesCacheReads ? 0 : cacheWriteTokens ?? 0); + if (!Number.isSafeInteger(inputTokens) || inputTokens < (cacheWriteTokens ?? 0)) return undefined; + return { + inputTokens, outputTokens, + ...(cachedInputTokens === undefined ? {} : { cachedInputTokens }), + ...(cacheWriteTokens === undefined ? {} : { cacheWriteTokens }), + }; +} + +export function usesNativeUsageDelivery(input: { + runnerd: boolean; remote: boolean; provider?: NativeExecutionInput["provider"] | null; +}) { + return input.runnerd && input.remote && input.provider !== undefined && input.provider !== null + && ["codex", "opencode", "acpx", "claude_managed", "aws_agentcore"].includes(input.provider.kind); +} + +export function isNativeUsageAccountingEvent(event: Pick) { + return ["usage.reported", "turn.started", "turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(event.eventType); +} diff --git a/server/src/services/native-runtime/native-usage-delivery.integration.test.ts b/server/src/services/native-runtime/native-usage-delivery.integration.test.ts new file mode 100644 index 00000000000..21eab93cb2c --- /dev/null +++ b/server/src/services/native-runtime/native-usage-delivery.integration.test.ts @@ -0,0 +1,283 @@ +import { compareCents, usdToUnits } from "@paperclipai/shared"; +import { spawn, execFileSync, type ChildProcess } from "node:child_process"; +import { once } from "node:events"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db"; +import { type PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; +import { accountRunCost } from "../run-cost-accounting.js"; +import { usageReceiptSpoolPath } from "../usage-receipts.js"; +import { createNativeUsageDelivery } from "./native-usage-delivery.js"; + +const runnerRoot = fileURLToPath(new URL("../../../../packages/paperclip-runner/runner/", import.meta.url)); +const fixtureProfile = process.env.CI ? "release" : "debug"; +const billing = { provider: "openai", biller: "openai", billingType: "metered_api" as const }; + +describe("producer-owned native usage", () => { + let database: Awaited>; + let db: Db; + beforeAll(async () => { + execFileSync("cargo", ["build", ...(fixtureProfile === "release" ? ["--release"] : []), "--locked", + "--manifest-path", resolve(runnerRoot, "Cargo.toml"), "-p", "paperclip-runner-core", "--example", "recovery-fixture"], + { timeout: 600_000, stdio: "pipe" }); + database = await startEmbeddedPostgresTestDatabase("native-usage-delivery-"); + db = createDb(database.connectionString); + }, 660_000); + afterAll(async () => { await database?.cleanup(); }); + const seed = async () => { + const companyId = randomUUID(), agentId = randomUUID(), runId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Usage delivery", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(agents).values({ id: agentId, companyId, name: "Producer", adapterType: "paperclip_runner" }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", status: "running", runtimeMode: "native", costAccountingPending: true }); + return { companyId, agentId, runId, runnerInstanceId: randomUUID(), provider: { kind: "codex" as const, model: "test-model" }, billing }; + }; + const event = (input: Pick>, "runId" | "runnerInstanceId">, sequence: number, type: string, turnId = "turn-1", amount = 0.25): PrpEvent => ({ + schema: "paperclip.prp.event.v1", schemaVersion: 1, sourceKind: "runner", sourceInstanceId: input.runnerInstanceId, + sourceEventId: `usage-${sequence}`, sourceSeq: sequence, priority: 0, runId: input.runId, + normalizedSessionId: "session-1", turnId, emittedAt: new Date().toISOString(), eventType: type, + payload: type === "usage.reported" ? { runDeltaAvailable: true, runDeltaBasis: "run", runDelta: { inputTokens: 120, outputTokens: 10, providerCostUsd: amount } } : {}, + } as PrpEvent); + + it("retains the same Runner report through a failed Postgres receipt and an empty replacement receiver", async () => { + const input = await seed(); + expect(existsSync(usageReceiptSpoolPath())).toBe(false); + const root = mkdtempSync(resolve(tmpdir(), "native-usage-runner-")); + const receiverA = resolve(root, "receiver-a"), receiverB = resolve(root, "receiver-b"); + const identity = { runnerInstanceId: input.runnerInstanceId, environmentLeaseId: randomUUID(), runId: input.runId, + normalizedSessionId: randomUUID(), turnId: "turn-1", itemId: "item-1" }; + const expectedRunnerVersion = "0.3.0", expectedRunnerDigest = `sha256:${"c".repeat(64)}`; + let receiver: ChildProcess | undefined, child: ChildProcess | undefined; + let exit: Promise | undefined, failed = 0, lastCaptureError = ""; + const receiptFile = resolve(root, "producer.json"), releaseFile = resolve(root, "release"), scriptFile = resolve(root, "script.json"); + writeFileSync(scriptFile, JSON.stringify({ steps: 3, waitAfter: 2, retentionLimit: 8, receiptFile, releaseFile, accounting: true })); + const receipts = () => db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, input.runId)); + try { + + await db.execute(sql`CREATE FUNCTION fail_native_usage_receipt() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'receipt database unavailable'; END $$`); + await db.execute(sql`CREATE TRIGGER fail_native_usage BEFORE INSERT ON run_usage_receipts FOR EACH ROW EXECUTE FUNCTION fail_native_usage_receipt()`); + const startReceiver = () => new Promise((resolve, reject) => { + const process = spawn(globalThis.process.execPath, ["--import", fileURLToPath(new URL("../../../node_modules/tsx/dist/loader.mjs", import.meta.url)), fileURLToPath(new URL("./__fixtures__/usage-receiver.ts", import.meta.url))], { + cwd: fileURLToPath(new URL("../../../../", import.meta.url)), stdio: ["ignore", "ignore", "pipe", "ipc"], env: { ...globalThis.process.env }, + }); + let diagnostics = ""; + process.stderr!.on("data", bytes => { diagnostics = (diagnostics + String(bytes)).slice(-16000); }); + const timer = setTimeout(() => { process.kill("SIGKILL"); reject(new Error(`receiver startup timeout: ${diagnostics}`)); }, 60_000); + process.once("exit", code => { clearTimeout(timer); reject(new Error(`receiver exited ${code}: ${diagnostics}`)); }); + process.on("message", message => { + const kind = (message as { kind?: string }).kind; + if (kind === "capture-failed") { failed++; lastCaptureError = String((message as { cause?: string }).cause); } + if (kind === "ready") { clearTimeout(timer); resolve(process); } + }); + }); + let requestId = 0; + const request = (target: ChildProcess, value: Record) => new Promise>((resolve, reject) => { + const id = ++requestId; + const timer = setTimeout(() => { target.off("message", listener); reject(new Error("receiver response timeout")); }, 30_000); + const listener = (message: unknown) => { + const result = message as Record; + if (result.requestId !== id) return; + clearTimeout(timer); target.off("message", listener); + if (result.error) reject(new Error(String(result.error))); else resolve(result); + }; + target.on("message", listener); + target.send({ ...value, requestId: id }); + }); + receiver = await startReceiver(); + const firstReceiver = await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverA, identity, input }); + const connectUrl = String(firstReceiver.connectUrl), port = Number(new URL(connectUrl).port), ticket = String(firstReceiver.ticket); + child = spawn(resolve(runnerRoot, `target/${fixtureProfile}/examples/recovery-fixture`), ["--fixture-script", scriptFile, "--connect-url", connectUrl, + "--state-dir", resolve(root, "producer"), "--runner-id", identity.runnerInstanceId, "--environment-lease-id", identity.environmentLeaseId, + "--run-id", identity.runId, "--session-id", identity.normalizedSessionId, "--turn-id", identity.turnId, "--item-id", identity.itemId, + "--runner-version", expectedRunnerVersion, "--runner-digest", expectedRunnerDigest, "--max-outbox-bytes", "65536", "--p0-reserve-bytes", "16384", "--reconnect-grace-ms", "120000"], + { env: { ...process.env, PAPERCLIP_RUNNER_BOOTSTRAP_TICKET: ticket }, stdio: ["ignore", "ignore", "pipe"] }); + exit = once(child, "exit"); + await request(receiver, { kind: "command", type: "turn.start", commandId: "start-once" }); + writeFileSync(releaseFile, "produce"); + await vi.waitFor(() => expect(failed).toBeGreaterThan(0), { timeout: 10_000 }); + expect(await receipts()).toHaveLength(0); + const producerState = () => JSON.parse(readFileSync(resolve(root, "producer/runner-state.json"), "utf8")) as { + outbox: Array<{ eventType: string; envelope: { payload: { sourceEventId: string } } }>; + }; + const pendingUsage = producerState().outbox.find(report => report.eventType === "usage.reported"); + expect(pendingUsage).toBeDefined(); + const pendingSourceEventId = pendingUsage!.envelope.payload.sourceEventId; + const stoppedReceiver = await request(receiver, { kind: "stop" }); + rmSync(receiverA, { recursive: true, force: true }); + await db.execute(sql`DROP TRIGGER fail_native_usage ON run_usage_receipts`); + await db.execute(sql`DROP FUNCTION fail_native_usage_receipt()`); + + receiver = await startReceiver(); + await request(receiver, { kind: "start", connectionString: database.connectionString, directory: receiverB, identity, input, port, checkpoint: stoppedReceiver.checkpoint }); + await vi.waitFor(async () => expect((await receipts()).filter(row => row.complete)).toHaveLength(1), { timeout: 15_000 }); + await vi.waitFor(() => expect(producerState().outbox).toEqual([]), { timeout: 10_000 }); + expect((await receipts()).some(row => (row.receiptJson.producerEvent as Record).sourceEventId === pendingSourceEventId)).toBe(true); + expect((await receipts()).filter(row => row.receiptJson.producerEvent && (row.receiptJson.producerEvent as Record).eventType === "usage.reported")).toHaveLength(1); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect(await accountRunCost(db, input.runId)).toBe(true); + expect(await accountRunCost(db, input.runId)).toBe(false); + const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)); + expect(ledger).toHaveLength(1); + expect(ledger[0].costCents).toBe(25); + const failedBeforeLateReport = failed; + await request(receiver, { kind: "command", type: "turn.steer", commandId: "release-late-usage" }); + await vi.waitFor(() => expect(failed).toBeGreaterThan(failedBeforeLateReport), { timeout: 10_000 }); + expect(lastCaptureError).toContain("native_usage_after_finalization_requires_reconciliation"); + const retainedLate = producerState().outbox.find(report => report.eventType === "usage.reported"); + expect(retainedLate).toBeDefined(); + expect(retainedLate!.envelope.payload).toMatchObject({ runId: input.runId, sourceInstanceId: input.runnerInstanceId }); + expect((await receipts()).filter(row => (row.receiptJson.producerEvent as Record)?.eventType === "usage.reported")).toHaveLength(1); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(1); + expect(existsSync(usageReceiptSpoolPath())).toBe(false); + } finally { + await db.execute(sql`DROP TRIGGER IF EXISTS fail_native_usage ON run_usage_receipts`); + await db.execute(sql`DROP FUNCTION IF EXISTS fail_native_usage_receipt()`); + if (receiver && receiver.exitCode === null && receiver.signalCode === null) receiver.kill("SIGKILL"); + if (child && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); + await exit; + rmSync(root, { force: true, recursive: true }); + } + }, 240_000); + + it("preserves source takeover fencing, budget holds, and missing-usage debt", async () => { + const input = await seed(); + const first = await createNativeUsageDelivery(db, input); + await db.insert(budgetPolicies).values({ companyId: input.companyId, scopeType: "agent", scopeId: input.agentId, + metric: "billed_cents", windowKind: "lifetime", amount: 10, notifyEnabled: false }); + await first.capture(event(input, 1, "usage.reported")); + const [agent] = await db.select().from(agents).where(eq(agents.id, input.agentId)); + expect(agent.lifecycleHolds).toContain("budget"); + const replacement = { ...input, runnerInstanceId: randomUUID() }; + const second = await createNativeUsageDelivery(db, replacement); + await second.capture(event(replacement, 1, "usage.reported", "turn-2", 0.5)); + await first.capture(event(input, 2, "turn.completed")); + const [pending] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + expect(pending.usageJson).toMatchObject({ costUsd: 0.5, accountingReceiptReady: false }); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect(await accountRunCost(db, input.runId)).toBe(false); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0); + await second.capture(event(replacement, 2, "turn.completed", "turn-2")); + expect(await accountRunCost(db, input.runId)).toBe(true); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(50); + }); + + it.each(["tokens", "cost", "incomplete"])("accepts equal finalized snapshots and rejects changed %s without repricing", async change => { + const input = await seed(); + const delivery = await createNativeUsageDelivery(db, input); + await delivery.capture(event(input, 1, "usage.reported")); + await delivery.capture(event(input, 2, "turn.completed")); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect(await accountRunCost(db, input.runId)).toBe(true); + await delivery.capture(event(input, 3, "usage.reported")); + const late = event(input, 4, "usage.reported"); + (late.payload as Record).runDelta = { inputTokens: change === "tokens" ? 150 : 120, outputTokens: change === "tokens" ? 15 : 10, providerCostUsd: change === "incomplete" ? 0.25 : 0.5 }; + if (change === "incomplete") (late.payload as Record).runDeltaAvailable = false; + await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + await expect(delivery.capture(late)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)))[0].costCents).toBe(25); + }); + + it("accepts equal finalized rate-card reports without requiring terminal-only derived pricing", async () => { + const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } }; + const delivery = await createNativeUsageDelivery(db, input); + const report = (sequence: number) => { + const reported = event(input, sequence, "usage.reported"); + reported.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 1000, outputTokens: 100000 } }; + return reported; + }; + await delivery.capture(report(1)); + await delivery.capture(event(input, 2, "turn.completed")); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect(await accountRunCost(db, input.runId)).toBe(true); + await delivery.capture(report(3)); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + expect(run.usageJson).toMatchObject({ pricingProvenance: { source: "rate_card" }, accountingReceiptReady: true }); + expect(usdToUnits(run.usageJson!.costUsdExact as string)).toBe(usdToUnits("1.002")); + const ledger = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId)); + expect(ledger).toHaveLength(1); + expect(ledger[0].costCents).toBe(100.2); + const correction = report(4); + (correction.payload as Record).runDelta = { inputTokens: 1000, outputTokens: 100000, providerCostUsd: 2 }; + await expect(delivery.capture(correction)).rejects.toThrow("native_usage_after_finalization_requires_reconciliation"); + }); + + it("keeps successor exposure pending after prior-turn accounting invalidates its live snapshot", async () => { + const input = await seed(); + const delivery = await createNativeUsageDelivery(db, input); + const paid = event(input, 1, "usage.reported"); + paid.payload = { runDeltaBasis: "run", runDeltaAvailable: true, runDelta: { inputTokens: 50, outputTokens: 5, providerCostUsd: 0.5 } }; + await delivery.capture(paid); + const invalidated = event(input, 2, "usage.reported"); + invalidated.payload = { runDeltaBasis: "run", runDeltaAvailable: false, runDelta: {}, reason: "accounting_reconciliation_required", + reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120, outputTokens: 12 }, priorOrigin: { runId: "old-run" } } }; + await delivery.capture(invalidated); + await delivery.capture(event(input, 3, "turn.failed")); + await db.update(heartbeatRuns).set({ status: "failed", errorCode: "codex_usage_accounting_integrity", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + expect(run).toMatchObject({ costAccountingPending: true, costAccountedAt: null, + usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } }); + expect(compareCents((run.usageJson?.budgetExposure as { costCents: string }).costCents, "50")).toBe(0); + const [required] = await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 2))); + expect(required.receiptJson.producerEvent).toMatchObject({ usage: { reconciliation: { receivedTurnId: "old-provider-turn", cumulative: { inputTokens: 120 }, priorOrigin: { runId: "old-run" } } } }); + expect(await accountRunCost(db, input.runId)).toBe(false); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0); + }); + + it("retains unmarked legacy Codex reports as incomplete unpriced debt", async () => { + const input = { ...await seed(), provider: { kind: "codex" as const, model: "gpt-6.1-sol" } }; + const delivery = await createNativeUsageDelivery(db, input); + const legacy = event(input, 1, "usage.reported"); + delete (legacy.payload as Record).runDeltaBasis; + await delivery.capture(legacy); + await delivery.capture(event(input, 2, "turn.completed")); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, input.runId)); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)))[0]).toMatchObject({ + costAccountingPending: true, costAccountedAt: null, usageJson: { accountingReceiptReady: false, costStatus: "unpriced" } }); + expect(await accountRunCost(db, input.runId)).toBe(false); + expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, input.runId))).toHaveLength(0); + }); + + it("replays partial and final usage once after a lost commit response and restores turn aggregation", async () => { + const input = { ...await seed(), provider: { kind: "opencode" as const, model: "test-model" } }; + let loseResponse = false; + const failingDb = new Proxy(db, { get(target, property) { + if (property === "transaction") return async (...args: Parameters) => { + const result = await db.transaction(...args); + if (loseResponse) { loseResponse = false; throw new Error("lost commit response"); } + return result; + }; + const value = Reflect.get(target, property); + return typeof value === "function" ? value.bind(target) : value; + } }) as Db; + let delivery = await createNativeUsageDelivery(failingDb, input); + const first = event(input, 1, "usage.reported"), done = event(input, 2, "turn.completed"); + loseResponse = true; + await expect(delivery.capture(first)).rejects.toThrow("lost commit response"); + await delivery.capture(first); + await delivery.capture(done); + delivery = await createNativeUsageDelivery(db, input); + await delivery.capture(first); + await delivery.capture(done); + const nextTurn = event(input, 3, "turn.started", "turn-1"); + nextTurn.payload = { providerTurnId: "provider-turn-2" }; + await delivery.capture(nextTurn); + const partial = event(input, 4, "usage.reported", "turn-1", 0.5); + (partial.payload as Record).runDeltaAvailable = false; + await delivery.capture(partial); + const final = event(input, 5, "usage.reported", "turn-1", 0.5); + await delivery.capture(final); + const terminal = event(input, 6, "turn.completed", "turn-1"); + terminal.payload = { providerTurnId: "provider-turn-2" }; + await delivery.capture(terminal); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); + expect(run.usageJson).toMatchObject({ inputTokens: 240, outputTokens: 20, accountingReceiptReady: true, costUsd: 0.75 }); + expect(run.usageJson?.accountingCaptureFailed).not.toBe(true); + expect(await db.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.runId, input.runId), eq(runUsageReceipts.sequence, 1)))).toHaveLength(1); + await expect(delivery.capture({ ...final, payload: { ...final.payload as object, runDeltaAvailable: false } })).rejects.toThrow("native_usage_source_conflict"); + }); +}); diff --git a/server/src/services/native-runtime/native-usage-delivery.ts b/server/src/services/native-runtime/native-usage-delivery.ts new file mode 100644 index 00000000000..2bbf52d0d1b --- /dev/null +++ b/server/src/services/native-runtime/native-usage-delivery.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import { and, asc, eq, isNull, sql } from "drizzle-orm"; +import { heartbeatRuns, runUsageReceipts, type Db } from "@paperclipai/db"; +import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils"; +import type { NativeExecutionInput, PrpEvent } from "../../vendor/paperclip-runner/index.js"; +import { priceAnthropicReceipt } from "../anthropic-pricing.js"; +import { priceCodexReceipt } from "../codex-pricing.js"; +import { liveRunUsageCostCents } from "../budget-live-usage.js"; +import { receiptFingerprint } from "../receipt-fingerprint.js"; +import { persistUsageReceipt, type UsageReceiptEnvelope } from "../usage-receipts.js"; +import { createNativeTurnAccounting, nativeUsageCostUsd, normalizeNativeUsage } from "./native-usage-accounting.js"; + +const terminal = new Set(["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"]); +const uuid = (identity: string) => { + const hex = createHash("sha256").update(identity).digest("hex"); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`; +}; + +export async function createNativeUsageDelivery(db: Db, input: { + companyId: string; runId: string; runnerInstanceId: string; + provider: NativeExecutionInput["provider"]; + billing: Pick; +}) { + const sourceId = uuid(`${input.runId}:${input.runnerInstanceId}`); + const [bound] = await db.update(heartbeatRuns).set({ usageJson: sql` + coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb) || '{"accountingReceiptOwner":"native_producer"}'::jsonb || + case when ${heartbeatRuns.usageJson}->>'accountingReceiptSourceId' = ${sourceId} + then '{}'::jsonb else ${JSON.stringify({ accountingReceiptSourceId: sourceId, accountingReceiptSequence: 0, accountingReceiptReady: false })}::jsonb end` + }).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), + isNull(heartbeatRuns.costAccountedAt))).returning({ id: heartbeatRuns.id }); + if (!bound) throw new Error("native_usage_run_not_available"); + const saved = await db.select({ receipt: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and( + eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId), + eq(runUsageReceipts.sourceId, sourceId))).orderBy(asc(runUsageReceipts.sequence)); + const accepted = new Map(); + const turnAccounting = ["opencode", "acpx"].includes(input.provider.kind) ? createNativeTurnAccounting(input.provider) : null; + let observedUsage: Record | null = null; + let observedTurn: string | undefined; + let activeProviderTurn: string | undefined; + const observe = (envelope: UsageReceiptEnvelope) => { + const fact = envelope.producerEvent!; + if (fact.eventType === "turn.started") activeProviderTurn = fact.providerTurnId ?? fact.sourceEventId; + const accountingTurn = fact.providerTurnId ?? activeProviderTurn ?? fact.turnId; + const interpretedUsage = fact.usage ? { ...fact.usage, runDeltaComplete: fact.usage.runDeltaComplete === true + && (input.provider.kind !== "codex" || fact.usage.runDeltaBasis === "run") } : undefined; + const event = { sourceInstanceId: input.runnerInstanceId, sourceSeq: envelope.sequence, + turnId: accountingTurn, eventType: fact.eventType, + payload: interpretedUsage ? { kind: "usage", usage: interpretedUsage } : {} } as PrpEvent; + if (turnAccounting) { + turnAccounting.observe(event); + return turnAccounting.snapshot(); + } + if (fact.eventType === "turn.started") observedTurn = undefined; + if (interpretedUsage) { + observedUsage = interpretedUsage; + observedTurn = normalizeNativeUsage(observedUsage, { inputIncludesCacheReads: input.provider.kind === "codex" }) ? accountingTurn : undefined; + } + return { usage: observedUsage, complete: terminal.has(fact.eventType) && observedTurn !== undefined && observedTurn === accountingTurn }; + }; + for (const row of saved) { + const envelope = row.receipt as UsageReceiptEnvelope; + if (!envelope.producerEvent) throw new Error("native_usage_receipt_missing_producer"); + accepted.set(envelope.producerEvent.sourceEventId, envelope); + observe(envelope); + } + let pending: UsageReceiptEnvelope | undefined; + let chain = Promise.resolve(); + const capture = async (event: PrpEvent) => { + if (event.eventType !== "usage.reported" && event.eventType !== "turn.started" && !terminal.has(event.eventType)) return; + if (event.runId !== input.runId || event.sourceInstanceId !== input.runnerInstanceId) throw new Error("native_usage_source_mismatch"); + const payload = event.payload as Record; + const fact: NonNullable = { + sourceEventId: event.sourceEventId, turnId: event.turnId, eventType: event.eventType, + ...(typeof payload.providerTurnId === "string" ? { providerTurnId: payload.providerTurnId } : {}), + ...(event.eventType === "usage.reported" ? { usage: { runDelta: payload.runDelta, runDeltaComplete: payload.runDeltaAvailable === true, runDeltaBasis: payload.runDeltaBasis, + ...(payload.reconciliation && typeof payload.reconciliation === "object" ? { reconciliation: payload.reconciliation } : {}) } } : {}), + }; + const previous = accepted.get(event.sourceEventId) ?? (pending?.producerEvent?.sourceEventId === event.sourceEventId ? pending : undefined); + if (previous && (previous.sequence !== event.sourceSeq || previous.receivedAt !== event.emittedAt + || receiptFingerprint(previous.producerEvent) !== receiptFingerprint(fact))) throw new Error("native_usage_source_conflict"); + if (accepted.has(event.sourceEventId)) return; + if (pending && pending.producerEvent?.sourceEventId !== event.sourceEventId) throw new Error("native_usage_report_pending"); + if (!pending) { + const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", + id: uuid(`${input.runId}:${input.runnerInstanceId}:${event.sourceEventId}`), companyId: input.companyId, + runId: input.runId, adapterType: "paperclip_runner", sourceId, sequence: event.sourceSeq, + receivedAt: event.emittedAt, producerEvent: fact, receipt: { complete: false } }; + const snapshot = observe(envelope); + const usage = normalizeNativeUsage(snapshot.usage, { inputIncludesCacheReads: input.provider.kind === "codex" }); + envelope.receipt = priceAnthropicReceipt(priceCodexReceipt({ ...input.billing, + model: input.provider.model ?? "unknown", usage, usageBasis: "per_run", complete: snapshot.complete && usage !== undefined, + costUsd: usage ? nativeUsageCostUsd(snapshot.usage, input.provider) ?? null : null, + costUsdExact: typeof snapshot.usage?.accountingCostUsdExact === "string" ? snapshot.usage.accountingCostUsdExact : undefined, + costStatus: usage === undefined || snapshot.usage?.accountingCostIncomplete === true ? "unpriced" + : input.provider.kind === "aws_agentcore" ? "estimated" : undefined, + })); + envelope.liveCostCents = liveRunUsageCostCents(envelope.receipt as AdapterUsageCheckpoint); + pending = envelope; + } + await persistUsageReceipt(db, pending); + accepted.set(event.sourceEventId, pending); + pending = undefined; + }; + return { capture(event: PrpEvent) { + const result = chain.then(() => capture(event)); + chain = result.catch(() => undefined); + return result; + } }; +} diff --git a/server/src/services/native-runtime/paperclip-control-plane-port.ts b/server/src/services/native-runtime/paperclip-control-plane-port.ts index 172fe5f0636..2ee3446f568 100644 --- a/server/src/services/native-runtime/paperclip-control-plane-port.ts +++ b/server/src/services/native-runtime/paperclip-control-plane-port.ts @@ -88,8 +88,8 @@ export class PaperclipControlPlanePort implements ControlPlanePort { readonly #assertRunAdmissionActive?: () => void; readonly #onResultCommitted?: () => Promise; readonly #onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise; - readonly #onCommittedEvent?: (event: PrpEvent) => Promise; - readonly #onDuplicateEvent?: (event: PrpEvent) => Promise; + readonly #onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise; + readonly #onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise; constructor( db: Db, @@ -103,8 +103,8 @@ export class PaperclipControlPlanePort implements ControlPlanePort { assertRunAdmissionActive?: () => void; onResultCommitted?: () => Promise; onEventPersistence?: (event: PrpEvent, startedAtMs: number, endedAtMs: number) => Promise; - onCommittedEvent?: (event: PrpEvent) => Promise; - onDuplicateEvent?: (event: PrpEvent) => Promise; + onCommittedEvent?: (event: PrpEvent, cursor: number) => Promise; + onDuplicateEvent?: (event: PrpEvent, cursor: number) => Promise; } = {}, ) { this.#db = db; @@ -313,13 +313,13 @@ export class PaperclipControlPlanePort implements ControlPlanePort { }); } catch { } - await this.#onCommittedEvent?.(event); + await this.#onCommittedEvent?.(event, persisted.row.seq); } else { // A recovered runner may replay the event whose durable side effects // parked the prior attempt. Do not repeat those effects, but let the // embedding runtime refresh observational state before appendEvent // returns to its synchronous governed-wait boundary. - await this.#onDuplicateEvent?.(event); + await this.#onDuplicateEvent?.(event, persisted.row.seq); } return { cursor: persisted.row.seq, diff --git a/server/src/services/object-byte-journal.test.ts b/server/src/services/object-byte-journal.test.ts new file mode 100644 index 00000000000..3caddc32c08 --- /dev/null +++ b/server/src/services/object-byte-journal.test.ts @@ -0,0 +1,326 @@ +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { StorageProvider } from "../storage/types.js"; +import { createObjectByteJournal, ObjectJournalConflictError } from "./object-byte-journal.js"; + +async function readBody(body: AsyncIterable) { + const chunks: Buffer[] = []; + for await (const bytes of body) chunks.push(Buffer.from(bytes)); + return Buffer.concat(chunks); +} + +function storage() { + const objects = new Map(); + const calls = { get: 0, put: 0, maxPut: 0 }; + let loseHeadResponse = false; + const etag = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + const missing = () => Object.assign(new Error("missing"), { name: "NoSuchKey" }); + const conflict = () => Object.assign(new Error("conflict"), { name: "PreconditionFailed" }); + const provider: StorageProvider = { + id: "s3", + async getObject({ objectKey, signal }) { + signal?.throwIfAborted(); + calls.get++; + const bytes = objects.get(objectKey); + if (!bytes) throw missing(); + return { stream: Readable.from([bytes]), etag: etag(bytes), contentLength: bytes.length }; + }, + async putObject() { + throw new Error("unconditional PUT forbidden"); + }, + async putObjectConditional(input) { + const old = objects.get(input.objectKey); + if (input.ifNoneMatch ? old : !old || etag(old) !== input.ifMatch) throw conflict(); + const bytes = Buffer.isBuffer(input.body) ? input.body : await readBody(input.body); + objects.set(input.objectKey, Buffer.from(bytes)); + calls.put++; + calls.maxPut = Math.max(calls.maxPut, bytes.length); + if (loseHeadResponse && input.objectKey.endsWith("/head.json")) { + loseHeadResponse = false; + throw new Error("response lost after PUT"); + } + return { etag: etag(bytes) }; + }, + async headObject({ objectKey }) { + return { exists: objects.has(objectKey) }; + }, + async deleteObject({ objectKey }) { + objects.delete(objectKey); + }, + async listObjects({ prefix }) { + return { keys: [...objects.keys()].filter((k) => k.startsWith(prefix)) }; + }, + }; + return { + provider, + objects, + calls, + loseResponse: () => { + loseHeadResponse = true; + }, + }; +} +const authority = async () => undefined; +afterEach(() => vi.useRealTimers()); + +describe("object byte journal", () => { + it("keeps a shared capability probe alive for another writer when one claim cancels", async () => { + const s = storage(); + const put = s.provider.putObjectConditional!; + let release!: () => void, started!: () => void, admitted!: () => void; + let probeSignal: AbortSignal | undefined; + const probing = new Promise((resolve) => { + started = resolve; + }); + const joined = new Promise((resolve) => { + admitted = resolve; + }); + s.provider.putObjectConditional = async (input) => { + if (input.objectKey.includes("conditional-probe-") && !probeSignal) { + probeSignal = input.signal; + await new Promise((resolve, reject) => { + release = resolve; + input.signal!.addEventListener("abort", () => reject(input.signal!.reason), { + once: true, + }); + started(); + }); + } + return put(input); + }; + const controller = new AbortController(); + const a = createObjectByteJournal({ provider: s.provider, prefix: "company/a/log" }); + const first = a.claim({ + expectedOwner: null, + assertAuthority: authority, + signal: controller.signal, + }); + void first.catch(() => undefined); + await probing; + const b = createObjectByteJournal({ provider: s.provider, prefix: "company/b/log" }); + const second = b.claim({ + expectedOwner: null, + assertAuthority: async () => { + admitted(); + }, + }); + await joined; + await Promise.resolve(); + controller.abort(); + await expect(first).rejects.toMatchObject({ name: "AbortError" }); + expect(probeSignal!.aborted).toBe(false); + release(); + const writer = await second; + expect((await b.head())!.head.owner).toBe(writer.head.owner); + }); + it("recovers committed bytes and stable ranges with no local filesystem", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "company/run/log" }); + let writer = await a.claim({ expectedOwner: null, assertAuthority: authority }); + const expected = Buffer.from("first\n再開\n".repeat(20)); + writer = await a.append(writer, expected.subarray(0, 17)); + writer = await a.append(writer, expected.subarray(17)); + const b = createObjectByteJournal({ provider: s.provider, prefix: "company/run/log" }); + const snapshot = (await b.head())!; + expect(await b.readRange(snapshot.head, 0, expected.length)).toEqual(expected); + expect(await b.readRange(snapshot.head, 7, 23)).toEqual(expected.subarray(7, 30)); + expect(await b.readRange(snapshot.head, expected.length, 100)).toEqual(Buffer.alloc(0)); + }); + it("fences stale appends/finalization and preserves a pinned reader snapshot", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let old = await a.claim({ expectedOwner: null, assertAuthority: authority }); + old = await a.append(old, Buffer.from("before")); + const snapshot = old.head; + let next = await a.claim({ expectedOwner: old.head.owner, assertAuthority: authority }); + next = await a.append(next, Buffer.from("after")); + await expect(a.append(old, Buffer.from("lost"))).rejects.toBeInstanceOf( + ObjectJournalConflictError, + ); + await expect(a.close(old)).rejects.toBeInstanceOf(ObjectJournalConflictError); + expect(await a.readRange(snapshot, 0, 100)).toEqual(Buffer.from("before")); + expect(await a.readRange(next.head, 0, 100)).toEqual(Buffer.from("beforeafter")); + }); + it("reconciles an uncertain manifest commit without duplicating output", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await a.claim({ expectedOwner: null, assertAuthority: authority }); + s.loseResponse(); + writer = await a.append(writer, Buffer.from("once")); + expect(writer.head.end).toBe(4); + expect(await a.readRange((await a.head())!.head, 0, 100)).toEqual(Buffer.from("once")); + }); + it("keeps index objects bounded over multiple tree levels", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await a.claim({ expectedOwner: null, assertAuthority: authority }); + for (let i = 0; i < 180; i++) + writer = await a.append(writer, Buffer.from(String(i).padStart(4, "0"))); + expect(await a.readRange(writer.head, 252, 16)).toEqual(Buffer.from("0063006400650066")); + expect(await a.readRange(writer.head, 700, 20)).toEqual(Buffer.from("01750176017701780179")); + expect(s.calls.maxPut).toBeLessThan(20_000); + expect(s.calls.put).toBeLessThan(180 * 5); + const reachable = new Set(); + function visit(ref: { key: string }, level: number) { + reachable.add(ref.key); + if (level === 0) return; + const node = JSON.parse(s.objects.get(ref.key)!.toString()); + for (const child of node.children) visit(child, level - 1); + } + for (const root of writer.head.roots!) visit(root, root.level); + expect([...reachable].sort()).toEqual( + [...s.objects.keys()].filter((key) => key.includes("/objects/")).sort(), + ); + }); + it("retries a transient head conflict without fencing its current writer", async () => { + const s = storage(), + journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority }); + const original = s.provider.putObjectConditional!; + let conflicts = 0; + s.provider.putObjectConditional = async (input) => { + if (input.objectKey.endsWith("/head.json") && conflicts++ === 0) + throw Object.assign(new Error("in-flight conflict"), { + name: "ConditionalRequestConflict", + $metadata: { httpStatusCode: 409 }, + }); + return original(input); + }; + writer = await journal.append(writer, Buffer.from("once")); + expect(await journal.readRange(writer.head, 0, 100)).toEqual(Buffer.from("once")); + expect(conflicts).toBe(2); + }); + it("does not accept a provider's 409 response as proof of conditional enforcement", async () => { + const s = storage(), + original = s.provider.putObjectConditional!; + s.provider.putObjectConditional = async (input) => { + if (s.objects.has(input.objectKey)) + throw Object.assign(new Error("conflict"), { $metadata: { httpStatusCode: 409 } }); + return original(input); + }; + const journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + await expect( + journal.claim({ expectedOwner: null, assertAuthority: authority }), + ).rejects.toThrow("conflict"); + expect(s.objects.size).toBe(0); + }); + it("retries transient segment failures and acknowledges exactly one batch", async () => { + const s = storage(), + journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority }); + const original = s.provider.putObjectConditional!; + let failures = 2; + s.provider.putObjectConditional = async (input) => { + if (input.objectKey.includes("/objects/") && failures-- > 0) + throw Object.assign(new Error("temporary outage"), { $metadata: { httpStatusCode: 503 } }); + return original(input); + }; + writer = await journal.append(writer, Buffer.from("one batch")); + expect(await journal.readRange(writer.head, 0, 100)).toEqual(Buffer.from("one batch")); + }); + it("rejects corrupt published data and cancelled reads", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await a.claim({ expectedOwner: null, assertAuthority: authority }); + writer = await a.append(writer, Buffer.from("data")); + const data = [...s.objects].find(([, bytes]) => bytes.toString() === "data")!; + s.objects.set(data[0], Buffer.from("oops")); + await expect(a.readRange(writer.head, 0, 4)).rejects.toThrow("integrity"); + await expect(a.readRange(writer.head, 0, 4, AbortSignal.abort())).rejects.toThrow(); + }); + it("retains a deletion tombstone and denies a subsequent claim", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await a.claim({ expectedOwner: null, assertAuthority: authority }); + writer = await a.append(writer, Buffer.from("private")); + await a.close(writer); + await a.remove(); + expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]); + expect((await a.head())!.head.deleted).toBe(true); + await expect( + a.claim({ expectedOwner: writer.head.owner, assertAuthority: authority }), + ).rejects.toThrow(); + }); + it("does not claim ownership when execution authority rejects", async () => { + const s = storage(), + a = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + await expect( + a.claim({ + expectedOwner: null, + assertAuthority: async () => { + throw new Error("revoked"); + }, + }), + ).rejects.toThrow("revoked"); + expect(s.calls.put).toBe(0); + }); + it("fences a pending first claim when deletion wins before journal creation", async () => { + const s = storage(), + journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let enter!: () => void, release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const blocked = new Promise((resolve) => { + release = resolve; + }); + const pending = journal.claim({ + expectedOwner: null, + assertAuthority: async () => { + enter(); + await blocked; + }, + }); + await entered; + await journal.remove(); + release(); + await expect(pending).rejects.toBeInstanceOf(ObjectJournalConflictError); + expect((await journal.head())!.head.deleted).toBe(true); + expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]); + }); + it("rejects a compatible provider which silently ignores conditional writes", async () => { + const s = storage(); + s.provider.putObjectConditional = async (input) => { + const bytes = Buffer.from(input.body as Buffer); + s.objects.set(input.objectKey, bytes); + return { etag: "ignored-condition" }; + }; + const journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + await expect( + journal.claim({ expectedOwner: null, assertAuthority: authority }), + ).rejects.toThrow("does not enforce conditional"); + expect(s.objects.size).toBe(0); + }); + it("fences an active writer before cleanup and preserves the retry obligation", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + const s = storage(), + journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + let writer = await journal.claim({ expectedOwner: null, assertAuthority: authority }); + writer = await journal.append(writer, Buffer.from("private")); + await expect(journal.remove()).rejects.toThrow("cleanup_pending"); + expect((await journal.head())!.head.deleted).toBe(true); + await expect(journal.append(writer, Buffer.from("late"))).rejects.toBeInstanceOf( + ObjectJournalConflictError, + ); + vi.setSystemTime(Date.now() + 30_001); + await journal.remove(); + expect([...s.objects.keys()]).toEqual(["c/r/log/head.json"]); + }); + it("does not publish a partial batch when a later segment fails", async () => { + const s = storage(), + journal = createObjectByteJournal({ provider: s.provider, prefix: "c/r/log" }); + const writer = await journal.claim({ expectedOwner: null, assertAuthority: authority }); + const original = s.provider.putObjectConditional!; + const failed = Buffer.alloc(1024 * 1024, 2); + const key = createHash("sha256").update(failed).digest("hex"); + s.provider.putObjectConditional = (input) => { + if (input.objectKey.endsWith(key)) throw Error("second segment failed"); + return original(input); + }; + await expect( + journal.append(writer, Buffer.concat([Buffer.alloc(1024 * 1024, 1), failed])), + ).rejects.toThrow("second segment failed"); + expect((await journal.head())!.head.end).toBe(0); + }); +}); diff --git a/server/src/services/object-byte-journal.ts b/server/src/services/object-byte-journal.ts new file mode 100644 index 00000000000..67f28d0c604 --- /dev/null +++ b/server/src/services/object-byte-journal.ts @@ -0,0 +1,574 @@ +import { createHash, randomUUID } from "node:crypto"; +import { z } from "zod"; +import { setTimeout as delay } from "node:timers/promises"; +import type { StorageProvider } from "../storage/types.js"; + +const MAX_INDEX_LEVEL = 52; +const MAX_OBJECT_BYTES = 4 * 1024 * 1024; +const offset = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER); +const referenceSchema = z + .object({ + key: z.string(), + start: offset, + end: offset, + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict(); +type Reference = z.infer; +const nodeSchema = z + .object({ + level: z.number().int().min(0).max(MAX_INDEX_LEVEL), + children: z.array(referenceSchema).min(1).max(64), + }) + .strict(); +type Node = z.infer; +const rootSchema = referenceSchema.extend({ level: z.number().int().min(0).max(MAX_INDEX_LEVEL) }); +type Root = z.infer; +const headSchema = z + .object({ + v: z.literal(1), + owner: z.string().uuid(), + generation: offset, + end: offset, + sequence: offset.default(0), + sourceCursor: offset.default(0), + root: referenceSchema.nullable(), + roots: z + .array(rootSchema) + .max(MAX_INDEX_LEVEL + 1) + .optional(), + closed: z.boolean(), + deleted: z.boolean(), + cleanupAfter: offset.default(0), + }) + .strict(); +export type ObjectJournalHead = z.infer; +export interface ObjectJournalWriter { + head: ObjectJournalHead; + etag: string; +} +export class ObjectJournalConflictError extends Error { + readonly code = "object_journal_writer_conflict"; + constructor() { + super("The output journal is owned by another writer"); + } +} +export function isObjectPreconditionFailure(error: unknown) { + const e = error as { name?: string; $metadata?: { httpStatusCode?: number } }; + return e?.name === "PreconditionFailed" || e?.$metadata?.httpStatusCode === 412; +} +function isRetryable(error: unknown) { + const e = error as { name?: string; code?: string; $metadata?: { httpStatusCode?: number } }; + const status = e?.$metadata?.httpStatusCode; + return ( + e?.name === "ConditionalRequestConflict" || + status === 409 || + status === 429 || + (status !== undefined && status >= 500) || + ["TimeoutError", "RequestTimeout", "NetworkingError"].includes(e?.name ?? "") || + ["ECONNRESET", "ECONNREFUSED", "ETIMEDOUT", "EPIPE"].includes(e?.code ?? "") + ); +} +async function retryObjectOperation(operation: () => Promise, signal?: AbortSignal) { + for (let attempt = 0; ; attempt++) { + signal?.throwIfAborted(); + try { + return await operation(); + } catch (error) { + if (attempt >= 2 || !isRetryable(error)) throw error; + await delay(100 * 2 ** attempt, undefined, { signal }); + } + } +} +function isMissing(error: unknown) { + const e = error as { name?: string; status?: number; $metadata?: { httpStatusCode?: number } }; + return ( + e?.name === "NoSuchKey" || + e?.name === "NotFound" || + e?.status === 404 || + e?.$metadata?.httpStatusCode === 404 + ); +} +const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +interface ConditionalQualification { + work: Promise; + controller: AbortController; + waiters: number; +} +const qualifiedProviders = new WeakSet(); +const qualifications = new WeakMap(); +async function qualifyConditionalWrites( + provider: StorageProvider, + prefix: string, + signal?: AbortSignal, +) { + signal?.throwIfAborted(); + if (qualifiedProviders.has(provider)) return; + let qualification = qualifications.get(provider); + if (!qualification || qualification.controller.signal.aborted) { + const controller = new AbortController(); + const probeSignal = AbortSignal.any([controller.signal, AbortSignal.timeout(15_000)]); + const current: ConditionalQualification = { controller, waiters: 0, work: Promise.resolve() }; + qualification = current; + current.work = (async () => { + const objectKey = `${prefix}/conditional-probe-${randomUUID()}`; + const body = Buffer.from("paperclip-conditional-probe-v1"); + const put = provider.putObjectConditional!.bind(provider); + try { + await retryObjectOperation( + () => + put({ + objectKey, + body, + contentLength: body.length, + contentType: "text/plain", + ifNoneMatch: "*", + signal: probeSignal, + }), + probeSignal, + ); + for (const condition of [ + { ifNoneMatch: "*" as const }, + { ifMatch: '"paperclip-impossible-etag"' }, + ]) { + let rejected = false; + try { + await retryObjectOperation( + () => + put({ + objectKey, + body, + contentLength: body.length, + contentType: "text/plain", + ...condition, + signal: probeSignal, + }), + probeSignal, + ); + } catch (error) { + if (!isObjectPreconditionFailure(error)) throw error; + rejected = true; + } + if (!rejected) throw new Error("Object storage does not enforce conditional publication"); + } + } finally { + await provider.deleteObject({ objectKey, signal: probeSignal }); + } + probeSignal.throwIfAborted(); + qualifiedProviders.add(provider); + })().finally(() => { + if (qualifications.get(provider) === current) qualifications.delete(provider); + }); + qualifications.set(provider, current); + void current.work.catch(() => undefined); + } + const pending = qualification; + pending.waiters++; + await new Promise((resolve, reject) => { + let finished = false; + const release = () => { + if (finished) return false; + finished = true; + signal?.removeEventListener("abort", abort); + pending.waiters--; + return true; + }; + const abort = () => { + if (!release()) return; + if (!pending.waiters) { + pending.controller.abort(signal!.reason); + void pending.work.then( + () => reject(signal!.reason), + () => reject(signal!.reason), + ); + } else reject(signal!.reason); + }; + signal?.addEventListener("abort", abort, { once: true }); + if (signal?.aborted) abort(); + pending.work.then( + () => { + if (release()) resolve(); + }, + (error) => { + if (release()) reject(error); + }, + ); + }); +} + +let activeIO = 0; +const waitingIO: Array<() => void> = []; +async function objectIO(operation: () => Promise): Promise { + if (activeIO >= 8) { + if (waitingIO.length >= 32) throw new Error("object_journal_io_capacity_exceeded"); + await new Promise((resolve) => waitingIO.push(resolve)); + } else activeIO++; + try { + return await operation(); + } finally { + const next = waitingIO.shift(); + if (next) next(); + else activeIO--; + } +} + +export interface ObjectByteJournalOptions { + provider: StorageProvider; + prefix: string; + + encode?: (bytes: Buffer) => Promise; + decode?: (bytes: Buffer) => Promise; +} + +export function createObjectByteJournal(options: ObjectByteJournalOptions) { + const { provider } = options; + const prefix = options.prefix.replace(/\/+$/, ""); + if ( + !prefix || + prefix.length > 512 || + prefix.startsWith("/") || + prefix.split("/").some((p) => !/^[a-zA-Z0-9._-]+$/.test(p)) + ) { + throw new Error("invalid_object_journal_prefix"); + } + if (!provider.putObjectConditional || !provider.listObjects) + throw new Error("Object storage requires conditional writes and scoped listing"); + const headKey = `${prefix}/head.json`; + const objectPrefix = `${prefix}/objects/`; + const putConditional = ( + input: Parameters>[0], + ) => objectIO(() => provider.putObjectConditional!(input)); + + async function getBytes(key: string, limit: number, signal?: AbortSignal) { + return objectIO(async () => { + signal?.throwIfAborted(); + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000); + const result = await retryObjectOperation( + () => provider.getObject({ objectKey: key, signal }), + signal, + ); + const chunks: Buffer[] = []; + let size = 0; + try { + for await (const chunk of result.stream) { + const bytes = Buffer.from(chunk); + size += bytes.length; + if (size > limit) throw new Error("object_journal_object_too_large"); + chunks.push(bytes); + } + } finally { + result.stream.destroy(); + } + signal?.throwIfAborted(); + return { bytes: Buffer.concat(chunks), etag: result.etag }; + }); + } + async function head(signal?: AbortSignal): Promise { + let result; + try { + result = await getBytes(headKey, 64 * 1024, signal); + } catch (error) { + if (isMissing(error)) return null; + throw error; + } + if (!result.etag) throw new Error("object_journal_missing_etag"); + const value = headSchema.parse(JSON.parse(result.bytes.toString("utf8"))); + if (value.roots) { + let end = 0, + level = MAX_INDEX_LEVEL + 1; + if (value.root) throw new Error("object_journal_invalid_head"); + for (const root of value.roots) { + validateReference(root); + if (root.start !== end || root.level >= level) + throw new Error("object_journal_invalid_head"); + end = root.end; + level = root.level; + } + if (end !== value.end) throw new Error("object_journal_invalid_head"); + } else if ((value.root?.end ?? 0) !== value.end || (value.root && value.root.start !== 0)) { + throw new Error("object_journal_invalid_head"); + } + return { head: value, etag: result.etag }; + } + async function publish(value: ObjectJournalHead, etag: string | null, signal?: AbortSignal) { + value = headSchema.parse(value); + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000); + const bytes = Buffer.from(JSON.stringify(value)); + return retryObjectOperation(async () => { + try { + const result = await putConditional({ + objectKey: headKey, + body: bytes, + contentLength: bytes.length, + contentType: "application/json", + signal, + ...(etag ? { ifMatch: etag } : { ifNoneMatch: "*" as const }), + }); + return { head: value, etag: result.etag }; + } catch (error) { + const observed = await head(signal); + if (observed && JSON.stringify(observed.head) === JSON.stringify(value)) return observed; + if (isObjectPreconditionFailure(error) || (observed?.etag ?? null) !== etag) + throw new ObjectJournalConflictError(); + throw error; + } + }, signal); + } + async function claim(input: { + expectedOwner: string | null; + assertAuthority: () => Promise; + signal?: AbortSignal; + }) { + const current = await head(input.signal); + if (current?.head.deleted || (current?.head.owner ?? null) !== input.expectedOwner) + throw new ObjectJournalConflictError(); + await input.assertAuthority(); + await qualifyConditionalWrites(provider, prefix, input.signal); + await input.assertAuthority(); + const value: ObjectJournalHead = { + v: 1, + owner: randomUUID(), + generation: (current?.head.generation ?? 0) + 1, + end: current?.head.end ?? 0, + sequence: current?.head.sequence ?? 0, + sourceCursor: current?.head.sourceCursor ?? 0, + root: current?.head.root ?? null, + ...(current?.head.roots ? { roots: current.head.roots } : {}), + closed: false, + deleted: false, + cleanupAfter: 0, + }; + return publish(value, current?.etag ?? null, input.signal); + } + function validateReference(ref: Reference) { + if ( + !ref.key.startsWith(objectPrefix) || + ref.key !== `${objectPrefix}${ref.sha256}` || + ref.end <= ref.start + ) + throw new Error("object_journal_invalid_reference"); + } + async function load(ref: Reference, signal?: AbortSignal) { + validateReference(ref); + const stored = await getBytes(ref.key, MAX_OBJECT_BYTES * 2, signal); + const bytes = options.decode ? await options.decode(stored.bytes) : stored.bytes; + if (bytes.length > MAX_OBJECT_BYTES || sha256(bytes) !== ref.sha256) + throw new Error("object_journal_integrity_error"); + return bytes; + } + async function save( + bytes: Buffer, + start: number, + end: number, + signal?: AbortSignal, + ): Promise { + if (bytes.length > MAX_OBJECT_BYTES) throw new Error("object_journal_segment_too_large"); + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000); + const digest = sha256(bytes), + key = `${objectPrefix}${digest}`; + const body = options.encode ? await options.encode(bytes) : bytes; + try { + await retryObjectOperation( + () => + putConditional({ + objectKey: key, + body, + contentLength: body.length, + contentType: "application/octet-stream", + ifNoneMatch: "*", + signal, + }), + signal, + ); + } catch (error) { + const ref = { key, sha256: digest, start, end }; + try { + await load(ref, signal); + } catch { + throw error; + } + } + return { key, start, end, sha256: digest }; + } + async function loadNode(ref: Reference, signal?: AbortSignal): Promise { + const node = nodeSchema.parse(JSON.parse((await load(ref, signal)).toString("utf8"))); + let next = ref.start; + for (const child of node.children) { + validateReference(child); + if (child.start !== next) throw new Error("object_journal_noncontiguous_index"); + next = child.end; + } + if (next !== ref.end) throw new Error("object_journal_invalid_index_range"); + return node; + } + const saveNode = (node: Node, signal?: AbortSignal) => + save( + Buffer.from(JSON.stringify(node)), + node.children[0]!.start, + node.children.at(-1)!.end, + signal, + ); + async function readRange( + snapshot: ObjectJournalHead, + start: number, + limit: number, + signal?: AbortSignal, + ): Promise { + if (snapshot.deleted) throw new Error("object_journal_deleted"); + if ( + !Number.isSafeInteger(start) || + start < 0 || + !Number.isSafeInteger(limit) || + limit < 0 || + limit > MAX_OBJECT_BYTES + ) + throw new Error("invalid_object_journal_range"); + if ((!snapshot.root && !snapshot.roots?.length) || limit === 0 || start >= snapshot.end) + return Buffer.alloc(0); + const end = Math.min(snapshot.end, start + limit), + chunks: Buffer[] = []; + async function readPayload(ref: Reference) { + const bytes = await load(ref, signal); + if (bytes.length !== ref.end - ref.start) + throw new Error("object_journal_invalid_segment_length"); + chunks.push( + bytes.subarray(Math.max(0, start - ref.start), Math.min(bytes.length, end - ref.start)), + ); + } + async function visit(ref: Reference, expectedLevel?: number) { + const node = await loadNode(ref, signal); + if (expectedLevel !== undefined && node.level !== expectedLevel) + throw new Error("object_journal_invalid_index_level"); + for (const child of node.children) { + if (child.end <= start || child.start >= end) continue; + if (node.level > 0) await visit(child, node.level - 1); + else await readPayload(child); + } + } + if (snapshot.roots) { + for (const root of snapshot.roots) { + if (root.end <= start || root.start >= end) continue; + if (root.level === 0) await readPayload(root); + else await visit(root, root.level - 1); + } + } else if (snapshot.root) await visit(snapshot.root); + const bytes = Buffer.concat(chunks); + if (bytes.length !== end - start) throw new Error("object_journal_missing_range"); + return bytes; + } + async function append( + writer: ObjectJournalWriter, + bytes: Buffer, + signal?: AbortSignal, + sequence = writer.head.sequence, + sourceCursor = writer.head.sourceCursor, + ) { + signal = signal + ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000); + if (writer.head.closed || writer.head.deleted) throw new ObjectJournalConflictError(); + if (bytes.length === 0) return writer; + if ( + bytes.length > 8 * 1024 * 1024 || + !Number.isSafeInteger(writer.head.end + bytes.length) || + !Number.isSafeInteger(sequence) || + sequence < writer.head.sequence || + !Number.isSafeInteger(sourceCursor) || + sourceCursor < writer.head.sourceCursor + ) + throw new Error("object_journal_invalid_batch"); + const roots: Root[] = writer.head.roots + ? [...writer.head.roots] + : writer.head.root + ? [{ ...writer.head.root, level: (await loadNode(writer.head.root, signal)).level + 1 }] + : []; + let end = writer.head.end; + for (let start = 0; start < bytes.length; start += 1024 * 1024) { + const part = bytes.subarray(start, start + 1024 * 1024); + let root: Root = { ...(await save(part, end, end + part.length, signal)), level: 0 }; + end = root.end; + while (roots.at(-1)?.level === root.level) { + const left = roots.pop()!; + const { level: leftLevel, ...leftRef } = left; + const { level, ...rightRef } = root; + if (level >= MAX_INDEX_LEVEL || leftLevel !== level) + throw new Error("object_journal_index_capacity_exceeded"); + root = { + ...(await saveNode({ level, children: [leftRef, rightRef] }, signal)), + level: level + 1, + }; + } + roots.push(root); + } + return publish( + { ...writer.head, end, root: null, roots, sequence, sourceCursor }, + writer.etag, + signal, + ); + } + + async function close(writer: ObjectJournalWriter, signal?: AbortSignal) { + return publish({ ...writer.head, closed: true }, writer.etag, signal); + } + async function remove(signal?: AbortSignal) { + let tombstone: ObjectJournalWriter | undefined; + for (let attempt = 0; attempt < 8; attempt++) { + const current = await head(signal); + if (current?.head.deleted) { + tombstone = current; + break; + } + const value: ObjectJournalHead = current + ? { + ...current.head, + deleted: true, + closed: true, + cleanupAfter: current.head.closed ? 0 : Date.now() + 30_000, + } + : { + v: 1, + sourceCursor: 0, + owner: randomUUID(), + generation: 0, + end: 0, + sequence: 0, + root: null, + deleted: true, + closed: true, + cleanupAfter: 0, + }; + try { + tombstone = await publish(value, current?.etag ?? null, signal); + break; + } catch (error) { + if (!(error instanceof ObjectJournalConflictError)) throw error; + } + } + if (!tombstone) throw new ObjectJournalConflictError(); + if (tombstone.head.cleanupAfter > Date.now()) throw new Error("object_journal_cleanup_pending"); + + let continuationToken: string | undefined; + do { + const deadline = () => + signal + ? AbortSignal.any([signal, AbortSignal.timeout(15_000)]) + : AbortSignal.timeout(15_000); + const page = await objectIO(() => + provider.listObjects!({ prefix: objectPrefix, continuationToken, signal: deadline() }), + ); + for (const key of page.keys) { + if (!key.startsWith(objectPrefix)) throw new Error("object_journal_invalid_listing"); + await objectIO(() => provider.deleteObject({ objectKey: key, signal: deadline() })); + } + continuationToken = page.continuationToken; + } while (continuationToken); + } + return { head, claim, append, close, readRange, remove }; +} +export type ObjectByteJournal = ReturnType; diff --git a/server/src/services/object-run-log-store.test.ts b/server/src/services/object-run-log-store.test.ts new file mode 100644 index 00000000000..df02e610e73 --- /dev/null +++ b/server/src/services/object-run-log-store.test.ts @@ -0,0 +1,188 @@ +import { createHash } from "node:crypto"; +import { Readable } from "node:stream"; +import { describe, expect, it, vi } from "vitest"; +import type { StorageProvider } from "../storage/types.js"; +import { createObjectRunLogStore } from "./object-run-log-store.js"; + +async function readBody(body: AsyncIterable) { + const chunks: Buffer[] = []; + for await (const bytes of body) chunks.push(Buffer.from(bytes)); + return Buffer.concat(chunks); +} + +function memory() { + const objects = new Map(); + let puts = 0, + unavailable = false; + const etag = (b: Buffer) => createHash("sha256").update(b).digest("hex"); + const provider: StorageProvider = { + id: "s3", + async putObject() { + throw Error("unconditional"); + }, + async putObjectConditional(i) { + if (unavailable) throw Error("storage offline"); + const old = objects.get(i.objectKey); + if (i.ifNoneMatch ? old : !old || etag(old) !== i.ifMatch) + throw Object.assign(Error("conflict"), { name: "PreconditionFailed" }); + const b = Buffer.isBuffer(i.body) ? i.body : await readBody(i.body); + objects.set(i.objectKey, Buffer.from(b)); + puts++; + return { etag: etag(b) }; + }, + async getObject(i) { + const b = objects.get(i.objectKey); + if (!b) throw Object.assign(Error("missing"), { name: "NoSuchKey" }); + return { stream: Readable.from([b]), etag: etag(b) }; + }, + async headObject(i) { + return { exists: objects.has(i.objectKey) }; + }, + async deleteObject(i) { + objects.delete(i.objectKey); + }, + async listObjects(i) { + return { keys: [...objects.keys()].filter((k) => k.startsWith(i.prefix)) }; + }, + }; + return { + provider, + puts: () => puts, + offline: (value: boolean) => { + unavailable = value; + }, + }; +} +const input = { companyId: "company", agentId: "agent", runId: "run" }; +const event = (seq: number) => ({ + stream: "stdout" as const, + chunk: "再開", + ts: "2026-10-30T00:00:00Z", + seq, +}); + +describe("S3 run log store", () => { + it("batches concurrent appends, confirms durability, and resumes offsets/sequence on a replacement", async () => { + const m = memory(), + options = { + provider: m.provider, + keyPrefix: "logs", + flushMs: 10, + assertWriterAuthority: async () => undefined, + }; + const a = createObjectRunLogStore(options), + handle = await a.begin(input), + before = m.puts(); + const sizes = await Promise.all( + Array.from({ length: 100 }, (_, i) => a.append(handle, event(i + 1))), + ); + expect(m.puts() - before).toBe(2); + const b = createObjectRunLogStore(options); + const read = await b.read(handle); + expect(read.content.trim().split("\n")).toHaveLength(100); + const successor = await b.begin(input); + expect(successor.initialSequence).toBe(100); + expect(successor.initialBytes).toBe(sizes.reduce((sum, size) => sum + size, 0)); + await b.append(successor, event(101)); + const summary = await b.finalize(successor); + expect(summary.bytes).toBeGreaterThan(successor.initialBytes!); + expect(summary.sha256).toBe( + createHash("sha256") + .update((await b.read(successor)).content) + .digest("hex"), + ); + await expect(a.append(handle, event(102))).rejects.toThrow("another writer"); + }); + it("finalization flushes already accepted writes and rejects later append admission", async () => { + const m = memory(), + store = createObjectRunLogStore({ + provider: m.provider, + keyPrefix: "logs", + flushMs: 1000, + assertWriterAuthority: async () => undefined, + }); + const h = await store.begin(input), + pending = store.append(h, event(1)); + const finalizing = store.finalize(h); + await expect(store.append(h, event(2))).rejects.toThrow("closed"); + const [size, result] = await Promise.all([pending, finalizing]); + expect(result.bytes).toBe(size); + }); + it("does not acknowledge a batch when storage fails", async () => { + const m = memory(), + store = createObjectRunLogStore({ + provider: m.provider, + keyPrefix: "logs", + flushMs: 1, + assertWriterAuthority: async () => undefined, + }); + const h = await store.begin(input); + m.offline(true); + await expect(store.append(h, event(1))).rejects.toThrow("storage offline"); + m.offline(false); + expect((await store.read(h)).content).toBe(""); + }); + it("preserves UTF-8 and byte offsets across tiny pages and aligned tails", async () => { + const m = memory(), + store = createObjectRunLogStore({ + provider: m.provider, + keyPrefix: "logs", + flushMs: 1, + assertWriterAuthority: async () => undefined, + }); + const h = await store.begin(input); + await store.append(h, { ...event(1), chunk: "再開😀" }); + await store.append(h, event(2)); + const full = Buffer.from((await store.read(h)).content); + let offset = 0, + text = ""; + for (;;) { + const page = await store.read(h, { offset, limitBytes: 7 }); + text += page.content; + if (page.nextOffset === undefined) break; + expect(page.nextOffset).toBeGreaterThan(offset); + offset = page.nextOffset; + } + expect(Buffer.from(text)).toEqual(full); + const start = full.indexOf(Buffer.from("再")) + 1; + const tail = await store.read(h, { offset: start, limitBytes: 1000, alignStartToRecord: true }); + expect(Buffer.from(tail.content)).toEqual(full); + }); + it("backpressures admission at capacity without poisoning the writer or losing accepted output", async () => { + const m = memory(), + original = m.provider.putObjectConditional!; + let release!: () => void, + entered = false; + const blocked = new Promise((resolve) => { + release = resolve; + }); + m.provider.putObjectConditional = async (input) => { + if (input.objectKey.includes("/objects/") && !entered) { + entered = true; + await blocked; + } + return original(input); + }; + const store = createObjectRunLogStore({ + provider: m.provider, + keyPrefix: "logs", + assertWriterAuthority: async () => undefined, + }); + const h = await store.begin(input); + const first = await store.enqueue!(h, { ...event(1), chunk: "a".repeat(7 * 1024 * 1024) }); + await vi.waitFor(() => expect(entered).toBe(true)); + let admitted = false; + const second = store.enqueue!(h, { ...event(2), chunk: "b".repeat(2 * 1024 * 1024) }).then( + (receipt) => { + admitted = true; + return receipt.committed; + }, + ); + await Promise.resolve(); + expect(admitted).toBe(false); + release(); + await Promise.all([first.committed, second]); + await store.append(h, event(3)); + expect((await store.finalize(h)).bytes).toBeGreaterThan(9 * 1024 * 1024); + }); +}); diff --git a/server/src/services/object-run-log-store.ts b/server/src/services/object-run-log-store.ts new file mode 100644 index 00000000000..e8ca21fe6ba --- /dev/null +++ b/server/src/services/object-run-log-store.ts @@ -0,0 +1,339 @@ +import { beginIdleTrackedWork } from "./task-admission.js"; +import { createHash, randomUUID } from "node:crypto"; +import type { StorageProvider } from "../storage/types.js"; +import type { RunLogStore, RunLogHandle, RunLogBeginInput } from "./run-log-store.js"; +import { createObjectByteJournal, type ObjectJournalWriter } from "./object-byte-journal.js"; + +const MAX_SEGMENT_BYTES = 1024 * 1024; +const MAX_PENDING_BYTES = 8 * MAX_SEGMENT_BYTES; +const MAX_READ_BYTES = 4 * MAX_SEGMENT_BYTES; +export interface ObjectRunLogStoreOptions { + provider: StorageProvider; + keyPrefix: string; + flushMs?: number; + segmentBytes?: number; + + assertWriterAuthority: (input: RunLogBeginInput) => Promise; +} +interface PendingAppend { + bytes: Buffer; + sequence: number; + sourceCursor: number; + resolve: (bytes: number) => void; + reject: (error: unknown) => void; +} +interface WriterState { + journal: ReturnType; + writer: ObjectJournalWriter; + pending: PendingAppend[]; + pendingBytes: number; + timer?: NodeJS.Timeout; + flushing?: Promise; + closed: boolean; + failure?: unknown; + signal?: AbortSignal; +} + +export function createObjectRunLogStore(options: ObjectRunLogStoreOptions): RunLogStore { + const flushMs = options.flushMs ?? 1000; + const segmentBytes = options.segmentBytes ?? MAX_SEGMENT_BYTES; + if ( + !Number.isSafeInteger(flushMs) || + flushMs < 1 || + flushMs > 30_000 || + !Number.isSafeInteger(segmentBytes) || + segmentBytes < 1024 || + segmentBytes > MAX_SEGMENT_BYTES + ) + throw new Error("Invalid object log batching limits"); + const writers = new Map(); + let totalPendingBytes = 0; + let admissionBytes = 0; + let admissions = 0; + const capacityWaiters = new Set<() => void>(); + const capacityChanged = () => { + for (const wake of capacityWaiters) wake(); + capacityWaiters.clear(); + }; + let activeUploads = 0; + const uploadWaiters: Array<() => void> = []; + async function acquireUpload(signal?: AbortSignal) { + signal?.throwIfAborted(); + if (activeUploads >= 4) + await new Promise((resolve, reject) => { + const enter = () => { + signal?.removeEventListener("abort", abort); + resolve(); + }; + const abort = () => { + const index = uploadWaiters.indexOf(enter); + if (index !== -1) uploadWaiters.splice(index, 1); + reject(signal!.reason); + }; + signal?.addEventListener("abort", abort, { once: true }); + uploadWaiters.push(enter); + }); + else activeUploads++; + return () => { + const next = uploadWaiters.shift(); + if (next) next(); + else activeUploads--; + }; + } + function journal(logRef: string) { + if (!/^[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+\/[a-zA-Z0-9._-]+$/.test(logRef)) + throw new Error("Invalid object log reference"); + return createObjectByteJournal({ + provider: options.provider, + prefix: `${options.keyPrefix}/segments-v1/${logRef}`, + }); + } + function schedule(state: WriterState) { + if (state.closed || state.failure || state.timer || state.flushing || !state.pending.length) + return; + state.timer = setTimeout(() => { + state.timer = undefined; + void flush(state).catch(() => undefined); + }, flushMs); + } + async function flush(state: WriterState): Promise { + if (state.timer) { + clearTimeout(state.timer); + state.timer = undefined; + } + if (state.flushing) { + await state.flushing; + if (state.pending.length) return flush(state); + return; + } + if (state.failure) throw state.failure; + if (!state.pending.length) return; + const batch = state.pending.splice(0); + const bytes = Buffer.concat(batch.map((p) => p.bytes)); + state.flushing = (async () => { + let release: (() => void) | undefined; + try { + release = await acquireUpload(state.signal); + state.writer = await state.journal.append( + state.writer, + bytes, + state.signal, + batch.reduce( + (cursor, item) => Math.max(cursor, item.sequence), + state.writer.head.sequence, + ), + batch.reduce( + (cursor, item) => Math.max(cursor, item.sourceCursor), + state.writer.head.sourceCursor, + ), + ); + for (const item of batch) item.resolve(item.bytes.length); + } catch (error) { + state.failure = error; + const waiting = state.pending.splice(0); + const waitingBytes = waiting.reduce((sum, item) => sum + item.bytes.length, 0); + state.pendingBytes -= waitingBytes; + totalPendingBytes -= waitingBytes; + for (const item of [...batch, ...waiting]) item.reject(error); + throw error; + } finally { + state.pendingBytes -= bytes.length; + totalPendingBytes -= bytes.length; + capacityChanged(); + release?.(); + state.flushing = undefined; + schedule(state); + } + })(); + await state.flushing; + } + async function enqueue(handle: RunLogHandle, event: Parameters[1]) { + const state = writers.get(handle); + if (!state || state.closed) throw new Error("Object run log writer is closed"); + state.signal?.throwIfAborted(); + if (state.failure) throw state.failure; + const bytes = Buffer.from( + `${JSON.stringify({ + ts: event.ts, + attemptId: handle.attemptId, + stream: event.stream, + chunk: event.chunk, + ...(event.seq === undefined ? {} : { seq: event.seq }), + })}\n`, + ); + if (bytes.length > MAX_PENDING_BYTES) + throw new Error("Object run log record exceeds pending-byte limit"); + let waiting = false; + try { + while ( + state.pendingBytes + bytes.length > MAX_PENDING_BYTES || + totalPendingBytes + bytes.length > 64 * MAX_SEGMENT_BYTES + ) { + if (state.closed) throw new Error("Object run log writer is closed"); + if (state.failure) throw state.failure; + if (!waiting) { + if (admissions >= 32 || admissionBytes + bytes.length > MAX_PENDING_BYTES) + throw new Error("Object run log admission capacity exceeded"); + waiting = true; + admissions++; + admissionBytes += bytes.length; + } + for (const writer of writers.values()) + if (writer.pending.length) void flush(writer).catch(() => undefined); + await new Promise((resolve, reject) => { + const wake = () => { + state.signal?.removeEventListener("abort", abort); + resolve(); + }; + const abort = () => { + capacityWaiters.delete(wake); + reject(state.signal!.reason); + }; + state.signal?.addEventListener("abort", abort, { once: true }); + capacityWaiters.add(wake); + }); + } + if (state.closed) throw new Error("Object run log writer is closed"); + state.signal?.throwIfAborted(); + if (state.failure) throw state.failure; + state.pendingBytes += bytes.length; + totalPendingBytes += bytes.length; + const finishIdle = beginIdleTrackedWork(); + const committed = new Promise((resolve, reject) => + state.pending.push({ + bytes, + sourceCursor: event.sourceCursor ?? state.writer.head.sourceCursor, + sequence: event.seq ?? state.writer.head.sequence, + resolve, + reject, + }), + ).finally(finishIdle); + if (state.pendingBytes >= segmentBytes) void flush(state).catch(() => undefined); + else schedule(state); + return { committed }; + } finally { + if (waiting) { + admissions--; + admissionBytes -= bytes.length; + } + } + } + return { + async begin(input) { + const logRef = `${input.companyId}/${input.agentId}/${input.runId}`; + const store = journal(logRef), + current = await store.head(input.signal); + const writer = await store.claim({ + expectedOwner: current?.head.owner ?? null, + assertAuthority: () => options.assertWriterAuthority(input), + signal: input.signal, + }); + const handle: RunLogHandle = { + store: "s3_segments", + logRef, + attemptId: randomUUID(), + initialBytes: writer.head.end, + initialSequence: writer.head.sequence, + initialSourceCursor: writer.head.sourceCursor, + }; + writers.set(handle, { + journal: store, + writer, + pending: [], + pendingBytes: 0, + closed: false, + signal: input.signal, + }); + return handle; + }, + enqueue, + async append(handle, event) { + return (await enqueue(handle, event)).committed; + }, + async flush(handle) { + const state = writers.get(handle); + if (!state) throw new Error("Object run log writer not found"); + await flush(state); + }, + async finalize(handle, finalizeOptions) { + const state = writers.get(handle); + if (!state) throw new Error("Object run log writer not found"); + state.closed = true; + capacityChanged(); + try { + await flush(state); + state.writer = await state.journal.close(state.writer, state.signal); + if (finalizeOptions?.computeDigest === false) + return { bytes: state.writer.head.end, sha256: null, compressed: false }; + const hash = createHash("sha256"); + for (let start = 0; start < state.writer.head.end; start += MAX_READ_BYTES) { + hash.update( + await state.journal.readRange(state.writer.head, start, MAX_READ_BYTES, state.signal), + ); + } + return { bytes: state.writer.head.end, sha256: hash.digest("hex"), compressed: false }; + } catch { + process.emitWarning( + "Run log persistence is incomplete; final size and digest are unknown.", + ); + return { bytes: null, sha256: null, compressed: false }; + } finally { + writers.delete(handle); + } + }, + async read(handle, opts = {}) { + if (handle.store !== "s3_segments") throw new Error("Invalid object run log backend"); + const store = journal(handle.logRef), + snapshot = await store.head(opts.signal); + if (!snapshot || snapshot.head.deleted) throw new Error("Object run log not found"); + const offset = Math.max(0, opts.offset ?? 0), + limit = opts.limitBytes ?? 256_000; + if (!Number.isSafeInteger(limit) || limit < 1 || limit > MAX_READ_BYTES) + throw new Error("Invalid object log read limit"); + const fetched = await store.readRange( + snapshot.head, + offset, + Math.min(MAX_READ_BYTES, limit + 3), + opts.signal, + ); + let end = Math.min(limit, fetched.length); + while (end < fetched.length && (fetched[end]! & 0xc0) === 0x80) end++; + if ( + end === fetched.length && + offset + end < snapshot.head.end && + fetched.length === MAX_READ_BYTES + ) { + let lead = end - 1; + while (lead >= 0 && (fetched[lead]! & 0xc0) === 0x80) lead--; + const first = fetched[lead] ?? 0; + const width = first >= 0xf0 ? 4 : first >= 0xe0 ? 3 : first >= 0xc0 ? 2 : 1; + if (end - lead < width) end = lead; + } + const bytes = fetched.subarray(0, end); + const nextOffset = + offset + bytes.length < snapshot.head.end ? offset + bytes.length : undefined; + if (!opts.alignStartToRecord || offset === 0 || bytes.length === 0) + return { content: bytes.toString("utf8"), nextOffset }; + const parts = [bytes]; + let before = offset; + + while (before > 0) { + const start = Math.max(0, before - 32_000); + const prefix = await store.readRange(snapshot.head, start, before - start, opts.signal); + const boundary = prefix.lastIndexOf(10); + parts.unshift(prefix.subarray(boundary + 1)); + if (boundary >= 0) break; + before = start; + if (parts.reduce((n, part) => n + part.length, 0) > MAX_PENDING_BYTES) + throw new Error("Object log record exceeds read alignment limit"); + } + return { + content: Buffer.concat(parts).toString("utf8"), + nextOffset: nextOffset ?? offset + bytes.length, + }; + }, + async flushInflightMirrors() { + await Promise.all([...writers.values()].map(flush)); + }, + }; +} diff --git a/server/src/services/presentation-ownership-read.test.ts b/server/src/services/presentation-ownership-read.test.ts index 427eb1edb86..c08037e78af 100644 --- a/server/src/services/presentation-ownership-read.test.ts +++ b/server/src/services/presentation-ownership-read.test.ts @@ -1,12 +1,15 @@ -import { randomBytes, randomUUID } from "node:crypto"; +import { createHash, randomBytes, randomUUID } from "node:crypto"; import { agents, companies, completionContracts, createDb, environmentLeases, heartbeatRunEvents, heartbeatRuns, issueComments, issues, nativeRunFinalizations, nativeRunResults, type Db } from "@paperclipai/db"; import { eq, type SQL } from "drizzle-orm"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { trace } from "@opentelemetry/api"; +import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; import { NativeRunCoordinatorStore } from "./native-runtime/native-run-coordinator-store.js"; import { getConversationOwnershipBlocker } from "./conversation-continuation.js"; import { isExternalChatPresentationContext } from "./heartbeat-run-summary.js"; import { finalizeNativeRun, recoverCommittedNativeChatResponse, repairCommittedNativeChatResponse } from "./native-runtime/native-run-finalizer.js"; +import { nativeResponsePresentationWork } from "./native-runtime/native-response-presentation-work.js"; import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; @@ -210,6 +213,53 @@ describe.skipIf(!support.supported)("presentation and ownership PostgreSQL read expect(outcomes[1]).toEqual(outcomes[0]); }); + it.each(["direct", "worker", "discovery and worker"])("traces %s reply recovery independently and leaves reply content out of spans", async lane => { + const f = await unpublishedAcceptedReply(); + const exporter = new InMemorySpanExporter(); + const provider = new TracerProvider({ spanProcessors: [new SimpleSpanProcessor({ exporter })] }); + trace.setGlobalTracerProvider(provider); + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); + try { + if (lane !== "direct") { + const worker = nativeResponsePresentationWork(db); + if (lane === "discovery and worker") { + expect(await worker.load({ companyId: f.companyId, issueId: f.issueId, runId: f.run.id })) + .toEqual([expect.objectContaining({ id: f.run.id })]); + } + expect(await worker.check({ id: f.run.id, companyId: f.companyId, + issueId: f.issueId, agentId: f.agentId, dueAt: 0 })).toBeNull(); + } else { + expect(await repairCommittedNativeChatResponse(db, { companyId: f.companyId, issueId: f.issueId, runId: f.run.id })).toBe(true); + } + const spans = exporter.getFinishedSpans(); + const roots = spans.filter(span => span.name === "task.response_presentation"); + expect(roots).toHaveLength(lane === "discovery and worker" ? 2 : 1); + for (const root of roots) { + expect(root.parentSpanContext).toBeUndefined(); + expect(root.attributes).toEqual({ "paperclip.task.run.run_id": createHash("sha256").update(f.run.id).digest("hex").slice(0, 12) }); + } + const root = roots.at(-1)!; + for (const name of ["response.transaction", "response.transaction.entry"]) { + const child = spans.find(span => span.name === name)!; + expect(child).toBeDefined(); + expect(child.parentSpanContext?.spanId).toBe(root.spanContext().spanId); + } + if (lane !== "direct") expect(spans.filter(span => span.name === "response.resolve")) + .toHaveLength(lane === "discovery and worker" ? 3 : 2); + expect(JSON.stringify(spans.map(span => span.attributes))).not.toContain(f.reply); + const comments = await db.select().from(issueComments).where(eq(issueComments.createdByRunId, f.run.id)); + expect(comments).toHaveLength(1); + expect(comments[0].body).toBe(f.reply); + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", ""); + expect(await repairCommittedNativeChatResponse(db, { companyId: f.companyId, issueId: f.issueId, runId: f.run.id })).toBe(false); + expect(exporter.getFinishedSpans()).toHaveLength(spans.length); + } finally { + vi.unstubAllEnvs(); + await provider.shutdown(); + trace.disable(); + } + }); + it.each([null, [], "scalar", 42, { source: 42, skipIssueComment: "true", paperclipWake: [] }, { source: "issue.comment.created", externalChatContinuation: true }, { source: "chat:fixture", skipIssueComment: true, paperclipWake: { externalInteractionContinuation: "true" } }, diff --git a/server/src/services/provider-trace-cancellation.integration.test.ts b/server/src/services/provider-trace-cancellation.integration.test.ts new file mode 100644 index 00000000000..2c8b3919580 --- /dev/null +++ b/server/src/services/provider-trace-cancellation.integration.test.ts @@ -0,0 +1,143 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { createServer, request as forward } from "node:http"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { + S3Client, + CreateBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, + DeleteBucketCommand, +} from "@aws-sdk/client-s3"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createProviderTraceObjectStore } from "./provider-trace-object-store.js"; +import { idleWorkSnapshot } from "./task-admission.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +afterEach(() => vi.unstubAllEnvs()); +describe.skipIf(!endpoint)("trace migration cancellation through the real S3 SDK", () => { + it.each(["open", "probe", "upload", "restore", "read"])( + "cancels a stalled %s and releases its capture", + async (stage) => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + const bucket = `pc-trace-abort-${randomUUID()}`; + const directory = await fs.mkdtemp(path.join(tmpdir(), "trace-abort-")); + const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + const ref = `s3-segments-v1:company/run/${randomUUID()}.ndjson`; + const controller = new AbortController(); + let enabled = true, + closed = false, + started!: () => void; + const stalled = new Promise((resolve) => { + started = resolve; + }); + const baseline = idleWorkSnapshot().active; + const proxy = createServer((request, response) => { + const url = request.url!; + const matches = + stage === "open" + ? request.method === "GET" && url.includes("head.json") + : stage === "probe" + ? request.method === "PUT" && url.includes("conditional-probe-") + : stage === "upload" + ? request.method === "PUT" && url.includes("/objects/") + : request.method === "GET" && url.includes("/objects/"); + if (enabled && matches) { + response.on("close", () => { + closed = true; + }); + started(); + return; + } + const upstream = forward( + new URL(url, endpoint), + { method: request.method, headers: request.headers }, + (result) => { + response.writeHead(result.statusCode!, result.headers); + result.pipe(response); + }, + ); + response.on("close", () => upstream.destroy()); + upstream.on("error", (error) => response.destroy(error)); + request.pipe(upstream); + }); + await new Promise((resolve) => proxy.listen(0, "127.0.0.1", resolve)); + const address = proxy.address(); + if (!address || typeof address === "string") throw new Error("proxy did not bind"); + const store = createProviderTraceObjectStore( + createS3StorageProvider({ + endpoint: `http://127.0.0.1:${address.port}`, + bucket, + region: "us-east-1", + forcePathStyle: true, + }), + ); + let operation: Promise | undefined; + try { + const file = path.join(directory, "import.ndjson"); + const body = Buffer.from('{"kind":"frame","rawBase64":"e30="}\n'); + if (stage === "read" || stage === "restore") { + const seed = createProviderTraceObjectStore( + createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }), + ); + const original = path.join(directory, "original.ndjson"); + await fs.writeFile(original, body); + await seed.open(ref, original, async () => undefined, { restore: false }); + await seed.finish(ref); + } + if (stage === "upload") { + await fs.writeFile(file, body); + await store.open(ref, file, async () => undefined, { + restore: false, + signal: controller.signal, + }); + operation = store.finish(ref); + } else if (stage === "read") operation = store.read(ref, controller.signal); + else + operation = store.open(ref, file, async () => undefined, { + restore: stage === "restore", + signal: controller.signal, + }); + const result = operation.then( + () => "resolved", + () => "aborted", + ); + await stalled; + controller.abort(); + expect(await Promise.race([result, delay(500).then(() => "still running")])).toBe( + "aborted", + ); + await vi.waitFor(() => expect(closed).toBe(true)); + expect(idleWorkSnapshot().active).toBe(baseline); + if (stage === "upload") expect(await fs.readFile(file)).toEqual(body); + } finally { + enabled = false; + controller.abort(); + proxy.closeAllConnections(); + await operation?.catch(() => undefined); + await store.stopRun("company", "run"); + await new Promise((resolve) => proxy.close(() => resolve())); + const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (objects.Contents?.length) + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: objects.Contents.map((object) => ({ Key: object.Key! })) }, + }), + ); + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + await fs.rm(directory, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/server/src/services/provider-trace-object-store.test.ts b/server/src/services/provider-trace-object-store.test.ts new file mode 100644 index 00000000000..955252dcbc5 --- /dev/null +++ b/server/src/services/provider-trace-object-store.test.ts @@ -0,0 +1,95 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createProviderTraceObjectStore } from "./provider-trace-object-store.js"; +import { + S3Client, + CreateBucketCommand, + ListObjectsV2Command, + GetObjectCommand, + DeleteObjectsCommand, + DeleteBucketCommand, +} from "@aws-sdk/client-s3"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +afterEach(() => vi.unstubAllEnvs()); +describe.skipIf(!endpoint)("encrypted object provider traces (S3 integration)", () => { + it.each(["environment", "existing key file"])( + "restores both sidecars on a new host, preserves exact downloads, and removes ciphertext (%s)", + async (source) => { + const bucket = `paperclip-trace-test-${randomUUID()}`; + const client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + const directory = await fs.mkdtemp(path.join(tmpdir(), "object-trace-")); + const keyFile = path.join(directory, "existing-vault.key"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", source === "environment" ? "a".repeat(64) : ""); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY_FILE", keyFile); + if (source === "existing key file") + await fs.writeFile(keyFile, "a".repeat(64), { mode: 0o600 }); + const filename = `${randomUUID()}.ndjson`, + ref = `s3-segments-v1:company/run/${filename}`; + const bytes = Buffer.from('{"kind":"frame","rawBase64":"cHJpdmF0ZQ=="}\n'); + const interpretation = Buffer.from('{"kind":"interpretation","frameId":1}\n'); + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + try { + const a = createProviderTraceObjectStore(provider), + fileA = path.join(directory, "a.ndjson"); + await a.open(ref, fileA, async () => undefined); + await fs.appendFile(fileA, bytes); + await fs.appendFile(`${fileA}.rehydration`, interpretation); + await a.finish(ref); + const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + for (const object of objects.Contents ?? []) { + if (object.Key?.endsWith("head.json")) continue; + const stored = await client.send( + new GetObjectCommand({ Bucket: bucket, Key: object.Key }), + ); + const payload = await stored.Body!.transformToByteArray(); + expect(Buffer.from(payload).includes(Buffer.from("rawBase64"))).toBe(false); + expect(Buffer.from(payload).includes(Buffer.from("interpretation"))).toBe(false); + } + await fs.rm(fileA); + await fs.rm(`${fileA}.rehydration`); + const b = createProviderTraceObjectStore(provider), + fileB = path.join(directory, "b.ndjson"); + expect(await b.read(ref)).toEqual(Buffer.concat([bytes, interpretation])); + await fs.mkdir(`${fileB}.rehydration`); + await expect(b.open(ref, fileB, async () => undefined)).rejects.toThrow(); + expect(await b.settled(ref)).toBe(true); + await expect(fs.stat(fileB)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await b.read(ref)).toEqual(Buffer.concat([bytes, interpretation])); + await fs.rmdir(`${fileB}.rehydration`); + await b.open(ref, fileB, async () => undefined); + expect(await fs.readFile(fileB)).toEqual(bytes); + expect(await fs.readFile(`${fileB}.rehydration`)).toEqual(interpretation); + await fs.appendFile(fileB, Buffer.from('{"kind":"trace_status","status":"complete"}\n')); + await b.finish(ref); + await b.remove(ref); + await expect(b.read(ref)).rejects.toThrow("unavailable"); + const left = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + expect(left.Contents?.every((o) => o.Key!.endsWith("head.json"))).toBe(true); + } finally { + const objects = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (objects.Contents?.length) + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: objects.Contents.map((o) => ({ Key: o.Key! })) }, + }), + ); + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + await fs.rm(directory, { recursive: true, force: true }); + client.destroy(); + } + }, + 30_000, + ); +}); diff --git a/server/src/services/provider-trace-object-store.ts b/server/src/services/provider-trace-object-store.ts new file mode 100644 index 00000000000..f122a5938eb --- /dev/null +++ b/server/src/services/provider-trace-object-store.ts @@ -0,0 +1,293 @@ +import { createCipheriv, createDecipheriv, randomBytes } from "node:crypto"; +import { beginIdleTrackedWork } from "./task-admission.js"; +import { watch, type FSWatcher } from "node:fs"; +import { promises as fs } from "node:fs"; +import { + createObjectByteJournal, + type ObjectByteJournal, + type ObjectJournalWriter, +} from "./object-byte-journal.js"; +import { deriveLocalEncryptedInstanceSecret } from "../secrets/local-encrypted-provider.js"; +import type { StorageProvider } from "../storage/types.js"; + +const CHUNK_BYTES = 1024 * 1024; +const MAX_CAPTURE_BYTES = 192 * CHUNK_BYTES; +const REF = /^s3-segments-v1:([a-zA-Z0-9-]+)\/([a-zA-Z0-9-]+)\/([a-f0-9-]+\.ndjson)$/; +export function objectProviderTraceRef(ref: string) { + return REF.exec(ref); +} +export function importingProviderTraceRef(ref: string) { + return ref.startsWith("s3-import-v1:") + ? objectProviderTraceRef(ref.replace("s3-import-v1:", "s3-segments-v1:")) + : null; +} +export function providerTraceCleanupRef(ref: string) { + return objectProviderTraceRef(ref) + ? ref + : importingProviderTraceRef(ref) + ? ref.replace("s3-import-v1:", "s3-segments-v1:") + : null; +} +interface Channel { + path: string; + journal: ObjectByteJournal; + writer: ObjectJournalWriter; + watcher?: FSWatcher; +} +interface Capture { + channels: Channel[]; + controller: AbortController; + signal: AbortSignal; + flush?: Promise; + timer?: NodeJS.Timeout; + stopped: boolean; + dirty: boolean; + error?: unknown; + finishIdle: () => void; +} + +export function createProviderTraceObjectStore( + provider: StorageProvider, + prefix = "provider-traces", +) { + deriveLocalEncryptedInstanceSecret("provider-trace:existing-key", process.env, { + allowCreate: false, + }); + const captures = new Map(); + const openings = new Map>(); + function schedule(capture: Capture) { + if (capture.stopped || !capture.dirty || capture.timer || capture.flush || capture.error) + return; + capture.timer = setTimeout(() => { + capture.timer = undefined; + void sync(capture).catch(() => undefined); + }, 1000); + capture.timer.unref(); + } + function channelsFor(ref: string, filePath = "") { + const match = objectProviderTraceRef(ref); + if (!match) throw new Error("invalid_object_trace_ref"); + return ["native", "rehydration"].map((channel, i) => { + const root = `${prefix}/segments-v1/${match[1]}/${match[2]}/${match[3]}/${channel}`; + const key = Buffer.from( + deriveLocalEncryptedInstanceSecret(`provider-trace:${root}`, process.env, { + allowCreate: false, + }), + "base64url", + ); + const aad = Buffer.from(root); + const journal = createObjectByteJournal({ + provider, + prefix: root, + async encode(bytes) { + const iv = randomBytes(12), + cipher = createCipheriv("aes-256-gcm", key, iv); + cipher.setAAD(aad); + const body = Buffer.concat([cipher.update(bytes), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), body]); + }, + async decode(bytes) { + if (bytes.length < 28) throw new Error("invalid_encrypted_trace_object"); + const cipher = createDecipheriv("aes-256-gcm", key, bytes.subarray(0, 12)); + cipher.setAAD(aad); + cipher.setAuthTag(bytes.subarray(12, 28)); + return Buffer.concat([cipher.update(bytes.subarray(28)), cipher.final()]); + }, + }); + return { journal, path: i === 0 ? filePath : `${filePath}.rehydration` }; + }); + } + async function sync(capture: Capture) { + if (capture.timer) { + clearTimeout(capture.timer); + capture.timer = undefined; + } + if (capture.flush) return capture.flush; + capture.dirty = false; + capture.flush = (async () => { + for (const channel of capture.channels) { + capture.signal.throwIfAborted(); + const file = await fs.open(channel.path, "r"); + try { + const stat = await file.stat(); + if (stat.size > MAX_CAPTURE_BYTES || stat.size < channel.writer.head.end) + throw new Error("provider_trace_capture_size_invalid"); + let offset = channel.writer.head.end; + while (offset < stat.size) { + const buffer = Buffer.alloc(Math.min(CHUNK_BYTES, stat.size - offset)); + const { bytesRead } = await file.read(buffer, 0, buffer.length, offset); + if (bytesRead !== buffer.length) throw new Error("provider_trace_capture_changed"); + channel.writer = await channel.journal.append(channel.writer, buffer, capture.signal); + offset += buffer.length; + } + } finally { + await file.close(); + } + } + })() + .catch((error) => { + capture.error = error; + throw error; + }) + .finally(() => { + capture.flush = undefined; + schedule(capture); + }); + return capture.flush; + } + async function open( + ref: string, + filePath: string, + assertAuthority: () => Promise, + options: { restore?: boolean; signal?: AbortSignal } = {}, + ) { + const existing = openings.get(ref); + if (existing) return existing; + const opening = openCapture(ref, filePath, assertAuthority, options).finally(() => + openings.delete(ref), + ); + openings.set(ref, opening); + await opening; + } + async function openCapture( + ref: string, + filePath: string, + assertAuthority: () => Promise, + options: { restore?: boolean; signal?: AbortSignal } = {}, + ) { + if (captures.has(ref)) return; + const controller = new AbortController(); + const capture: Capture = { + channels: [], + controller, + signal: options.signal + ? AbortSignal.any([controller.signal, options.signal]) + : controller.signal, + stopped: false, + dirty: false, + finishIdle: () => undefined, + }; + try { + for (const source of channelsFor(ref, filePath)) { + capture.signal.throwIfAborted(); + const current = await source.journal.head(capture.signal); + const writer = await source.journal.claim({ + expectedOwner: current?.head.owner ?? null, + assertAuthority, + signal: capture.signal, + }); + capture.channels.push({ ...source, writer }); + const file = await fs.open(source.path, options.restore === false ? "a+" : "w", 0o600); + try { + await file.chmod(0o600); + for ( + let start = 0; + options.restore !== false && start < writer.head.end; + start += CHUNK_BYTES + ) { + await file.write( + await source.journal.readRange(writer.head, start, CHUNK_BYTES, capture.signal), + ); + } + } finally { + await file.close(); + } + } + capture.signal.throwIfAborted(); + capture.finishIdle = beginIdleTrackedWork(); + captures.set(ref, capture); + for (const channel of capture.channels) { + channel.watcher = watch(channel.path, () => { + capture.dirty = true; + schedule(capture); + }); + channel.watcher.on("error", (error) => { + capture.error = error; + }); + channel.watcher.unref(); + } + } catch (error) { + capture.stopped = true; + for (const channel of capture.channels) channel.watcher?.close(); + await Promise.allSettled( + capture.channels.map(async (channel) => { + await channel.journal.close(channel.writer, capture.signal); + }), + ); + if (options.restore !== false) + await Promise.allSettled(capture.channels.map((channel) => fs.unlink(channel.path))); + captures.delete(ref); + capture.finishIdle(); + throw error; + } + } + async function finish(ref: string) { + const capture = captures.get(ref); + if (!capture) return; + capture.stopped = true; + for (const channel of capture.channels) channel.watcher?.close(); + try { + await capture.flush; + if (capture.error) throw capture.error; + await sync(capture); + for (const channel of capture.channels) + channel.writer = await channel.journal.close(channel.writer, capture.signal); + } finally { + if (capture.timer) clearTimeout(capture.timer); + captures.delete(ref); + capture.finishIdle(); + } + } + async function* readChunks(ref: string, signal?: AbortSignal) { + let nativeTail: number | undefined; + for (const [index, { journal }] of channelsFor(ref).entries()) { + const snapshot = await journal.head(signal); + if (!snapshot || snapshot.head.deleted) throw new Error("provider_trace_payload_unavailable"); + if (snapshot.head.end > MAX_CAPTURE_BYTES) + throw new Error("provider_trace_capture_size_invalid"); + if (index === 1 && snapshot.head.end && nativeTail !== undefined && nativeTail !== 10) + yield Buffer.from("\n"); + for (let offset = 0; offset < snapshot.head.end; offset += CHUNK_BYTES) { + const bytes = await journal.readRange(snapshot.head, offset, CHUNK_BYTES, signal); + if (index === 0 && bytes.length) nativeTail = bytes.at(-1); + yield bytes; + } + } + } + async function read(ref: string, signal?: AbortSignal) { + const chunks: Buffer[] = []; + for await (const bytes of readChunks(ref, signal)) chunks.push(bytes); + return Buffer.concat(chunks); + } + async function stop(ref: string) { + const capture = captures.get(ref); + if (!capture) return; + capture.stopped = true; + capture.controller.abort(new Error("provider_trace_capture_stopped")); + if (capture.timer) clearTimeout(capture.timer); + for (const channel of capture.channels) channel.watcher?.close(); + await capture.flush?.catch(() => undefined); + captures.delete(ref); + capture.finishIdle(); + } + async function stopRun(companyId: string, runId: string) { + const scope = `s3-segments-v1:${companyId}/${runId}/`; + await Promise.all([...captures.keys()].filter((ref) => ref.startsWith(scope)).map(stop)); + } + async function remove(ref: string, signal?: AbortSignal) { + await stop(ref); + const results = await Promise.allSettled( + channelsFor(ref).map(({ journal }) => journal.remove(signal)), + ); + for (const result of results) if (result.status === "rejected") throw result.reason; + } + async function settled(ref: string) { + for (const { journal } of channelsFor(ref)) { + const snapshot = await journal.head(); + if (!snapshot || !snapshot.head.closed || snapshot.head.deleted) return false; + } + return true; + } + return { open, finish, read, readChunks, remove, settled, stopRun }; +} +export type ProviderTraceObjectStore = ReturnType; diff --git a/server/src/services/provider-trace-retention-work.ts b/server/src/services/provider-trace-retention-work.ts new file mode 100644 index 00000000000..7aa20d65f5f --- /dev/null +++ b/server/src/services/provider-trace-retention-work.ts @@ -0,0 +1,39 @@ +import type { Db } from "@paperclipai/db"; +import { providerTraceStore } from "./provider-trace-store.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { registerIdleLocalWorkProbe } from "./idle-local-work.js"; + +export function registerProviderTraceRetentionWork( + db: Db, + coordinator: ReturnType, + canRun: () => boolean, +) { + const traces = providerTraceStore(db); + let deadline: number | null | undefined; + + const unregister = registerIdleLocalWorkProbe(() => + deadline === undefined ? "unknown" : deadline === null ? "none" : "present", + ); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.providerTraceRetention, { + canRun, + retryMs: 30_000, + onSettled: () => { + deadline = undefined; + }, + run: (signal) => + signal.aborted ? Promise.resolve() : traces.cleanupExpired(new Date(), signal).then(() => undefined), + next: async () => { + deadline = await traces.nextRetentionDeadline(); + return deadline; + }, + }); + return { + ...worker, + stop: async () => { + unregister(); + await worker.stop(); + }, + }; +} diff --git a/server/src/services/provider-trace-retention.integration.test.ts b/server/src/services/provider-trace-retention.integration.test.ts new file mode 100644 index 00000000000..800da92ca12 --- /dev/null +++ b/server/src/services/provider-trace-retention.integration.test.ts @@ -0,0 +1,280 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { sql } from "drizzle-orm"; +import { createDb, providerTraceRecords, type Db } from "@paperclipai/db"; +import { + S3Client, + CreateBucketCommand, + DeleteBucketCommand, + DeleteObjectsCommand, + ListObjectsV2Command, +} from "@aws-sdk/client-s3"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { providerTraceStore } from "./provider-trace-store.js"; +import { idleWorkSnapshot } from "./task-admission.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; + +vi.mock("./activity-log.js", () => ({ logActivity: vi.fn(async () => undefined) })); +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL"); +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +describe.skipIf(!databaseUrl)("provider trace retention and capture ownership", () => { + let db: Db, directory: string, client: S3Client | undefined; + const schema = `trace_retention_${randomUUID().replaceAll("-", "")}`; + const bucket = `pc-trace-retention-${randomUUID()}`; + let bucketCreated = false; + const owners: Array<{ + store: ReturnType; + run: string; + company: string; + }> = []; + beforeAll(async () => { + db = createDb(databaseUrl!, { maxConnections: 1 }); + await db.execute(sql.raw(`create schema ${schema}`)); + await db.execute(sql.raw(`set search_path=${schema}`)); + await db.execute(sql`create table provider_trace_records(id uuid primary key default gen_random_uuid(), company_id uuid not null, run_id uuid not null unique, + status text not null, provider text not null, trace_ref text not null, frame_count int not null default 0, + byte_count bigint not null default 0, digest text, reason text, requested_by text not null, expires_at timestamptz not null, + deleted_at timestamptz, created_at timestamptz not null default now(), updated_at timestamptz not null default now())`); + if (endpoint) { + client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + bucketCreated = true; + } + }); + beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "trace-retention-")); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(directory, "config.json")); + vi.stubEnv("PROVIDER_TRACE_BASE_PATH", directory); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "e".repeat(64)); + await db.execute(sql`truncate provider_trace_records`); + }); + afterEach(async () => { + for (const { store, run, company } of owners.splice(0)) + await store.finalize(run, company).catch(() => undefined); + vi.useRealTimers(); + vi.unstubAllEnvs(); + await fs.rm(directory, { recursive: true, force: true }); + }); + afterAll(async () => { + if (client && bucketCreated) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + } + if (db) { + await db.execute(sql.raw(`drop schema ${schema} cascade`)); + await db.$client.end(); + } + }); + function external() { + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", endpoint!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_PREFIX", ""); + } + it("schedules expiry when a new local trace is created after startup becomes quiet", async () => { + const store = providerTraceStore(db), + companyId = randomUUID(), + runId = randomUUID(); + expect(await store.nextRetentionDeadline()).toBeNull(); + const capture = await store.prepare({ + companyId, + runId, + provider: "codex", + requestedBy: "test", + }); + expect(await store.nextRetentionDeadline()).toBe( + new Date(capture.metadata.expiresAt).getTime(), + ); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", "http://127.0.0.1:1"); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", undefined); + const replacement = providerTraceStore(db); + expect(await replacement.getByRun(runId, companyId)).not.toBeNull(); + await replacement.remove(runId, companyId); + expect(await store.nextRetentionDeadline()).toBeNull(); + }); + it("drains more than one page of expired local files without requiring another producer", async () => { + const companyId = randomUUID(), + expiresAt = new Date(Date.now() - 60_000); + const rows = Array.from({ length: 201 }, () => ({ + companyId, + runId: randomUUID(), + status: "complete", + provider: "codex", + traceRef: `${randomUUID()}.ndjson`, + requestedBy: "test", + expiresAt, + })); + for (const row of rows) await fs.writeFile(path.join(directory, row.traceRef), "{}\n"); + await db.insert(providerTraceRecords).values(rows); + const store = providerTraceStore(db); + expect(await store.nextRetentionDeadline()).toBeLessThanOrEqual(Date.now()); + expect(await store.cleanupExpired()).toHaveLength(100); + expect(await store.nextRetentionDeadline()).not.toBeNull(); + expect(await store.cleanupExpired()).toHaveLength(100); + expect(await store.nextRetentionDeadline()).not.toBeNull(); + expect(await store.cleanupExpired()).toHaveLength(1); + expect(await store.nextRetentionDeadline()).toBeNull(); + expect(await fs.readdir(directory)).toEqual([]); + }); + it("retains an expired local cleanup deadline after filesystem deletion fails", async () => { + const companyId = randomUUID(), + runId = randomUUID(), + traceRef = `${randomUUID()}.ndjson`; + const file = path.join(directory, traceRef); + await fs.mkdir(file); + await db + .insert(providerTraceRecords) + .values({ + companyId, + runId, + traceRef, + status: "complete", + provider: "codex", + requestedBy: "test", + expiresAt: new Date(Date.now() - 60_000), + }); + const store = providerTraceStore(db); + await expect(store.cleanupExpired()).rejects.toThrow(); + expect(await store.nextRetentionDeadline()).toBeLessThanOrEqual(Date.now()); + await fs.rmdir(file); + await fs.writeFile(file, "{}\n"); + await store.cleanupExpired(); + expect(await store.nextRetentionDeadline()).toBeNull(); + }); + it.skipIf(!endpoint)( + "shares capture ownership across routes and workers and releases deleted captures", + async () => { + external(); + const company = randomUUID(), + run = randomUUID(), + baseline = idleWorkSnapshot().active; + const heartbeat = providerTraceStore(db), + route = providerTraceStore(db); + owners.push({ store: heartbeat, run, company }); + await heartbeat.prepare({ + companyId: company, + runId: run, + provider: "codex", + requestedBy: "test", + assertWriterAuthority: async () => undefined, + }); + expect(idleWorkSnapshot().active).toBe(baseline + 1); + const other = randomUUID(); + await db.insert(providerTraceRecords).values({ + companyId: company, + runId: other, + status: "complete", + provider: "codex", + traceRef: `s3-segments-v1:${company}/${other}/${randomUUID()}.ndjson`, + requestedBy: "test", + expiresAt: new Date(Date.now() + 3600_000), + }); + await expect(route.remove(run, company)).rejects.toThrow("cleanup_pending"); + expect(await route.nextRetentionDeadline()).toBeLessThanOrEqual(Date.now()); + await route.remove(other, company); + expect(idleWorkSnapshot().active).toBe(baseline); + expect(await heartbeat.finalize(run, company)).toBeNull(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.now() + 31_000); + await route.remove(run, company); + expect(await route.nextRetentionDeadline()).toBeNull(); + expect(await fs.readdir(directory)).toEqual([]); + }, + ); + it.skipIf(!endpoint)( + "keeps an interrupted import readable locally and fences its staged remote prefix on deletion", + async () => { + external(); + const companyId = randomUUID(), + runId = randomUUID(), + filename = `${randomUUID()}.ndjson`; + const traceRef = `s3-import-v1:${companyId}/${runId}/${filename}`; + const bytes = Buffer.from('{"kind":"frame","rawBase64":"e30="}\n'); + await fs.writeFile(path.join(directory, filename), bytes); + await db.insert(providerTraceRecords).values({ + companyId, + runId, + traceRef, + status: "complete", + provider: "codex", + requestedBy: "test", + expiresAt: new Date(Date.now() + 3600_000), + }); + const store = providerTraceStore(db); + expect((await store.download(runId, companyId))!.bytes).toEqual(bytes); + await store.remove(runId, companyId); + expect(await store.download(runId, companyId)).toBeNull(); + expect(await store.nextRetentionDeadline()).toBeNull(); + expect(await fs.readdir(directory)).toEqual([]); + const objects = await client!.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `provider-traces/segments-v1/${companyId}/${runId}/${filename}/`, + }), + ); + expect(objects.Contents).toHaveLength(2); + expect(objects.Contents!.every((object) => object.Key!.endsWith("head.json"))).toBe(true); + }, + ); + it.skipIf(!endpoint)( + "does not resurrect a trace deleted before its first journal claim", + async () => { + external(); + const company = randomUUID(), + run = randomUUID(), + baseline = idleWorkSnapshot().active; + const heartbeat = providerTraceStore(db), + route = providerTraceStore(db); + owners.push({ store: heartbeat, run, company }); + let enter!: () => void, release!: () => void; + const entered = new Promise((resolve) => { + enter = resolve; + }); + const blocked = new Promise((resolve) => { + release = resolve; + }); + const preparing = heartbeat.prepare({ + companyId: company, + runId: run, + provider: "codex", + requestedBy: "test", + assertWriterAuthority: async () => { + enter(); + await blocked; + }, + }); + await entered; + await route.remove(run, company); + release(); + await expect(preparing).rejects.toThrow(); + expect(idleWorkSnapshot().active).toBe(baseline); + expect(await heartbeat.finalize(run, company)).toBeNull(); + expect(await route.download(run, company)).toBeNull(); + const objects = await client!.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: `provider-traces/segments-v1/${company}/${run}/`, + }), + ); + expect(objects.Contents).toHaveLength(2); + for (const object of objects.Contents!) expect(object.Key).toMatch(/head\.json$/); + }, + ); +}); diff --git a/server/src/services/provider-trace-store.ts b/server/src/services/provider-trace-store.ts index eecda608bc5..be89478ba3a 100644 --- a/server/src/services/provider-trace-store.ts +++ b/server/src/services/provider-trace-store.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; -import { and, eq, gt, inArray, isNull, lte, ne, or } from "drizzle-orm"; +import { and, asc, eq, gt, inArray, isNull, lte, ne, or, sql } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; import { providerTraceRecords } from "@paperclipai/db"; import type { @@ -9,6 +9,10 @@ import type { ProviderTraceMetadata, } from "@paperclipai/shared"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { createProviderTraceObjectStore, objectProviderTraceRef, importingProviderTraceRef, providerTraceCleanupRef, type ProviderTraceObjectStore } from "./provider-trace-object-store.js"; +import { resolveRunEvidenceStorage } from "./run-evidence-storage.js"; +import { loadStorageConfig } from "../storage/config.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; import { logActivity } from "./activity-log.js"; export const PROVIDER_TRACE_MAX_BYTES = 64 * 1024 * 1024; @@ -31,6 +35,8 @@ function traceRoot() { } function tracePath(traceRef: string) { + const external = objectProviderTraceRef(traceRef) ?? importingProviderTraceRef(traceRef); + if (external) traceRef = external[3]!; if ( path.basename(traceRef) !== traceRef || !/^[a-f0-9-]+\.ndjson$/.test(traceRef) @@ -150,24 +156,6 @@ async function readTraceBytes(traceRef: string) { return Buffer.concat([nativeBytes, separator, rehydrationBytes]); } -async function readEntries(traceRef: string) { - const content = (await readTraceBytes(traceRef)).toString("utf8"); - return content.split("\n").flatMap((line) => { - if (!line.trim()) return []; - try { - return [record(JSON.parse(line))]; - } catch { - return [ - { - kind: "trace_status", - status: "incomplete", - reason: "invalid_sidecar_record", - }, - ]; - } - }); -} - export function assessProviderTraceEntries( entries: Record[], options: { @@ -266,8 +254,34 @@ export function assessProviderTraceEntries( return { status: "complete", reason: null }; } -export function providerTraceStore(db: Db) { - async function removeTraceFiles(traceRef: string) { +const traceObjectStores = new WeakMap(); + +export function providerTraceStore(db: Db, options: { objectStore?: ProviderTraceObjectStore } = {}) { + let external = options.objectStore; + function objectStore() { + if (external) return external; + const shared = traceObjectStores.get(db); + if (shared) return external = shared; + const config = resolveRunEvidenceStorage(); + if (!config) throw new Error("External provider traces require instance S3 storage"); + external = createProviderTraceObjectStore(config.provider, config.tracePrefix); + traceObjectStores.set(db, external); + return external; + } + async function readBytes(traceRef: string) { + return objectProviderTraceRef(traceRef) ? objectStore().read(traceRef) : readTraceBytes(traceRef); + } + async function entriesFor(traceRef: string) { + return (await readBytes(traceRef)).toString("utf8").split("\n").flatMap(line => { + if (!line.trim()) return []; + try { return [record(JSON.parse(line))]; } + catch { return [{ kind: "trace_status", status: "incomplete", reason: "invalid_sidecar_record" }]; } + }); + } + async function removeTraceFiles(traceRef: string, signal?: AbortSignal) { + signal?.throwIfAborted(); + const cleanupRef = providerTraceCleanupRef(traceRef); + if (cleanupRef) await objectStore().remove(cleanupRef, signal); for (const filePath of [tracePath(traceRef), rehydrationTracePath(traceRef)]) { try { await fs.unlink(filePath); @@ -277,7 +291,7 @@ export function providerTraceStore(db: Db) { } } - async function expireRow(row: TraceRow, now: Date) { + async function expireRow(row: TraceRow, now: Date, signal?: AbortSignal) { let expired = row; let newlyExpired = false; if (!row.deletedAt) { @@ -300,7 +314,11 @@ export function providerTraceStore(db: Db) { expired = updated; newlyExpired = true; } else if (row.expiresAt.getTime() > now.getTime()) { - return null; + if (row.status !== "deleted" || !providerTraceCleanupRef(row.traceRef)) return null; + await removeTraceFiles(row.traceRef, signal); + const [cleaned] = await db.update(providerTraceRecords).set({ traceRef: EXPIRED_PROVIDER_TRACE_REF, updatedAt: now }) + .where(and(eq(providerTraceRecords.id, row.id), eq(providerTraceRecords.traceRef, row.traceRef))).returning(); + return cleaned ?? row; } // The database tombstone is authoritative before filesystem cleanup, so @@ -308,7 +326,7 @@ export function providerTraceStore(db: Db) { // marker only after both raw sidecars are absent; failures are retried by // later reads and cleanup passes. if (expired.traceRef !== EXPIRED_PROVIDER_TRACE_REF) { - await removeTraceFiles(expired.traceRef); + await removeTraceFiles(expired.traceRef, signal); await db .update(providerTraceRecords) .set({ @@ -342,24 +360,27 @@ export function providerTraceStore(db: Db) { return expired; } - async function cleanupExpired(now = new Date()) { + async function cleanupExpired(now = new Date(), signal?: AbortSignal) { const expired = await db .select() .from(providerTraceRecords) .where( and( - lte(providerTraceRecords.expiresAt, now), - or( - isNull(providerTraceRecords.deletedAt), - ne(providerTraceRecords.traceRef, EXPIRED_PROVIDER_TRACE_REF), - ), + or(lte(providerTraceRecords.expiresAt, now), and(eq(providerTraceRecords.status, "deleted"), + or(sql`${providerTraceRecords.traceRef} like 's3-segments-v1:%'`, sql`${providerTraceRecords.traceRef} like 's3-import-v1:%'`))), + or(isNull(providerTraceRecords.deletedAt), ne(providerTraceRecords.traceRef, EXPIRED_PROVIDER_TRACE_REF)), ), - ); + ).orderBy(asc(providerTraceRecords.expiresAt)).limit(100); const removed: TraceRow[] = []; + let failure: unknown; for (const row of expired) { - const removedRow = await expireRow(row, now); - if (removedRow) removed.push(removedRow); + try { + signal?.throwIfAborted(); + const removedRow = await expireRow(row, now, signal); + if (removedRow) removed.push(removedRow); + } catch (error) { failure ??= error; } } + if (failure) throw failure; return removed; } @@ -368,8 +389,8 @@ export function providerTraceStore(db: Db) { companyId: string; provider: string; requestedBy: string; + assertWriterAuthority?: () => Promise; }) { - await cleanupExpired().catch(() => []); const existing = await db .select() .from(providerTraceRecords) @@ -377,39 +398,47 @@ export function providerTraceStore(db: Db) { .limit(1) .then((rows) => rows[0] ?? null); if (existing) { + if (importingProviderTraceRef(existing.traceRef)) throw new Error("provider_trace_import_in_progress"); const now = new Date(); const expired = existing.expiresAt.getTime() <= now.getTime(); if (expired) await expireRow(existing, now); if (existing.deletedAt || expired) { throw new Error("provider_trace_unavailable"); } - return { - metadata: asMetadata(existing), - path: tracePath(existing.traceRef), - }; + if (objectProviderTraceRef(existing.traceRef)) { + if (!input.assertWriterAuthority) throw new Error("External trace capture requires admitted writer authority"); + await fs.mkdir(traceRoot(), { recursive: true, mode: 0o700 }); + await objectStore().open(existing.traceRef, tracePath(existing.traceRef), input.assertWriterAuthority); + } + return { metadata: asMetadata(existing), path: tracePath(existing.traceRef) }; } + const selected = loadStorageConfig().storageProvider === "s3" ? "s3_segments" : "local_file"; + if (selected === "s3_segments") { + if (!input.assertWriterAuthority) throw new Error("External trace capture requires admitted writer authority"); + objectStore(); + } await fs.mkdir(traceRoot(), { recursive: true, mode: 0o700 }); - const traceRef = `${randomUUID()}.ndjson`; + const filename = `${randomUUID()}.ndjson`; + const traceRef = selected === "s3_segments" ? `s3-segments-v1:${input.companyId}/${input.runId}/${filename}` : filename; const filePath = tracePath(traceRef); const handle = await fs.open(filePath, "wx", 0o600); await handle.close(); await fs.chmod(filePath, 0o600); const expiresAt = new Date(Date.now() + PROVIDER_TRACE_RETENTION_MS); - const row = await db - .insert(providerTraceRecords) - .values({ - runId: input.runId, - companyId: input.companyId, - status: "capturing", - provider: input.provider, - traceRef, - requestedBy: input.requestedBy, - expiresAt, - }) - .returning() - .then((rows) => rows[0]); + const values = { + runId: input.runId, companyId: input.companyId, status: "capturing", provider: input.provider, + traceRef, requestedBy: input.requestedBy, expiresAt, + }; + const row = await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.providerTraceRetention); + return (await tx.insert(providerTraceRecords).values(values).returning())[0]; + }); if (!row) throw new Error("provider_trace_metadata_create_failed"); + if (objectProviderTraceRef(traceRef)) { + if (!input.assertWriterAuthority) throw new Error("External trace capture requires admitted writer authority"); + await objectStore().open(traceRef, filePath, input.assertWriterAuthority); + } return { metadata: asMetadata(row), path: filePath }; } @@ -462,7 +491,11 @@ export function providerTraceStore(db: Db) { return rows.map(asMetadata); } - async function finalize( + async function finalize(runId: string, companyId: string, failureReason?: string | null) { + try { return await finalizeCapture(runId, companyId, failureReason); } + finally { await external?.stopRun(companyId, runId); } + } + async function finalizeCapture( runId: string, companyId: string, failureReason?: string | null, @@ -471,14 +504,18 @@ export function providerTraceStore(db: Db) { if (!row) return null; let bytes: Buffer; try { - bytes = await readTraceBytes(row.traceRef); + if (objectProviderTraceRef(row.traceRef)) { + await objectStore().finish(row.traceRef); + if (!await objectStore().settled(row.traceRef)) throw new Error("trace_capture_unsettled"); + } + bytes = await readBytes(row.traceRef); } catch { const updatedAt = new Date(); const finalized = await db .update(providerTraceRecords) .set({ status: "incomplete", - reason: failureReason ?? "trace_sidecar_missing", + reason: failureReason ?? (objectProviderTraceRef(row.traceRef) ? "trace_storage_incomplete" : "trace_sidecar_missing"), updatedAt, }) .where(and( @@ -538,7 +575,7 @@ export function providerTraceStore(db: Db) { async function inspect(runId: string, companyId: string) { const row = await getReadableByRun(runId, companyId); if (!row) return { trace: null, entries: [] }; - const entries = await readEntries(row.traceRef).catch(() => []); + const entries = await entriesFor(row.traceRef).catch(() => []); return { trace: asMetadata(row), entries: entries.map((entry) => @@ -550,7 +587,7 @@ export function providerTraceStore(db: Db) { async function readExactEntries(runId: string, companyId: string) { const row = await getReadableByRun(runId, companyId); if (!row) return null; - return readEntries(row.traceRef); + return entriesFor(row.traceRef); } async function revealFrame( @@ -560,7 +597,7 @@ export function providerTraceStore(db: Db) { ) { const row = await getReadableByRun(runId, companyId); if (!row) return null; - const entry = (await readEntries(row.traceRef)).find( + const entry = (await entriesFor(row.traceRef)).find( (candidate) => candidate.kind === "frame" && Number(candidate.frameId) === frameId, ); @@ -572,26 +609,38 @@ export function providerTraceStore(db: Db) { async function download(runId: string, companyId: string) { const row = await getReadableByRun(runId, companyId); if (!row) return null; - return { row, bytes: await readTraceBytes(row.traceRef) }; + return { row, bytes: await readBytes(row.traceRef) }; } async function remove(runId: string, companyId: string) { const row = await getByRun(runId, companyId); - if (!row || row.deletedAt) return null; - // Manual deletion is filesystem-first and strict: a reported success must - // never leave raw sidecars orphaned behind an inaccessible database row. + if (!row) return null; + if (providerTraceCleanupRef(row.traceRef)) { + const now = new Date(); + await db.transaction(async tx => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.providerTraceRetention); + await tx.update(providerTraceRecords).set({ status: "deleted", deletedAt: row.deletedAt ?? now, updatedAt: now }) + .where(and(eq(providerTraceRecords.id, row.id), eq(providerTraceRecords.traceRef, row.traceRef))); + }); + + await removeTraceFiles(row.traceRef); + return (await db.update(providerTraceRecords).set({ traceRef: EXPIRED_PROVIDER_TRACE_REF, updatedAt: new Date() }) + .where(and(eq(providerTraceRecords.id, row.id), eq(providerTraceRecords.traceRef, row.traceRef))).returning())[0] ?? null; + } + if (row.deletedAt) return null; await removeTraceFiles(row.traceRef); const now = new Date(); - return db - .update(providerTraceRecords) - .set({ - status: "deleted", - deletedAt: now, - updatedAt: now, - }) - .where(eq(providerTraceRecords.id, row.id)) - .returning() - .then((rows) => rows[0] ?? null); + return db.update(providerTraceRecords).set({ status: "deleted", deletedAt: now, updatedAt: now }) + .where(eq(providerTraceRecords.id, row.id)).returning().then(rows => rows[0] ?? null); + } + + async function nextRetentionDeadline() { + const [row] = await db.select({ at: providerTraceRecords.expiresAt, deletedAt: providerTraceRecords.deletedAt }) + .from(providerTraceRecords).where(and(ne(providerTraceRecords.traceRef, EXPIRED_PROVIDER_TRACE_REF), + or(isNull(providerTraceRecords.deletedAt), lte(providerTraceRecords.expiresAt, new Date()), + sql`${providerTraceRecords.traceRef} like 's3-segments-v1:%'`, sql`${providerTraceRecords.traceRef} like 's3-import-v1:%'`))) + .orderBy(asc(sql`case when ${providerTraceRecords.deletedAt} is not null then 0 else 1 end`), asc(providerTraceRecords.expiresAt)).limit(1); + return row ? row.deletedAt ? Date.now() : row.at.getTime() : null; } return { @@ -605,5 +654,6 @@ export function providerTraceStore(db: Db) { getByRun, listMetadataForRuns, cleanupExpired, + nextRetentionDeadline, }; } diff --git a/server/src/services/quota-credential-recovery.ts b/server/src/services/quota-credential-recovery.ts index 2158c760e93..990673528be 100644 --- a/server/src/services/quota-credential-recovery.ts +++ b/server/src/services/quota-credential-recovery.ts @@ -28,7 +28,9 @@ export const quotaCredentialHash = (value: string) => createHash("sha256").updat export const quotaCredentialRecoveryPath = (companyId: string, grantId: string) => path.join(idleQuotaRecoveryPath(), companyId, `${grantId}.json`); -/** Called only while holding the company credential mutation lock. The local +/** Legacy settlement and local-only Google retirement. Server quota refresh + * must never create a new record here. Called only while holding the company + * credential mutation lock. The local * encrypted provider uses the instance secrets master key, including when the * destination vault is remote. No plaintext token enters this journal. */ export function quotaCredentialRecovery(companyId: string, grantId: string) { diff --git a/server/src/services/run-evidence-migration-work.integration.test.ts b/server/src/services/run-evidence-migration-work.integration.test.ts new file mode 100644 index 00000000000..3ab1419de41 --- /dev/null +++ b/server/src/services/run-evidence-migration-work.integration.test.ts @@ -0,0 +1,147 @@ +import { randomUUID, createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { sql } from "drizzle-orm"; +import { createDb, type Db } from "@paperclipai/db"; +import { + S3Client, + CreateBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, + DeleteBucketCommand, +} from "@aws-sdk/client-s3"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { registerRunEvidenceMigrationWork } from "./run-evidence-migration-work.js"; +import { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { createObjectRunLogStore } from "./object-run-log-store.js"; +import { createProviderTraceObjectStore } from "./provider-trace-object-store.js"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL"); +describe.skipIf(!endpoint || !databaseUrl)( + "automatic retained history upgrade (Postgres and S3)", + () => { + let db: Db, client: S3Client, bucket: string, directory: string; + const schema = `automatic_migration_${randomUUID().replaceAll("-", "")}`; + beforeAll(async () => { + db = createDb(databaseUrl!, { maxConnections: 1 }); + await db.execute(sql.raw(`create schema ${schema}`)); + await db.execute(sql.raw(`set search_path=${schema}`)); + await db.execute(sql`create table heartbeat_runs (id uuid primary key, company_id uuid not null, agent_id uuid not null, + status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz)`); + await db.execute( + sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`, + ); + await db.execute(sql`create table heartbeat_run_events(company_id uuid,run_id uuid,seq bigint,payload jsonb, + source_instance_id text,source_event_id text,source_seq bigint,source_payload_sha256 text,event_type text)`); + await db.execute(sql`create table provider_trace_records (id uuid primary key, company_id uuid not null, run_id uuid not null, + status text not null, provider text not null, trace_ref text not null, frame_count int not null default 0, + byte_count bigint not null default 0, digest text, reason text, requested_by text not null, expires_at timestamptz not null, + deleted_at timestamptz, created_at timestamptz not null default now(), updated_at timestamptz not null default now())`); + directory = await fs.mkdtemp(path.join(os.tmpdir(), "evidence-migration-")); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(directory, "config.json")); + vi.stubEnv("RUN_LOG_BASE_PATH", path.join(directory, "logs")); + vi.stubEnv("PROVIDER_TRACE_BASE_PATH", path.join(directory, "traces")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "b".repeat(64)); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", endpoint!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_PREFIX", ""); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + bucket = `pc-auto-migration-${randomUUID()}`; + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + client = new S3Client({ + endpoint, + region: "us-east-1", + forcePathStyle: true, + }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + }, 30_000); + afterAll(async () => { + if (client) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((o) => ({ Key: o.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + } + if (db) { + await db.execute(sql.raw(`drop schema ${schema} cascade`)); + await db.$client.end(); + } + if (directory) await fs.rm(directory, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + it("copies retained logs and a trace automatically, then reads them without their source directories", async () => { + const companyId = randomUUID(), + agentId = randomUUID(), + runId = randomUUID(), + traceId = randomUUID(); + const logRef = `${companyId}/${agentId}/${runId}.ndjson`; + const filename = `${randomUUID()}.ndjson`; + const log = Buffer.from('{"stream":"stdout","chunk":"retained 再開","seq":1}\n'); + const trace = Buffer.from('{"kind":"frame","rawBase64":"e30="}\n'); + await fs.mkdir(path.dirname(path.join(directory, "logs", logRef)), { recursive: true }); + await fs.writeFile(path.join(directory, "logs", logRef), log); + await fs.mkdir(path.join(directory, "traces"), { recursive: true }); + await fs.writeFile(path.join(directory, "traces", filename), trace); + const expiry = new Date(Date.now() + 3600_000).toISOString(); + await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status,log_store,log_ref,log_bytes,log_sha256) + values(${runId},${companyId},${agentId},'succeeded','local_file',${logRef},${log.length},${createHash("sha256").update(log).digest("hex")})`); + await db.execute(sql`insert into provider_trace_records(id,company_id,run_id,status,provider,trace_ref,requested_by,expires_at,digest) + values(${traceId},${companyId},${runId},'complete','codex',${filename},'test',${expiry},${`sha256:${createHash("sha256").update(trace).digest("hex")}`})`); + const onError = vi.fn(); + const coordinator = createDeliveryWorkCoordinator({ owner: db, canRun: () => true, onError }); + const worker = registerRunEvidenceMigrationWork(db, coordinator, () => true)!; + try { + await worker.ready; + await vi.waitFor(() => expect(coordinator.nextWakeAt()).toBeNull(), { timeout: 5_000 }); + expect(onError).not.toHaveBeenCalled(); + await fs.rm(path.join(directory, "logs"), { recursive: true }); + await fs.rm(path.join(directory, "traces"), { recursive: true }); + const [run] = await db.$client.unsafe( + "select log_store,log_ref from heartbeat_runs where id=$1", + [runId], + ); + const [saved] = await db.$client.unsafe( + "select trace_ref,expires_at from provider_trace_records where id=$1", + [traceId], + ); + expect(run!.log_store).toBe("s3_segments"); + expect(new Date(saved!.expires_at).toISOString()).toBe(expiry); + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + const logs = createObjectRunLogStore({ + provider, + keyPrefix: "run-logs", + assertWriterAuthority: async () => { + throw Error("read-only"); + }, + }); + expect( + Buffer.from((await logs.read({ store: run!.log_store, logRef: run!.log_ref })).content), + ).toEqual(log); + expect(await createProviderTraceObjectStore(provider).read(saved!.trace_ref)).toEqual( + trace, + ); + } finally { + await worker.stop(); + await coordinator.stop(); + } + }); + }, +); diff --git a/server/src/services/run-evidence-migration-work.ts b/server/src/services/run-evidence-migration-work.ts new file mode 100644 index 00000000000..2ff3b1c8843 --- /dev/null +++ b/server/src/services/run-evidence-migration-work.ts @@ -0,0 +1,84 @@ +import type { Db } from "@paperclipai/db"; +import { loadStorageConfig, type StorageConfig } from "../storage/config.js"; +import { logger } from "../middleware/logger.js"; +import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; +import { registerDeadlineWork } from "./deadline-work.js"; +import { registerIdleLocalWorkProbe, type IdleLocalWork } from "./idle-local-work.js"; +import { migrateRunEvidence } from "./run-evidence-migration.js"; + +export function registerRunEvidenceMigrationWork( + db: Db, + coordinator: ReturnType, + canRun: () => boolean, + storage: Pick = loadStorageConfig(), +) { + if (storage.storageProvider !== "s3") return null; + let runCursor: string | undefined, traceCursor: string | undefined; + let logsDone = false, + tracesDone = false, + blocked = 0, + migrated = 0; + let retryMs = 5 * 60_000; + let next: number | null = Date.now(); + let pending: IdleLocalWork = "unknown"; + const unregister = registerIdleLocalWorkProbe(() => pending); + const worker = registerDeadlineWork(coordinator, DELIVERY_QUEUES.runEvidenceMigration, { + canRun, + retryMs: 30_000, + continuationMs: 1_000, + async run(signal) { + if (next === null) return; + const result = await migrateRunEvidence(db, { + apply: true, + limit: 1, + afterRunId: runCursor, + afterTraceId: traceCursor, + kinds: [...(logsDone ? [] : ["log" as const]), ...(tracesDone ? [] : ["trace" as const])], + signal, + }); + signal.throwIfAborted(); + for (const item of result.items) { + if (item.state === "blocked") { + blocked++; + logger.warn( + { kind: item.kind, id: item.id, reason: item.reason }, + "Run evidence migration deferred; source storage is still required", + ); + } else if (item.state === "migrated") migrated++; + } + if (!logsDone) { + runCursor = result.nextRunId; + logsDone = runCursor === undefined; + } + if (!tracesDone) { + traceCursor = result.nextTraceId; + tracesDone = traceCursor === undefined; + } + pending = "present"; + next = Date.now() + 1_000; + if (logsDone && tracesDone) { + logger.info( + { migrated, blocked }, + blocked + ? "Run evidence migration will retry unresolved history; keep source storage" + : "Run evidence migration complete", + ); + next = blocked ? Date.now() + retryMs : null; + pending = blocked ? "present" : "none"; + retryMs = Math.min(retryMs * 2, 30 * 60_000); + runCursor = traceCursor = undefined; + logsDone = tracesDone = false; + blocked = migrated = 0; + } + }, + next: async () => next, + }); + return { + ...worker, + stop: async () => { + await worker.stop(); + unregister(); + }, + }; +} diff --git a/server/src/services/run-evidence-migration.integration.test.ts b/server/src/services/run-evidence-migration.integration.test.ts new file mode 100644 index 00000000000..6af6a77dd15 --- /dev/null +++ b/server/src/services/run-evidence-migration.integration.test.ts @@ -0,0 +1,349 @@ +import { randomUUID, createHash } from "node:crypto"; +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { sql } from "drizzle-orm"; +import { createDb, type Db } from "@paperclipai/db"; +import { + S3Client, + CreateBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, + DeleteBucketCommand, +} from "@aws-sdk/client-s3"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { migrateRunEvidence } from "./run-evidence-migration.js"; +import { createS3StorageProvider } from "../storage/s3-provider.js"; +import { createObjectRunLogStore } from "./object-run-log-store.js"; +import { createProviderTraceObjectStore } from "./provider-trace-object-store.js"; +import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; +import { formatNativeRunLogEvent } from "./native-runtime/native-run-log-export.js"; +import { nativeSha256 } from "./native-runtime/canonical.js"; +import { createObjectByteJournal } from "./object-byte-journal.js"; +import type { PrpEvent } from "../vendor/paperclip-runner/index.js"; +import { compactRunLogChunk } from "./heartbeat/run-log.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +const databaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_RUN_EVIDENCE_MIGRATION_DATABASE_URL"); +describe.skipIf(!endpoint || !databaseUrl)("run evidence migration (Postgres and S3)", () => { + let db: Db, client: S3Client, bucket: string, directory: string; + const schema = `migration_${randomUUID().replaceAll("-", "")}`; + beforeAll(async () => { + db = createDb(databaseUrl!, { maxConnections: 1 }); + await db.execute(sql.raw(`create schema ${schema}`)); + await db.execute(sql.raw(`set search_path=${schema}`)); + await db.execute(sql`create table heartbeat_runs (id uuid primary key, company_id uuid not null, agent_id uuid not null, + status text not null, runtime_mode text default 'legacy', log_store text, log_ref text, log_bytes bigint, log_sha256 text, last_output_seq int not null default 0, controller_lease_expires_at timestamptz)`); + await db.execute( + sql`create table native_run_finalizations(run_id uuid, company_id uuid, lease_expires_at timestamptz)`, + ); + await db.execute(sql`create table heartbeat_run_events(company_id uuid,run_id uuid,seq bigint,payload jsonb, + source_instance_id text,source_event_id text,source_seq bigint,source_payload_sha256 text,event_type text)`); + await db.execute(sql`create table provider_trace_records (id uuid primary key, company_id uuid not null, run_id uuid not null, + status text not null, provider text not null, trace_ref text not null, frame_count int not null default 0, + byte_count bigint not null default 0, digest text, reason text, requested_by text not null, expires_at timestamptz not null, + deleted_at timestamptz, created_at timestamptz not null default now(), updated_at timestamptz not null default now())`); + directory = await fs.mkdtemp(path.join(os.tmpdir(), "evidence-migration-")); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(directory, "config.json")); + vi.stubEnv("RUN_LOG_BASE_PATH", path.join(directory, "logs")); + vi.stubEnv("PROVIDER_TRACE_BASE_PATH", path.join(directory, "traces")); + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "b".repeat(64)); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", endpoint!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_PREFIX", ""); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + bucket = `paperclip-migration-test-${randomUUID()}`; + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + client = new S3Client({ + endpoint, + region: "us-east-1", + forcePathStyle: true, + }); + await client.send(new CreateBucketCommand({ Bucket: bucket })); + }, 30_000); + afterAll(async () => { + if (client) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((o) => ({ Key: o.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + client.destroy(); + } + if (db) { + await db.execute(sql.raw(`drop schema ${schema} cascade`)); + await db.$client.end(); + } + if (directory) await fs.rm(directory, { recursive: true, force: true }); + vi.unstubAllEnvs(); + }); + it("preserves exact historical bytes and expiry, while refusing active/missing/corrupt sources", async () => { + const company = randomUUID(), + agent = randomUUID(), + run = randomUUID(), + trace = randomUUID(), + filename = `${randomUUID()}.ndjson`; + const log = Buffer.from( + Array.from( + { length: 100 }, + (_, i) => + JSON.stringify({ + ts: "then", + seq: i + 1, + stream: "stdout", + chunk: `再開 ${i}`, + }) + "\n", + ).join(""), + ); + const logRef = `${company}/${agent}/${run}.ndjson`; + await fs.mkdir(path.dirname(path.join(directory, "logs", logRef)), { + recursive: true, + }); + await fs.writeFile(path.join(directory, "logs", logRef), log); + await fs.mkdir(path.join(directory, "traces"), { recursive: true }); + const native = Buffer.from('{"kind":"frame","rawBase64":"e30="}\n'.repeat(33000).trimEnd()), + rehydration = Buffer.from('{"kind":"interpretation","frameId":1}\n'); + const expectedTrace = Buffer.concat([native, Buffer.from("\n"), rehydration]); + await fs.writeFile(path.join(directory, "traces", filename), native); + await fs.writeFile(path.join(directory, "traces", `${filename}.rehydration`), rehydration); + const expiry = new Date(Date.now() + 3600_000); + await db.execute( + sql`insert into heartbeat_runs(id, company_id, agent_id, status, log_store, log_ref, log_bytes, log_sha256, last_output_seq) values (${run}, ${company}, ${agent}, 'succeeded', 'local_file', ${logRef}, ${log.length}, ${createHash("sha256").update(log).digest("hex")}, 100)`, + ); + await db.execute(sql`insert into provider_trace_records(id, company_id, run_id, status, provider, trace_ref, requested_by, expires_at, digest) + values (${trace}, ${company}, ${run}, 'complete', 'codex', ${filename}, 'test', ${expiry.toISOString()}, ${`sha256:${createHash( + "sha256", + ) + .update(expectedTrace) + .digest("hex")}`})`); + const active = randomUUID(), + missing = randomUUID(), + corrupt = randomUUID(); + await db.execute(sql`insert into heartbeat_runs(id, company_id, agent_id, status, log_store, log_ref, log_bytes, log_sha256) + values (${active}, ${company}, ${agent}, 'running', 'local_file', ${logRef}, ${log.length}, null), + (${missing}, ${company}, ${agent}, 'succeeded', 'local_file', 'missing.ndjson', 10, null), + (${corrupt}, ${company}, ${agent}, 'succeeded', 'local_file', ${logRef}, ${log.length + 1}, null)`); + const legacyLease = randomUUID(), + nativeLease = randomUUID(), + leaseExpiry = new Date(Date.now() + 60_000).toISOString(); + await db.execute(sql`insert into heartbeat_runs(id, company_id, agent_id, status, log_store, log_ref, log_bytes, controller_lease_expires_at) + values (${legacyLease}, ${company}, ${agent}, 'succeeded', 'local_file', ${logRef}, ${log.length}, ${leaseExpiry}), + (${nativeLease}, ${company}, ${agent}, 'succeeded', 'local_file', ${logRef}, ${log.length}, null)`); + await db.execute( + sql`insert into native_run_finalizations values (${nativeLease}, ${company}, ${leaseExpiry})`, + ); + const inventory = await migrateRunEvidence(db, { + apply: false, + companyId: company, + }); + expect(inventory.items.find((i) => i.id === run)?.state).toBe("inventoried"); + const result = await migrateRunEvidence(db, { + apply: true, + companyId: company, + }); + expect(result.items.filter((i) => i.state === "migrated")).toHaveLength(2); + expect(result.items.filter((i) => i.state === "blocked")).toHaveLength(5); + const records = await db.$client.unsafe( + "select log_store, log_ref from heartbeat_runs where id=$1", + [run], + ); + expect(records[0].log_store).toBe("s3_segments"); + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + const reader = createObjectRunLogStore({ + provider, + keyPrefix: "run-logs", + assertWriterAuthority: async () => { + throw Error("read-only"); + }, + }); + await fs.rm(path.join(directory, "logs"), { recursive: true }); + await fs.rm(path.join(directory, "traces"), { recursive: true }); + expect( + Buffer.from( + ( + await reader.read({ + store: "s3_segments", + logRef: records[0].log_ref, + }) + ).content, + ), + ).toEqual(log); + const saved = await db.$client.unsafe( + "select trace_ref, expires_at, status from provider_trace_records where id=$1", + [trace], + ); + expect(new Date(saved[0].expires_at).getTime()).toBe(expiry.getTime()); + expect(saved[0].status).toBe("complete"); + expect(await createProviderTraceObjectStore(provider).read(saved[0].trace_ref)).toEqual( + expectedTrace, + ); + }, 30_000); + it("blocks a native archive with a missing committed tail and checkpoints only verified source coverage", async () => { + const companyId = randomUUID(), + agentId = randomUUID(); + const runs: string[] = []; + for (const state of ["missing", "finalized", "covered"]) { + const complete = state !== "missing", + finalized = state === "finalized"; + const runId = randomUUID(), + sourceInstanceId = randomUUID(), + normalizedSessionId = randomUUID(); + runs.push(runId); + const events: PrpEvent[] = [1, 2].map((sourceSeq) => ({ + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + runId, + sourceInstanceId, + sourceSeq, + sourceEventId: `${sourceInstanceId}:${sourceSeq}`, + sourceKind: "runner", + normalizedSessionId, + priority: 2, + emittedAt: "2026-10-30T00:00:00Z", + eventType: "item.delta", + payload: { text: `part ${sourceSeq}` }, + })); + const logRef = `${companyId}/${agentId}/${runId}.ndjson`; + const bytes = Buffer.from( + events + .slice(0, complete ? 2 : 1) + .map( + (event, index) => + JSON.stringify({ + ts: "then", + seq: index + 1, + stream: "stdout", + chunk: formatNativeRunLogEvent(event), + }) + "\n", + ) + .join(""), + ); + await fs.mkdir(path.dirname(path.join(directory, "logs", logRef)), { recursive: true }); + await fs.writeFile(path.join(directory, "logs", logRef), bytes); + await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status,runtime_mode,log_store,log_ref,log_bytes,log_sha256) + values(${runId},${companyId},${agentId},'succeeded','native','local_file',${logRef},${finalized ? bytes.length : null},${finalized ? createHash("sha256").update(bytes).digest("hex") : null})`); + for (const [index, event] of events.entries()) + await db.execute(sql`insert into heartbeat_run_events + values(${companyId},${runId},${index + 1},${JSON.stringify({ prpEvent: event })}::jsonb,${sourceInstanceId},${event.sourceEventId},${event.sourceSeq},${nativeSha256(event)},${event.eventType})`); + } + const migrated = await migrateRunEvidence(db, { apply: true, companyId }); + expect(migrated.items.find((item) => item.id === runs[0])).toMatchObject({ + state: "blocked", + reason: "native_log_source_coverage_unproven", + }); + expect(migrated.items.find((item) => item.id === runs[1])?.state).toBe("migrated"); + expect(migrated.items.find((item) => item.id === runs[2])?.state).toBe("migrated"); + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + for (const runId of runs.slice(1)) { + const journal = createObjectByteJournal({ + provider, + prefix: `run-logs/segments-v1/${companyId}/${agentId}/${runId}`, + }); + expect((await journal.head())!.head.sourceCursor).toBe(2); + } + const [blocked] = await db.$client.unsafe("select log_store from heartbeat_runs where id=$1", [ + runs[0]!, + ]); + expect(blocked!.log_store).toBe("local_file"); + }); + it("migrates a digest-verified large native event while blocking unproven and corrupt files", async () => { + const companyId = randomUUID(), + agentId = randomUUID(); + const cases = [ + "finalized", + "missing-size", + "missing-digest", + "wrong-digest", + "wrong-size", + ] as const; + const runs = new Map(); + for (const state of cases) { + const runId = randomUUID(), + sourceInstanceId = randomUUID(); + const event: PrpEvent = { + schema: "paperclip.prp.event.v1", + schemaVersion: 1, + runId, + sourceInstanceId, + sourceSeq: 1, + sourceEventId: `${sourceInstanceId}:1`, + sourceKind: "runner", + normalizedSessionId: randomUUID(), + priority: 2, + emittedAt: "2026-10-30T00:00:00Z", + eventType: "item.delta", + payload: { text: "large tool result ".repeat(12_000) }, + }; + const chunk = compactRunLogChunk(formatNativeRunLogEvent(event)); + expect(chunk).toContain("paperclip truncated run log chunk"); + expect(() => JSON.parse(chunk)).toThrow(); + const bytes = Buffer.from( + `${JSON.stringify({ ts: "then", seq: 1, stream: "stdout", chunk })}\n`, + ); + const logRef = `${companyId}/${agentId}/${runId}.ndjson`; + runs.set(state, { runId, logRef, bytes }); + await fs.mkdir(path.dirname(path.join(directory, "logs", logRef)), { recursive: true }); + await fs.writeFile(path.join(directory, "logs", logRef), bytes); + const digest = createHash("sha256").update(bytes).digest("hex"); + await db.execute(sql`insert into heartbeat_runs(id,company_id,agent_id,status,runtime_mode,log_store,log_ref,log_bytes,log_sha256) + values(${runId},${companyId},${agentId},'succeeded','native','local_file',${logRef}, + ${state === "missing-size" ? null : bytes.length + (state === "wrong-size" ? 1 : 0)}, + ${state === "missing-digest" ? null : state === "wrong-digest" ? "0".repeat(64) : digest})`); + await db.execute(sql`insert into heartbeat_run_events values(${companyId},${runId},1, + ${JSON.stringify({ prpEvent: event })}::jsonb,${sourceInstanceId},${event.sourceEventId},1,${nativeSha256(event)},${event.eventType})`); + } + const result = await migrateRunEvidence(db, { apply: true, companyId }); + const finalized = runs.get("finalized")!; + expect(result.items.find((item) => item.id === finalized.runId)?.state).toBe("migrated"); + for (const state of cases.slice(1)) { + const run = runs.get(state)!; + expect(result.items.find((item) => item.id === run.runId)).toMatchObject({ + state: "blocked", + reason: state.startsWith("wrong-") + ? "legacy_length_or_digest_mismatch" + : "native_log_source_coverage_unproven", + }); + const [row] = await db.$client.unsafe("select log_store from heartbeat_runs where id=$1", [ + run.runId, + ]); + expect(row!.log_store).toBe("local_file"); + } + const provider = createS3StorageProvider({ + endpoint, + bucket, + region: "us-east-1", + forcePathStyle: true, + }); + const logRef = `${companyId}/${agentId}/${finalized.runId}`; + const journal = createObjectByteJournal({ provider, prefix: `run-logs/segments-v1/${logRef}` }); + expect((await journal.head())!.head.sourceCursor).toBe(1); + await fs.unlink(path.join(directory, "logs", finalized.logRef)); + const reader = createObjectRunLogStore({ + provider, + keyPrefix: "run-logs", + assertWriterAuthority: async () => { + throw new Error("read-only"); + }, + }); + expect(Buffer.from((await reader.read({ store: "s3_segments", logRef })).content)).toEqual( + finalized.bytes, + ); + }); +}); diff --git a/server/src/services/run-evidence-migration.ts b/server/src/services/run-evidence-migration.ts new file mode 100644 index 00000000000..acf9aeae59b --- /dev/null +++ b/server/src/services/run-evidence-migration.ts @@ -0,0 +1,438 @@ +import { createHash } from "node:crypto"; +import { createReadStream, promises as fs } from "node:fs"; +import path from "node:path"; +import { and, asc, desc, eq, gt, inArray, isNull, lte, or, sql } from "drizzle-orm"; +import { + heartbeatRuns, + heartbeatRunEvents, + nativeRunFinalizations, + providerTraceRecords, + type Db, +} from "@paperclipai/db"; +import { resolvePaperclipInstanceRoot } from "../home-paths.js"; +import { createObjectByteJournal } from "./object-byte-journal.js"; +import { + createProviderTraceObjectStore, + importingProviderTraceRef, +} from "./provider-trace-object-store.js"; +import { openLegacyRunLogForMigration } from "./run-log-store.js"; +import { resolveRunEvidenceStorage } from "./run-evidence-storage.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { + createNativeRunLogCoverageVerifier, + nativeRunLogSourceWhere, +} from "./native-runtime/native-run-log-export.js"; + +const SETTLED = ["succeeded", "failed", "cancelled", "timed_out"]; +export interface RunEvidenceMigrationItem { + kind: "log" | "trace"; + id: string; + state: "inventoried" | "migrated" | "blocked"; + bytes?: number; + sha256?: string; + reason?: string; +} + +export async function migrateRunEvidence( + db: Db, + options: { + apply: boolean; + companyId?: string; + afterRunId?: string; + afterTraceId?: string; + limit?: number; + kinds?: readonly ("log" | "trace")[]; + signal?: AbortSignal; + }, +): Promise<{ + items: RunEvidenceMigrationItem[]; + nextRunId?: string; + nextTraceId?: string; +}> { + const limit = options.limit ?? 50; + options.signal?.throwIfAborted(); + if (!Number.isInteger(limit) || limit < 1 || limit > 100) + throw new Error("Migration limit must be between 1 and 100"); + const config = resolveRunEvidenceStorage(); + if (!config) throw new Error("Run evidence migration requires instance S3 storage"); + function drainedWriter(now = new Date()) { + return and( + inArray(heartbeatRuns.status, SETTLED), + or( + isNull(heartbeatRuns.controllerLeaseExpiresAt), + lte(heartbeatRuns.controllerLeaseExpiresAt, now), + ), + sql`not exists (select 1 from ${nativeRunFinalizations} where ${and( + eq(nativeRunFinalizations.runId, heartbeatRuns.id), + eq(nativeRunFinalizations.companyId, heartbeatRuns.companyId), + gt(nativeRunFinalizations.leaseExpiresAt, now), + )})`, + ); + } + async function assertDrained(runId: string, companyId: string) { + const [run] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and(eq(heartbeatRuns.id, runId), eq(heartbeatRuns.companyId, companyId), drainedWriter()), + ); + if (!run) throw new Error("legacy_writer_not_drained"); + } + const items: RunEvidenceMigrationItem[] = []; + const logs = + options.kinds && !options.kinds.includes("log") + ? [] + : await db + .select({ + id: heartbeatRuns.id, + companyId: heartbeatRuns.companyId, + agentId: heartbeatRuns.agentId, + status: heartbeatRuns.status, + runtimeMode: heartbeatRuns.runtimeMode, + logRef: heartbeatRuns.logRef, + bytes: heartbeatRuns.logBytes, + sha256: heartbeatRuns.logSha256, + lastSequence: heartbeatRuns.lastOutputSeq, + }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.logStore, "local_file"), + options.companyId ? eq(heartbeatRuns.companyId, options.companyId) : undefined, + options.afterRunId ? gt(heartbeatRuns.id, options.afterRunId) : undefined, + ), + ) + .orderBy(asc(heartbeatRuns.id)) + .limit(limit); + for (const run of logs) { + options.signal?.throwIfAborted(); + const item: RunEvidenceMigrationItem = { + kind: "log", + id: run.id, + state: "inventoried", + }; + items.push(item); + try { + if (!SETTLED.includes(run.status)) throw new Error("legacy_writer_not_settled"); + if (!run.logRef) throw new Error("legacy_reference_missing"); + await assertDrained(run.id, run.companyId); + const ref = `${run.companyId}/${run.agentId}/${run.id}`; + const journal = createObjectByteJournal({ + provider: config.provider, + prefix: `${config.keyPrefix || "run-logs"}/segments-v1/${ref}`, + }); + let writer = options.apply + ? await journal.claim({ + expectedOwner: (await journal.head(options.signal))?.head.owner ?? null, + signal: options.signal, + assertAuthority: async () => { + const [current] = await db + .select({ id: heartbeatRuns.id }) + .from(heartbeatRuns) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.logStore, "local_file"), + eq(heartbeatRuns.logRef, run.logRef!), + drainedWriter(), + ), + ); + if (!current) throw new Error("legacy_writer_changed"); + }, + }) + : null; + const source = await openLegacyRunLogForMigration(run.logRef); + if (source.source === "mirror" && (run.bytes === null || !run.sha256)) { + source.stream.destroy(); + throw new Error("mirror_completeness_unproven"); + } + const hash = createHash("sha256"); + const nativeCoverage = + run.runtimeMode === "native" && (run.bytes === null || run.sha256 === null) + ? createNativeRunLogCoverageVerifier(db, run.companyId, run.id) + : undefined; + let offset = 0, + tail = Buffer.alloc(0), + sequence = run.lastSequence; + try { + for await (const chunk of source.stream) { + options.signal?.throwIfAborted(); + const bytes = Buffer.from(chunk); + hash.update(bytes); + await nativeCoverage?.write(bytes); + tail = Buffer.concat([tail, bytes]).subarray(-262144); + if (writer) { + const overlap = Math.min(bytes.length, Math.max(0, writer.head.end - offset)); + if ( + overlap && + !(await journal.readRange(writer.head, offset, overlap, options.signal)).equals( + bytes.subarray(0, overlap), + ) + ) + throw new Error("migration_prefix_conflict"); + if (overlap < bytes.length) + writer = await journal.append(writer, bytes.subarray(overlap), options.signal); + } + offset += bytes.length; + } + } finally { + source.stream.destroy(); + } + const digest = hash.digest("hex"); + if ( + (source.bytes !== undefined && source.bytes !== offset) || + (run.bytes !== null && run.bytes !== offset) || + (run.sha256 && run.sha256 !== digest) + ) + throw new Error("legacy_length_or_digest_mismatch"); + for (const line of tail.toString("utf8").split("\n")) { + try { + const entry = JSON.parse(line); + if (Number.isSafeInteger(entry.seq)) sequence = Math.max(sequence, entry.seq); + } catch {} + } + item.bytes = offset; + item.sha256 = digest; + let sourceCursor = (await nativeCoverage?.finish()) ?? 0; + if (run.runtimeMode === "native" && !nativeCoverage) { + const [source] = await db + .select({ seq: heartbeatRunEvents.seq }) + .from(heartbeatRunEvents) + .where(nativeRunLogSourceWhere(run.companyId, run.id)) + .orderBy(desc(heartbeatRunEvents.seq)) + .limit(1); + sourceCursor = source?.seq ?? 0; + } + if (writer) { + options.signal?.throwIfAborted(); + if (writer.head.end !== offset) throw new Error("migration_prefix_exceeds_source"); + + const remoteHash = createHash("sha256"); + for (let start = 0; start < offset; start += 1024 * 1024) + remoteHash.update( + await journal.readRange(writer.head, start, 1024 * 1024, options.signal), + ); + if (remoteHash.digest("hex") !== digest) + throw new Error("migration_remote_digest_mismatch"); + writer.head = { ...writer.head, sequence, sourceCursor }; + await journal.close(writer, options.signal); + options.signal?.throwIfAborted(); + const [updated] = await db + .update(heartbeatRuns) + .set({ logStore: "s3_segments", logRef: ref }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.companyId, run.companyId), + eq(heartbeatRuns.logStore, "local_file"), + eq(heartbeatRuns.logRef, run.logRef), + run.bytes === null + ? isNull(heartbeatRuns.logBytes) + : eq(heartbeatRuns.logBytes, run.bytes), + run.sha256 === null + ? isNull(heartbeatRuns.logSha256) + : eq(heartbeatRuns.logSha256, run.sha256), + drainedWriter(), + run.runtimeMode === "native" + ? sql`not exists (select 1 from ${heartbeatRunEvents} where ${and( + nativeRunLogSourceWhere(run.companyId, run.id), + gt(heartbeatRunEvents.seq, sourceCursor), + )})` + : undefined, + ), + ) + .returning({ id: heartbeatRuns.id }); + if (!updated) throw new Error("migration_publication_conflict"); + item.state = "migrated"; + } + } catch (error) { + options.signal?.throwIfAborted(); + item.state = "blocked"; + item.reason = error instanceof Error ? error.message : "migration_failed"; + } + } + options.signal?.throwIfAborted(); + const traces = + options.kinds && !options.kinds.includes("trace") + ? [] + : await db + .select({ trace: providerTraceRecords, runStatus: heartbeatRuns.status }) + .from(providerTraceRecords) + .innerJoin( + heartbeatRuns, + and( + eq(heartbeatRuns.id, providerTraceRecords.runId), + eq(heartbeatRuns.companyId, providerTraceRecords.companyId), + ), + ) + .where( + and( + sql`${providerTraceRecords.traceRef} not like 's3-segments-v1:%'`, + isNull(providerTraceRecords.deletedAt), + gt(providerTraceRecords.expiresAt, new Date()), + options.companyId ? eq(providerTraceRecords.companyId, options.companyId) : undefined, + options.afterTraceId ? gt(providerTraceRecords.id, options.afterTraceId) : undefined, + ), + ) + .orderBy(asc(providerTraceRecords.id)) + .limit(limit); + const traceObjects = + traces.length && options.apply + ? createProviderTraceObjectStore(config.provider, config.tracePrefix) + : null; + for (const { trace, runStatus } of traces) { + options.signal?.throwIfAborted(); + const item: RunEvidenceMigrationItem = { + kind: "trace", + id: trace.id, + state: "inventoried", + }; + items.push(item); + try { + if (!SETTLED.includes(runStatus) || trace.status === "capturing") + throw new Error("legacy_trace_writer_not_settled"); + await assertDrained(trace.runId, trace.companyId); + const importing = importingProviderTraceRef(trace.traceRef); + if (importing && (importing[1] !== trace.companyId || importing[2] !== trace.runId)) + throw new Error("trace_import_binding_mismatch"); + const filename = importing?.[3] ?? trace.traceRef; + if (!/^[a-f0-9-]+\.ndjson$/.test(filename)) throw new Error("invalid_legacy_trace_ref"); + const file = path.join( + process.env.PROVIDER_TRACE_BASE_PATH || + path.join(resolvePaperclipInstanceRoot(), "data", "provider-traces"), + filename, + ); + for (const sidecar of [file, `${file}.rehydration`]) { + const stat = await fs.stat(sidecar).catch((error) => { + if (error.code === "ENOENT" && sidecar !== file) return null; + throw error; + }); + if (stat && (!stat.isFile() || stat.size > 192 * 1024 * 1024)) + throw new Error("legacy_trace_capture_size_invalid"); + } + const sourceHash = createHash("sha256"); + let byteCount = 0, + nativeTail: number | undefined; + for (const [index, sidecar] of [file, `${file}.rehydration`].entries()) { + const source = createReadStream(sidecar, { + highWaterMark: 1024 * 1024, + signal: options.signal, + }); + try { + let first = true; + for await (const chunk of source) { + const bytes = Buffer.from(chunk); + if ( + index === 1 && + first && + bytes.length && + nativeTail !== undefined && + nativeTail !== 10 + ) { + sourceHash.update("\n"); + byteCount++; + } + first = false; + sourceHash.update(bytes); + byteCount += bytes.length; + if (index === 0 && bytes.length) nativeTail = bytes.at(-1); + } + } catch (error) { + if (index !== 1 || (error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } finally { + source.destroy(); + } + } + const digest = `sha256:${sourceHash.digest("hex")}`; + if (trace.digest && trace.digest !== digest) throw new Error("legacy_trace_digest_mismatch"); + item.bytes = byteCount; + item.sha256 = digest; + if (traceObjects) { + options.signal?.throwIfAborted(); + const ref = `s3-segments-v1:${trace.companyId}/${trace.runId}/${filename}`; + const pendingRef = ref.replace("s3-segments-v1:", "s3-import-v1:"); + if (!importing) { + const staged = await db.transaction(async (tx) => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.providerTraceRetention); + return tx + .update(providerTraceRecords) + .set({ traceRef: pendingRef }) + .where( + and( + eq(providerTraceRecords.id, trace.id), + eq(providerTraceRecords.traceRef, trace.traceRef), + isNull(providerTraceRecords.deletedAt), + gt(providerTraceRecords.expiresAt, new Date()), + ), + ) + .returning({ id: providerTraceRecords.id }); + }); + if (!staged.length) throw new Error("trace_migration_staging_conflict"); + trace.traceRef = pendingRef; + } + await traceObjects.open( + ref, + file, + async () => { + await assertDrained(trace.runId, trace.companyId); + const [current] = await db + .select({ id: providerTraceRecords.id }) + .from(providerTraceRecords) + .where( + and( + eq(providerTraceRecords.id, trace.id), + eq(providerTraceRecords.traceRef, trace.traceRef), + isNull(providerTraceRecords.deletedAt), + gt(providerTraceRecords.expiresAt, new Date()), + ), + ); + if (!current) throw new Error("trace_migration_authority_changed"); + }, + { restore: false, signal: options.signal }, + ); + await traceObjects.finish(ref); + const remoteHash = createHash("sha256"); + let remoteBytes = 0; + for await (const chunk of traceObjects.readChunks(ref, options.signal)) { + remoteHash.update(chunk); + remoteBytes += chunk.length; + } + if (remoteBytes !== byteCount || `sha256:${remoteHash.digest("hex")}` !== digest) + throw new Error("trace_migration_remote_digest_mismatch"); + options.signal?.throwIfAborted(); + const updated = await db.transaction(async (tx) => { + await notifyDeliveryWork(tx, DELIVERY_QUEUES.providerTraceRetention); + return tx + .update(providerTraceRecords) + .set({ traceRef: ref }) + .where( + and( + eq(providerTraceRecords.id, trace.id), + eq(providerTraceRecords.traceRef, trace.traceRef), + isNull(providerTraceRecords.deletedAt), + gt(providerTraceRecords.expiresAt, new Date()), + sql`exists (select 1 from ${heartbeatRuns} where ${and( + eq(heartbeatRuns.id, trace.runId), + eq(heartbeatRuns.companyId, trace.companyId), + drainedWriter(), + )})`, + ), + ) + .returning({ id: providerTraceRecords.id }); + }); + if (!updated.length) throw new Error("trace_migration_publication_conflict"); + item.state = "migrated"; + } + } catch (error) { + options.signal?.throwIfAborted(); + item.state = "blocked"; + item.reason = error instanceof Error ? error.message : "trace_migration_failed"; + } + } + return { + items, + nextRunId: logs.length === limit ? logs.at(-1)!.id : undefined, + nextTraceId: traces.length === limit ? traces.at(-1)!.trace.id : undefined, + }; +} diff --git a/server/src/services/run-evidence-storage.integration.test.ts b/server/src/services/run-evidence-storage.integration.test.ts new file mode 100644 index 00000000000..7c3cfef386b --- /dev/null +++ b/server/src/services/run-evidence-storage.integration.test.ts @@ -0,0 +1,225 @@ +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + S3Client, + CreateBucketCommand, + ListObjectsV2Command, + DeleteObjectsCommand, + DeleteBucketCommand, +} from "@aws-sdk/client-s3"; +import type { RunLogStore } from "./run-log-store.js"; + +const endpoint = process.env.PAPERCLIP_LOG_TEST_S3_ENDPOINT; +describe.skipIf(!endpoint)("instance storage for run evidence", () => { + const buckets = [0, 1].map(() => `pc-evidence-config-${randomUUID()}`); + const stores: RunLogStore[] = []; + let client: S3Client, directory: string; + beforeAll(async () => { + directory = await fs.mkdtemp(path.join(tmpdir(), "evidence-config-")); + vi.stubEnv("PAPERCLIP_HOME", directory); + vi.stubEnv("PAPERCLIP_CONFIG", path.join(directory, "config.json")); + vi.stubEnv("PAPERCLIP_DISABLE_CWD_ENV_FILE", "true"); + vi.stubEnv("RUN_LOG_BASE_PATH", path.join(directory, "logs")); + for (const key of [ + "RUN_LOG_STORE", + "PROVIDER_TRACE_STORE", + "RUN_LOG_S3_BUCKET", + "RUN_LOG_S3_PREFIX", + "PROVIDER_TRACE_S3_PREFIX", + ]) + vi.stubEnv(key, undefined); + client = new S3Client({ endpoint, region: "us-east-1", forcePathStyle: true }); + for (const bucket of buckets) await client.send(new CreateBucketCommand({ Bucket: bucket })); + for (const bucket of buckets) { + vi.resetModules(); + selectStack(bucket); + const { getRunLogStore } = await import("./run-log-store.js"); + stores.push(getRunLogStore()); + } + }); + afterAll(async () => { + if (client) { + for (const bucket of buckets) { + for (;;) { + const page = await client.send(new ListObjectsV2Command({ Bucket: bucket })); + if (!page.Contents?.length) break; + await client.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { Objects: page.Contents.map((object) => ({ Key: object.Key! })) }, + }), + ); + } + await client.send(new DeleteBucketCommand({ Bucket: bucket })); + } + client.destroy(); + } + vi.unstubAllEnvs(); + if (directory) await fs.rm(directory, { recursive: true, force: true }); + }); + function selectStack(bucket: string, prefix = "existing-root") { + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_BUCKET", bucket); + vi.stubEnv("PAPERCLIP_STORAGE_S3_REGION", "us-east-1"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_ENDPOINT", endpoint!); + vi.stubEnv("PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", "true"); + vi.stubEnv("PAPERCLIP_STORAGE_S3_PREFIX", prefix); + } + it("automatically uses each stack's existing bucket and prefix for identical run identities", async () => { + const handles: Awaited>[] = []; + for (const [index, bucket] of buckets.entries()) { + const store = stores[index]!; + const handle = await store.begin({ + companyId: "same-company", + agentId: "same-agent", + runId: "same-run", + newRun: true, + assertWriterAuthority: async () => undefined, + }); + handles.push(handle); + expect(handle.store).toBe("s3_segments"); + await store.append(handle, { stream: "stdout", ts: "then", chunk: `stack-${index}`, seq: 1 }); + await store.finalize(handle); + const keys = (await client.send(new ListObjectsV2Command({ Bucket: bucket }))).Contents!.map( + (object) => object.Key!, + ); + expect( + keys.every((key) => + key.startsWith("existing-root/run-logs/segments-v1/same-company/same-agent/same-run/"), + ), + ).toBe(true); + } + expect(handles[0]!.logRef).toBe(handles[1]!.logRef); + for (const [index, store] of stores.entries()) { + const content = (await store.read(handles[1 - index]!)).content; + expect(content).toContain(`stack-${index}`); + expect(content).not.toContain(`stack-${1 - index}`); + } + await fs.rm(path.join(directory, "logs"), { recursive: true, force: true }); + expect((await stores[0]!.read(handles[0]!)).content).toContain("stack-0"); + }); + it("honors config-file-only storage and keeps a locally configured instance local", async () => { + for (const key of [ + "PAPERCLIP_STORAGE_PROVIDER", + "PAPERCLIP_STORAGE_S3_BUCKET", + "PAPERCLIP_STORAGE_S3_REGION", + "PAPERCLIP_STORAGE_S3_ENDPOINT", + "PAPERCLIP_STORAGE_S3_PREFIX", + "PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE", + ]) + vi.stubEnv(key, undefined); + await fs.writeFile( + path.join(directory, "config.json"), + JSON.stringify({ + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00Z", source: "configure" }, + database: { mode: "embedded-postgres" }, + logging: { mode: "file" }, + server: {}, + storage: { + provider: "s3", + s3: { + bucket: buckets[0], + region: "us-east-1", + endpoint, + prefix: "file-config-root", + forcePathStyle: true, + }, + }, + }), + ); + vi.resetModules(); + const { getRunLogStore } = await import("./run-log-store.js"); + const external = getRunLogStore(); + const handle = await external.begin({ + companyId: "file-company", + agentId: "agent", + runId: "run", + newRun: true, + assertWriterAuthority: async () => undefined, + }); + expect(handle.store).toBe("s3_segments"); + await external.append(handle, { stream: "stdout", ts: "then", chunk: "file-config" }); + await external.finalize(handle); + expect( + ( + await client.send( + new ListObjectsV2Command({ Bucket: buckets[0], Prefix: "file-config-root/run-logs/" }), + ) + ).KeyCount, + ).toBeGreaterThan(0); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "local_disk"); + vi.resetModules(); + const local = (await import("./run-log-store.js")).getRunLogStore(); + const localHandle = await local.begin({ + companyId: "local-company", + agentId: "agent", + runId: "run", + newRun: true, + }); + expect(localHandle.store).toBe("local_file"); + expect(() => local.read(handle)).toThrow("Object run log storage is not configured"); + await local.append(localHandle, { stream: "stdout", ts: "then", chunk: "local" }); + await local.finalize(localHandle); + expect((await local.read(localHandle)).content).toContain("local"); + vi.stubEnv("PAPERCLIP_STORAGE_PROVIDER", "s3"); + vi.resetModules(); + const restored = (await import("./run-log-store.js")).getRunLogStore(); + expect((await restored.read(handle)).content).toContain("file-config"); + await fs.unlink(path.join(directory, "config.json")); + }); + it("scopes trace publication and cleanup to the existing prefix, company and stack", async () => { + vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", "a".repeat(64)); + const { resolveRunEvidenceStorage } = await import("./run-evidence-storage.js"); + const { createProviderTraceObjectStore } = await import("./provider-trace-object-store.js"); + const { createStorageService } = await import("../storage/service.js"); + const filename = `${randomUUID()}.ndjson`; + const records = []; + for (const [prefix, company] of [ + ["stack-a", "same-company"], + ["stack-b", "same-company"], + ["stack-a", "other-company"], + ]) { + selectStack(buckets[0]!, prefix); + const config = resolveRunEvidenceStorage()!; + const store = createProviderTraceObjectStore(config.provider, config.tracePrefix); + const storage = createStorageService(config.provider); + const ref = `s3-segments-v1:${company}/same-run/${filename}`; + const file = path.join(directory, `${prefix}-${company}.ndjson`); + const body = Buffer.from(`${JSON.stringify({ fixture: `${prefix}:${company}` })}\n`); + await store.open(ref, file, async () => undefined); + await fs.appendFile(file, body); + await store.finish(ref); + const product = await storage.putFile({ + companyId: company!, + namespace: "attachments", + originalFilename: "keep.txt", + contentType: "text/plain", + body: Buffer.from("keep"), + }); + records.push({ store, storage, ref, body, product, company: company! }); + } + expect(records[0]!.ref).toBe(records[1]!.ref); + await records[0]!.store.remove(records[0]!.ref); + await expect(records[0]!.store.read(records[0]!.ref)).rejects.toThrow("unavailable"); + for (const record of records.slice(1)) + expect(await record.store.read(record.ref)).toEqual(record.body); + for (const record of records) { + const object = await record.storage.getObject(record.company, record.product.objectKey); + const chunks = []; + for await (const chunk of object.stream) chunks.push(Buffer.from(chunk)); + expect(Buffer.concat(chunks).toString()).toBe("keep"); + } + const keys = ( + await client.send(new ListObjectsV2Command({ Bucket: buckets[0], Prefix: "stack-b/" })) + ).Contents!.map((object) => object.Key!); + expect( + keys.some((key) => + key.startsWith("stack-b/provider-traces/segments-v1/same-company/same-run/"), + ), + ).toBe(true); + expect(keys.some((key) => key.startsWith("stack-b/same-company/attachments/"))).toBe(true); + }); +}); diff --git a/server/src/services/run-evidence-storage.ts b/server/src/services/run-evidence-storage.ts new file mode 100644 index 00000000000..ed897549874 --- /dev/null +++ b/server/src/services/run-evidence-storage.ts @@ -0,0 +1,11 @@ +import { loadStorageConfig, type StorageConfig } from "../storage/config.js"; +import { createStorageProviderFromConfig } from "../storage/provider-registry.js"; + +export function resolveRunEvidenceStorage(config: StorageConfig = loadStorageConfig()) { + if (config.storageProvider !== "s3") return undefined; + return { + provider: createStorageProviderFromConfig(config), + keyPrefix: "run-logs", + tracePrefix: "provider-traces", + }; +} diff --git a/server/src/services/run-log-store.ts b/server/src/services/run-log-store.ts index 2c11aade946..05dfcfa0e9f 100644 --- a/server/src/services/run-log-store.ts +++ b/server/src/services/run-log-store.ts @@ -7,13 +7,21 @@ import { notFound } from "../errors.js"; import { resolvePaperclipInstanceRoot } from "../home-paths.js"; import { createS3StorageProvider } from "../storage/s3-provider.js"; import type { StorageProvider } from "../storage/types.js"; +import { createObjectRunLogStore } from "./object-run-log-store.js"; +import { resolveRunEvidenceStorage } from "./run-evidence-storage.js"; -export type RunLogStoreType = "local_file"; +export type RunLogStoreType = "local_file" | "s3_segments"; +export function isRunLogStoreType(value: unknown): value is RunLogStoreType { + return value === "local_file" || value === "s3_segments"; +} export interface RunLogHandle { store: RunLogStoreType; logRef: string; attemptId?: string; + initialBytes?: number; + initialSequence?: number; + initialSourceCursor?: number; } export interface RunLogReadOptions { @@ -50,15 +58,24 @@ export interface RunLogBeginInput { // began this log, so no durable prefix can exist to restore. Omitted means // the run may be a retry or recovery and the durable mirror is checked. newRun?: boolean; + + assertWriterAuthority?: () => Promise; + previousStore?: RunLogStoreType; + signal?: AbortSignal; } export interface RunLogStore { begin(input: RunLogBeginInput): Promise; append( handle: RunLogHandle, - event: { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string; seq?: number }, + event: { stream: "stdout" | "stderr" | "system"; chunk: string; ts: string; seq?: number; sourceCursor?: number }, ): Promise; - finalize(handle: RunLogHandle): Promise; + enqueue?( + handle: RunLogHandle, + event: Parameters[1], + ): Promise<{ committed: Promise }>; + flush?(handle: RunLogHandle): Promise; + finalize(handle: RunLogHandle, options?: { computeDigest?: boolean }): Promise; read(handle: RunLogHandle, opts?: RunLogReadOptions): Promise; // Optional so existing fakes/fixtures keep compiling: uploads every dirty // in-flight mirror immediately (graceful-shutdown path). No-op when the @@ -509,7 +526,7 @@ export function createDurableRunLogStore(options: DurableRunLogStoreOptions): Ru // NOT redirect the product's workspace/file storage (smaller blast radius). // Unset RUN_LOG_S3_BUCKET -> no mirror -> local-only (safe degrade). Creds come // from the standard AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY chain. -function resolveRunLogS3(): DurableRunLogStoreOptions["s3"] { +export function resolveRunLogS3(): DurableRunLogStoreOptions["s3"] { const bucket = process.env.RUN_LOG_S3_BUCKET?.trim(); if (!bucket) return undefined; const provider = createS3StorageProvider({ @@ -538,7 +555,40 @@ let cachedStore: RunLogStore | null = null; export function getRunLogStore() { if (cachedStore) return cachedStore; const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs"); - cachedStore = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() }); + const local = createDurableRunLogStore({ basePath, s3: resolveRunLogS3() }); + const s3 = resolveRunEvidenceStorage(); + const external = s3 ? createObjectRunLogStore({ + provider: s3.provider, keyPrefix: s3.keyPrefix || "run-logs", + assertWriterAuthority: async input => { + if (!input.assertWriterAuthority) throw new Error("Object run logs require admitted writer authority"); + await input.assertWriterAuthority(); + }, + }) : null; + const writer = external ?? local; + cachedStore = { + begin: input => { + if (input.previousStore === "local_file") return local.begin(input); + if (input.previousStore === "s3_segments") { + if (!external) throw new Error("Object run log storage is not configured"); + return external.begin(input); + } + return writer.begin(input); + }, + append: (handle, event) => handle.store === "s3_segments" ? external!.append(handle, event) : local.append(handle, event), + enqueue: (handle, event) => handle.store === "s3_segments" + ? external!.enqueue!(handle, event) + : Promise.resolve({ committed: local.append(handle, event) }), + flush: async handle => { if (handle.store === "s3_segments") await external!.flush!(handle); }, + finalize: (handle, options) => handle.store === "s3_segments" ? external!.finalize(handle, options) : local.finalize(handle), + read: (handle, opts) => { + if (handle.store === "s3_segments") { + if (!external) throw new Error("Object run log storage is not configured"); + return external.read(handle, opts); + } + return local.read(handle, opts); + }, + flushInflightMirrors: async () => { await Promise.all([local.flushInflightMirrors?.(), external?.flushInflightMirrors?.()]); }, + }; return cachedStore; } @@ -549,3 +599,21 @@ export function getRunLogStore() { export async function flushInFlightRunLogMirrors(): Promise { await cachedStore?.flushInflightMirrors?.(); } + + +export async function openLegacyRunLogForMigration(logRef: string) { + const basePath = process.env.RUN_LOG_BASE_PATH ?? path.resolve(resolvePaperclipInstanceRoot(), "data", "run-logs"); + const file = resolveWithin(basePath, logRef); + try { + const stat = await fs.stat(file); + if (!stat.isFile()) throw new Error("Legacy run log is not a regular file"); + return { stream: createReadStream(file), bytes: stat.size, source: "local" as const }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + const config = resolveRunLogS3(); + if (!config) throw notFound("Legacy run log not found"); + const key = `${config.keyPrefix?.replace(/\/+$/, "") || "run-logs"}/${logRef}`; + const object = await config.provider.getObject({ objectKey: key, signal: AbortSignal.timeout(30_000) }); + return { stream: object.stream, bytes: object.contentLength, source: "mirror" as const }; +} diff --git a/server/src/services/run-secret-redaction.ts b/server/src/services/run-secret-redaction.ts index 5982aad5445..ad07a402920 100644 --- a/server/src/services/run-secret-redaction.ts +++ b/server/src/services/run-secret-redaction.ts @@ -95,6 +95,9 @@ export function createRunSecretRedactionRegistry(db: Db) { } return { + redactionValuesForRun: async (companyId: string, runId: string) => valuesForRuns(await db + .select({ contextSnapshot: registrySnapshot }).from(heartbeatRuns) + .where(and(eq(heartbeatRuns.companyId, companyId), eq(heartbeatRuns.id, runId)))), register: async (companyId: string, runId: string, value: string) => { const fingerprintSha256 = createHash("sha256").update(value).digest("hex"); await db.transaction(async (tx) => { diff --git a/server/src/services/usage-receipts.ts b/server/src/services/usage-receipts.ts index 0f99401d9da..74493640746 100644 --- a/server/src/services/usage-receipts.ts +++ b/server/src/services/usage-receipts.ts @@ -40,6 +40,8 @@ const envelopeSchema = z.object({ schema: z.literal("paperclip/accounting-receipt/v1"), id: z.string().uuid(), companyId: z.string().uuid(), runId: z.string().uuid(), sourceId: z.string().uuid(), sequence: z.number().int().positive(), receivedAt: z.iso.datetime(), adapterType: z.string().max(100), receipt: checkpointSchema, + producerEvent: z.object({ sourceEventId: z.string(), turnId: z.string().optional(), providerTurnId: z.string().optional(), + eventType: z.string(), usage: z.record(z.string(), z.unknown()).optional() }).optional(), // Derived by the recorder, not a provider's final charge. Keeping it in the // fsynced envelope preserves known partial attempts through database replay. liveCostCents: z.string().max(160).refine(value => { @@ -75,6 +77,27 @@ export async function persistUsageReceipt(db: Db, raw: UsageReceiptEnvelope) { if (!run) throw notFound("Accounting run not found"); const [existing] = await tx.select().from(runUsageReceipts).where(and(eq(runUsageReceipts.companyId, envelope.companyId), eq(runUsageReceipts.runId, envelope.runId), eq(runUsageReceipts.sourceId, envelope.sourceId), eq(runUsageReceipts.sequence, envelope.sequence))); if (existing && existing.receiptHash !== hash) throw conflict("Conflicting usage checkpoint replay"); + if (!existing && run.costAccountedAt && envelope.producerEvent) { + const previous = run.usageJson ?? {}; + const [saved] = await tx.select({ receiptJson: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and( + eq(runUsageReceipts.companyId, envelope.companyId), eq(runUsageReceipts.runId, envelope.runId), + sql`${runUsageReceipts.id}::text = ${previous.accountingReceiptId ?? null}`)); + const accepted = saved ? envelopeSchema.parse(saved.receiptJson).receipt : undefined; + const monetaryAmount = (receipt: UsageReceiptEnvelope["receipt"] | undefined) => { + const value = receipt?.cacheAdjustedCostUsd ?? receipt?.costUsdExact ?? receipt?.costUsd; + return value == null ? null : usdToUnits(value); + }; + const incomingMoney = monetaryAmount(envelope.receipt); + const authoritativeMoney = incomingMoney !== null && envelope.receipt.pricingProvenance?.source !== "rate_card"; + const authoritativeProvenance = envelope.receipt.pricingProvenance + && !["rate_card", "unknown"].includes(envelope.receipt.pricingProvenance.source); + if ((envelope.producerEvent.eventType === "usage.reported" && (!envelope.receipt.usage || !accepted)) + || Object.entries(envelope.receipt.usage ?? {}).some(([key, value]) => previous[key] !== value) + || (authoritativeMoney && incomingMoney !== monetaryAmount(accepted)) + || (authoritativeProvenance && receiptFingerprint(envelope.receipt.pricingProvenance) !== receiptFingerprint(accepted?.pricingProvenance ?? null))) { + throw conflict("native_usage_after_finalization_requires_reconciliation"); + } + } if (!existing) await tx.insert(runUsageReceipts).values({ id: envelope.id, companyId: envelope.companyId, runId: envelope.runId, sourceId: envelope.sourceId, sequence: envelope.sequence, receiptHash: hash, receiptJson: envelope, complete: envelope.receipt.complete, receivedAt: new Date(envelope.receivedAt), @@ -367,3 +390,33 @@ export async function createRunUsageRecorder(db: Db, input: { companyId: string; }), }; } + +export async function readAcceptedRunUsageCheckpoint(db: Db, input: { companyId: string; runId: string }): Promise { + const [run] = await db.select({ usageJson: heartbeatRuns.usageJson }).from(heartbeatRuns).where(and( + eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId))); + const state = run?.usageJson; + if (typeof state?.accountingReceiptId !== "string") return undefined; + const [saved] = await db.select({ receiptJson: runUsageReceipts.receiptJson }).from(runUsageReceipts).where(and( + eq(runUsageReceipts.companyId, input.companyId), eq(runUsageReceipts.runId, input.runId), + sql`${runUsageReceipts.id}::text = ${state.accountingReceiptId}`)); + if (!saved) return undefined; + const receipt = envelopeSchema.parse(saved.receiptJson); + if (receipt.sourceId !== state.accountingReceiptSourceId || receipt.sequence !== state.accountingReceiptSequence) return undefined; + return { ...receipt.receipt, complete: receipt.receipt.complete && state.accountingReceiptReady === true } as AdapterUsageCheckpoint; +} + +const producerAccountingProjectionFields = [ + "accountingReceiptOwner", "accountingReceiptSourceId", "accountingReceiptSequence", "accountingReceiptReady", + "accountingReceiptId", "accountingReceiptReceivedAt", "accountingCaptureFailed", + "inputTokens", "outputTokens", "cachedInputTokens", "cacheWriteTokens", + "provider", "biller", "model", "billingType", "usageSource", "providerRequestId", "usageByModel", + "costUsd", "costUsdExact", "cacheAdjustedCostUsd", "costStatus", "pricingProvenance", "budgetExposure", +]; + +export function preserveAcceptedRunUsageProjectionSql(patch: Record) { + const fields = sql.join(producerAccountingProjectionFields.map(field => sql`${field}`), sql`, `); + return sql`case when ${heartbeatRuns.usageJson}->>'accountingReceiptOwner' = 'native_producer' + then ${JSON.stringify(patch)}::jsonb || (select coalesce(jsonb_object_agg(key, value), '{}'::jsonb) + from jsonb_each(coalesce(${heartbeatRuns.usageJson}, '{}'::jsonb)) where key in (${fields})) + else ${JSON.stringify(patch)}::jsonb end`; +} diff --git a/server/src/storage/config.test.ts b/server/src/storage/config.test.ts new file mode 100644 index 00000000000..e3e725d83a1 --- /dev/null +++ b/server/src/storage/config.test.ts @@ -0,0 +1,95 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { paperclipConfigSchema } from "@paperclipai/shared"; +import { loadStorageConfig } from "./config.js"; +import { resolveRunEvidenceStorage } from "../services/run-evidence-storage.js"; +import { + projectCloudRuntimeConfig, + setCloudRuntimeConfigSnapshot, +} from "../services/cloud-runtime-config-snapshot.js"; + +afterEach(() => { + setCloudRuntimeConfigSnapshot(null); + vi.unstubAllEnvs(); +}); + +const file = paperclipConfigSchema.parse({ + $meta: { version: 1, updatedAt: "2026-10-30T00:00:00Z", source: "configure" }, + database: { mode: "embedded-postgres" }, + logging: { mode: "file" }, + server: {}, + storage: { + provider: "s3", + s3: { + bucket: "stack-file", + region: "auto", + endpoint: "https://objects.example.test", + prefix: "existing-stack-root", + forcePathStyle: true, + }, + }, +}); +describe("shared instance storage configuration", () => { + it("reuses the saved Cloud storage profile after its legacy config file is gone", () => { + vi.stubEnv("PAPERCLIP_MANAGED_CONFIG", ""); + vi.stubEnv("PAPERCLIP_CONFIG", "/nonexistent/legacy-config.json"); + for (const name of [ + "PROVIDER", + "S3_BUCKET", + "S3_REGION", + "S3_ENDPOINT", + "S3_PREFIX", + "S3_FORCE_PATH_STYLE", + ]) + vi.stubEnv(`PAPERCLIP_STORAGE_${name}`, undefined); + setCloudRuntimeConfigSnapshot({ + version: 1, + revision: 4, + config: projectCloudRuntimeConfig(file), + }); + expect(loadStorageConfig()).toMatchObject({ + storageProvider: "s3", + storageS3Bucket: "stack-file", + storageS3Prefix: "existing-stack-root", + }); + expect(resolveRunEvidenceStorage()).toMatchObject({ + keyPrefix: "run-logs", + tracePrefix: "provider-traces", + }); + }); + it("keeps the existing config-file defaults and S3 connection fields", () => { + expect(loadStorageConfig(file, {})).toMatchObject({ + storageProvider: "s3", + storageS3Bucket: "stack-file", + storageS3Region: "auto", + storageS3Endpoint: "https://objects.example.test", + storageS3Prefix: "existing-stack-root", + storageS3ForcePathStyle: true, + }); + }); + it("preserves environment precedence and an explicit false path-style setting", () => { + expect( + loadStorageConfig(file, { + PAPERCLIP_STORAGE_PROVIDER: "s3", + PAPERCLIP_STORAGE_S3_BUCKET: "stack-env", + PAPERCLIP_STORAGE_S3_REGION: "us-east-1", + PAPERCLIP_STORAGE_S3_ENDPOINT: "https://other.example.test", + PAPERCLIP_STORAGE_S3_PREFIX: "", + PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE: "false", + }), + ).toMatchObject({ + storageProvider: "s3", + storageS3Bucket: "stack-env", + storageS3Region: "us-east-1", + storageS3Endpoint: "https://other.example.test", + storageS3Prefix: "", + storageS3ForcePathStyle: false, + }); + }); + it("keeps an unconfigured instance local when only a legacy mirror is configured", () => { + const config = loadStorageConfig(null, { + RUN_LOG_S3_BUCKET: "legacy-mirror", + }); + expect(config.storageProvider).toBe("local_disk"); + expect(resolveRunEvidenceStorage(config)).toBeUndefined(); + }); +}); diff --git a/server/src/storage/config.ts b/server/src/storage/config.ts new file mode 100644 index 00000000000..c66f986595f --- /dev/null +++ b/server/src/storage/config.ts @@ -0,0 +1,39 @@ +import { STORAGE_PROVIDERS, type PaperclipConfig, type StorageProvider } from "@paperclipai/shared"; +import { readConfigFile } from "../config-file.js"; +import { resolveDefaultStorageDir, resolveHomeAwarePath } from "../home-paths.js"; + +export interface StorageConfig { + storageProvider: StorageProvider; + storageLocalDiskBaseDir: string; + storageS3Bucket: string; + storageS3Region: string; + storageS3Endpoint: string | undefined; + storageS3Prefix: string; + storageS3ForcePathStyle: boolean; +} + +export function loadStorageConfig( + fileConfig: PaperclipConfig | null = readConfigFile(), + env: NodeJS.ProcessEnv = process.env, +): StorageConfig { + const storage = fileConfig?.storage; + const selected = env.PAPERCLIP_STORAGE_PROVIDER; + const provider = + selected && STORAGE_PROVIDERS.includes(selected as StorageProvider) + ? (selected as StorageProvider) + : null; + return { + storageProvider: provider ?? storage?.provider ?? "local_disk", + storageLocalDiskBaseDir: resolveHomeAwarePath( + env.PAPERCLIP_STORAGE_LOCAL_DIR ?? storage?.localDisk?.baseDir ?? resolveDefaultStorageDir(), + ), + storageS3Bucket: env.PAPERCLIP_STORAGE_S3_BUCKET ?? storage?.s3?.bucket ?? "paperclip", + storageS3Region: env.PAPERCLIP_STORAGE_S3_REGION ?? storage?.s3?.region ?? "us-east-1", + storageS3Endpoint: env.PAPERCLIP_STORAGE_S3_ENDPOINT ?? storage?.s3?.endpoint ?? undefined, + storageS3Prefix: env.PAPERCLIP_STORAGE_S3_PREFIX ?? storage?.s3?.prefix ?? "", + storageS3ForcePathStyle: + env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE !== undefined + ? env.PAPERCLIP_STORAGE_S3_FORCE_PATH_STYLE === "true" + : (storage?.s3?.forcePathStyle ?? false), + }; +} diff --git a/server/src/storage/provider-registry.ts b/server/src/storage/provider-registry.ts index 6a494534aae..2784405cbe7 100644 --- a/server/src/storage/provider-registry.ts +++ b/server/src/storage/provider-registry.ts @@ -1,9 +1,9 @@ -import type { Config } from "../config.js"; +import type { StorageConfig } from "./config.js"; import type { StorageProvider } from "./types.js"; import { createLocalDiskStorageProvider } from "./local-disk-provider.js"; import { createS3StorageProvider } from "./s3-provider.js"; -export function createStorageProviderFromConfig(config: Config): StorageProvider { +export function createStorageProviderFromConfig(config: StorageConfig): StorageProvider { if (config.storageProvider === "local_disk") { return createLocalDiskStorageProvider(config.storageLocalDiskBaseDir); } diff --git a/server/src/storage/s3-provider.ts b/server/src/storage/s3-provider.ts index faa2deb949f..279655e481e 100644 --- a/server/src/storage/s3-provider.ts +++ b/server/src/storage/s3-provider.ts @@ -4,6 +4,7 @@ import { GetObjectCommand, HeadObjectCommand, PutObjectCommand, + ListObjectsV2Command, } from "@aws-sdk/client-s3"; import { putS3Multipart } from "./s3-multipart.js"; import { withS3FailureDiagnostics } from "./s3-failure.js"; @@ -81,6 +82,33 @@ export function createS3StorageProvider(config: S3ProviderConfig): StorageProvid return { id: "s3", + async putObjectConditional(input) { + const output = await client.send(new PutObjectCommand({ + Bucket: bucket, + Key: buildKey(prefix, input.objectKey), + Body: input.body, + ContentType: input.contentType, + ContentLength: input.contentLength, + IfMatch: input.ifMatch, + IfNoneMatch: input.ifNoneMatch, + }), { abortSignal: input.signal }); + if (!output.ETag) throw new Error("Object storage did not return a publication ETag"); + return { etag: output.ETag }; + }, + + async listObjects(input) { + const keyPrefix = buildKey(prefix, input.prefix); + const output = await client.send(new ListObjectsV2Command({ + Bucket: bucket, Prefix: keyPrefix, + ContinuationToken: input.continuationToken, MaxKeys: 1000, + }), { abortSignal: input.signal }); + const strip = prefix ? prefix.length + 1 : 0; + return { + keys: (output.Contents ?? []).flatMap(object => object.Key ? [object.Key.slice(strip)] : []), + continuationToken: output.IsTruncated ? output.NextContinuationToken : undefined, + }; + }, + async putObject(input) { const key = buildKey(prefix, input.objectKey); if (input.contentLength > 16 * 1024 * 1024) { diff --git a/server/src/storage/types.ts b/server/src/storage/types.ts index 60f6d1d24a6..97c80515b35 100644 --- a/server/src/storage/types.ts +++ b/server/src/storage/types.ts @@ -43,6 +43,16 @@ export interface StorageProvider { getObject(input: GetObjectInput): Promise; headObject(input: GetObjectInput): Promise; deleteObject(input: GetObjectInput): Promise; + + putObjectConditional?(input: PutObjectInput & ( + | { ifNoneMatch: "*"; ifMatch?: never } + | { ifMatch: string; ifNoneMatch?: never } + )): Promise<{ etag: string }>; + + listObjects?(input: { prefix: string; continuationToken?: string; signal?: AbortSignal }): Promise<{ + keys: string[]; + continuationToken?: string; + }>; } export type PutFileInput = { diff --git a/tests/perf/agent-chat/README.md b/tests/perf/agent-chat/README.md new file mode 100644 index 00000000000..847d3982f34 --- /dev/null +++ b/tests/perf/agent-chat/README.md @@ -0,0 +1,39 @@ +# Long-running agent chats + +`agent-chat-startup.spec.ts` drives the real standing-chat route with a retained +600-message, 300-run synthetic conversation. Completed runs have valid native +assistant events; there are no model calls. It verifies recent-only startup, +one backward page per upward gesture, delayed/error recovery, an unobscured latest reply, retained rows and +drafts, Back, older links, one saved live message, and sending before the first upward scroll. The send check uses separate retained fixture files per browser project so queued board input cannot alter the main history dataset. The live-message check posts one reply with a temporary key for the fixture agent, then revokes it. It starts no agent execution. Delayed inline images verify both following the latest reply and retaining an unpinned reading anchor. Additional checks keep message paging available during a receipts outage and verify that successive pages request each newly revealed metadata id once and avoid rereading previously hydrated terminal streams. Native stream recovery still refreshes on retry, resume, mode change, or reconnect. The disposable board user dismisses the current optional announcement before the journeys. + +Use a disposable loopback instance with Agent Chat enabled. The first seed needs +its matching PostgreSQL URL, with a database name ending in `_test`: + +```sh +PAPERCLIP_ISSUE_PERF_BASE_URL=http://127.0.0.1:3221 \ +PAPERCLIP_CHAT_PERF_DATABASE_URL=postgres://paperclip:paperclip@127.0.0.1:55503/fast_chat_preview_test \ + pnpm exec playwright test --config tests/perf/agent-chat/playwright.config.ts +``` + +The recipe enables Streamlined UI and the disposable board user's Coordinator +preference. It creates isolated fixture records through the API and database; +never point it at a shared instance. The fixture is retained at +`test-results/agent-chat/fixture-600.json`. Override +`PAPERCLIP_CHAT_PERF_SEED_FILE` to reuse an existing matching fixture; retained +fixtures need no database connection for browser journeys. +`PAPERCLIP_CHAT_PERF_MESSAGES=6000` creates a larger fixture for the same assertions. +The rig blocks service workers so fixture responses are intercepted deterministically and build-update reloads cannot change the selected UI artifact. The ordinary local prototype keeps the normal service worker enabled. +The rig runs the same journeys on desktop and mobile. Use `--project=desktop` +for the smaller iteration loop, then run both projects before handoff. + +`PAPERCLIP_TASK_COLD_OPEN_DATABASE_URL` lets the focused server suite reuse a +separate disposable loopback database ending in `_test`; it applies migrations +and retains randomly scoped fixture records until that database is discarded. +The normal CI path still uses the embedded PostgreSQL helper. + +Measure startup latency separately from these assertions: use matched production +builds, the same retained fixture and server, alternate baseline/candidate order, +and stop this task's builds and suites during timing samples. Readiness is the +visible recent conversation at its intended position with an editable composer, +not the loading shell. Retain per-frame geometry and server timings alongside +sample medians/ranges; do not treat fewer transcript requests as latency proof. diff --git a/tests/perf/agent-chat/agent-chat-fixture.ts b/tests/perf/agent-chat/agent-chat-fixture.ts new file mode 100644 index 00000000000..e9dffd6fc6f --- /dev/null +++ b/tests/perf/agent-chat/agent-chat-fixture.ts @@ -0,0 +1,71 @@ +import fs from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { expect, type APIRequestContext } from "@playwright/test"; + +export interface ChatFixture { + company: { id: string; issuePrefix: string }; + agent: { id: string }; + issue: { id: string }; + comments: { id: string; body: string }[]; +} +export async function longChatFixture(request: APIRequestContext, suffix = "", options: { delayedImage?: boolean } = {}): Promise { + const count = Number(process.env.PAPERCLIP_CHAT_PERF_MESSAGES ?? 600); + if (!Number.isSafeInteger(count) || count < 100 || count > 10_000) throw new Error("Use 100–10000 synthetic messages"); + const baseFile = process.env.PAPERCLIP_CHAT_PERF_SEED_FILE ?? `test-results/agent-chat/fixture-${count}.json`; + const file = suffix ? `${baseFile}.${suffix}.json` : baseFile; + const saved: ChatFixture | null = await fs.readFile(file, "utf8").then(JSON.parse).catch(error => { + if (error.code !== "ENOENT") throw error; + return null; + }); + if (saved && (await request.get(`/api/issues/${saved.issue.id}`)).ok()) return saved; + const databaseUrl = process.env.PAPERCLIP_CHAT_PERF_DATABASE_URL; + if (!databaseUrl) throw new Error("Set PAPERCLIP_CHAT_PERF_DATABASE_URL for a disposable local PostgreSQL database, or retain a seed file"); + const target = new URL(databaseUrl); + if (!["localhost", "127.0.0.1", "[::1]"].includes(target.hostname) || !target.pathname.endsWith("_test")) + throw new Error("The chat fixture requires a disposable loopback database ending in _test"); + const { createDb, closeRegisteredClients, heartbeatRuns, heartbeatRunEvents, issueComments, activityLog } = await import("../../../packages/db/src/index"); + const companyResponse = await request.post("/api/companies", { data: { name: `Long chat ${count}` } }); + expect(companyResponse.ok(), await companyResponse.text()).toBeTruthy(); + const company = await companyResponse.json(); + const agentResponse = await request.post(`/api/companies/${company.id}/agents`, { data: { + name: "History Fixture", role: "engineer", status: "paused", adapterType: "process", adapterConfig: { command: "/usr/bin/true" }, + } }); + expect(agentResponse.ok(), await agentResponse.text()).toBeTruthy(); + const agent = await agentResponse.json(); + const chatResponse = await request.post(`/api/companies/${company.id}/chats/${agent.id}`, { data: {} }); + expect(chatResponse.ok(), await chatResponse.text()).toBeTruthy(); + const issue = await chatResponse.json(); + const db = createDb(databaseUrl); + const comments: (typeof issueComments.$inferInsert)[] = []; + const runs: (typeof heartbeatRuns.$inferInsert)[] = []; + const events: (typeof heartbeatRunEvents.$inferInsert)[] = []; + const activity: (typeof activityLog.$inferInsert)[] = []; + const base = Date.now() - 30 * 24 * 3600_000; + for (let index = 0; index < count; index++) { + const human = index % 2 === 0, id = randomUUID(), runId = human ? null : randomUUID(); + const createdAt = new Date(base + index * 60_000); + let body = `History message ${index}. ` + (human ? "Please investigate these findings." : "The investigation is complete.\n\n- Checked the saved evidence.\n- Verified the result.\n\n" + "Saved findings and verification details. ".repeat(index % 7 + 2)); + if (options.delayedImage && index === count - 10) body += "\n\nInline evidence: ![Delayed evidence](/__chat-history-image.svg)"; + if (runId) { + runs.push({ id: runId, companyId: company.id, agentId: agent.id, issueId: issue.id, runtimeMode: "native", status: "succeeded", + createdAt: new Date(+createdAt - 1000), startedAt: new Date(+createdAt - 900), finishedAt: createdAt, contextSnapshot: { issueId: issue.id, conversationSessionGeneration: issue.conversationSessionGeneration, wakeCommentId: comments.at(-1)?.id } }); + events.push({ companyId: company.id, agentId: agent.id, runId, seq: 1, eventType: "item.completed", createdAt, + payload: { prpEvent: { schema: "paperclip.prp.event.v1", schemaVersion: 1, eventType: "item.completed", runId, + emittedAt: createdAt.toISOString(), itemId: `answer-${runId}`, payload: { kind: "agentMessage", text: body } } } }); + } + comments.push({ id, companyId: company.id, issueId: issue.id, authorType: human ? "user" : "agent", authorUserId: human ? issue.conversationUserId : null, + authorAgentId: human ? null : agent.id, createdByRunId: runId, body, createdAt, updatedAt: createdAt }); + activity.push({ companyId: company.id, actorType: human ? "user" : "agent", actorId: human ? issue.conversationUserId : agent.id, + agentId: human ? null : agent.id, runId, entityType: "issue", entityId: issue.id, action: "issue.comment_added", details: { commentId: id }, createdAt }); + } + try { + for (let start = 0; start < runs.length; start += 100) await db.insert(heartbeatRuns).values(runs.slice(start, start + 100)); + for (let start = 0; start < comments.length; start += 100) await db.insert(issueComments).values(comments.slice(start, start + 100)); + for (let start = 0; start < events.length; start += 100) await db.insert(heartbeatRunEvents).values(events.slice(start, start + 100)); + for (let start = 0; start < activity.length; start += 100) await db.insert(activityLog).values(activity.slice(start, start + 100)); + } finally { await closeRegisteredClients(databaseUrl); } + const fixture = { company, agent, issue, comments: comments.map(({ id, body }) => ({ id: id!, body })) }; + await fs.mkdir("test-results/agent-chat", { recursive: true }); + await fs.writeFile(file, JSON.stringify(fixture)); + return fixture; +} diff --git a/tests/perf/agent-chat/agent-chat-startup.spec.ts b/tests/perf/agent-chat/agent-chat-startup.spec.ts new file mode 100644 index 00000000000..f78e1ff86a1 --- /dev/null +++ b/tests/perf/agent-chat/agent-chat-startup.spec.ts @@ -0,0 +1,309 @@ +import fs from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { expect, test, type Page } from "@playwright/test"; +import { longChatFixture, type ChatFixture } from "./agent-chat-fixture"; + +let fixture: ChatFixture; +let chatPath: string; +const scroller = (page: Page) => page.getByTestId("task-chat-scroller"); +const editor = (page: Page) => page.getByTestId("task-chat-composer-stack").locator('[contenteditable="true"]').last(); +async function ready(page: Page) { + await expect(page.getByTestId("task-chat-thread").locator(':scope > [aria-busy="false"]')).toBeVisible({ timeout: 20_000 }); + await expect(editor(page)).toBeVisible(); +} +async function anchor(page: Page) { + return scroller(page).evaluate(root => { + const rect = root.getBoundingClientRect(); + const row = [...root.querySelectorAll("[data-thread-anchor]")].find(row => row.getBoundingClientRect().height && row.getBoundingClientRect().bottom > rect.top && row.getBoundingClientRect().top < rect.bottom)!; + return { id: row.dataset.threadAnchor!, top: row.getBoundingClientRect().top - rect.top }; + }); +} +async function offset(page: Page, id: string) { + return scroller(page).evaluate((root, id) => { + const row = [...root.querySelectorAll("[data-thread-anchor]")].find(row => row.dataset.threadAnchor === id)!; + return row.getBoundingClientRect().top - root.getBoundingClientRect().top; + }, id); +} +async function scrollUp(page: Page, distance = 100_000) { + await scroller(page).hover(); + const wheel = page.evaluate(() => new Promise(resolve => { + window.addEventListener("wheel", () => requestAnimationFrame(() => requestAnimationFrame(() => resolve())), { once: true, capture: true }); + })); + await page.mouse.wheel(0, -distance); + await wheel; + await scroller(page).evaluate(root => new Promise(resolve => { + let previous = root.scrollTop, stable = 0; + const sample = () => { + const next = root.scrollTop; + stable = next === previous ? stable + 1 : 0; + previous = next; + if (stable >= 3) resolve(); else requestAnimationFrame(sample); + }; + requestAnimationFrame(sample); + })); +} +test.beforeAll(async ({ request }) => { + const experimental = await request.patch("/api/instance/settings/experimental", { data: { enableAgentChat: true, enableStreamlinedUi: true } }); + expect(experimental.ok()).toBeTruthy(); + fixture = await longChatFixture(request); + const preference = await request.put(`/api/companies/${fixture.company.id}/workspace-preferences/me`, { data: { coordinatorWorkspaceEnabled: true } }); + expect(preference.ok()).toBeTruthy(); + const announcementResponse = await request.get("/api/announcements/current"); + const announcement = await announcementResponse.json(); + if (announcement) { + const dismissed = await request.post(`/api/announcements/${announcement.id}/dismiss`, { data: { companyId: fixture.company.id } }); + expect(dismissed.ok()).toBeTruthy(); + } + chatPath = `/${fixture.company.issuePrefix}/chats/${fixture.agent.id}`; +}); + +test("cold long chat reveals one recent window without full-history reads or redraws", async ({ page, request }, info) => { + const snapshot = await request.get(`/api/issues/${fixture.issue.id}/recent-comments`); + const latestId = (await snapshot.json()).latestCommentId; + const reads: string[] = [], failures: string[] = []; + page.on("request", request => { if (request.url().includes(`/api/issues/${fixture.issue.id}/`)) reads.push(request.url()); }); + page.on("pageerror", error => failures.push(error.message)); + await page.addInitScript(() => { + const frames: { height: number; latest: number; rows: number }[] = []; + Object.assign(window, { chatFrames: frames }); + const sample = () => { + const root = document.querySelector('[data-testid="task-chat-thread"] > [aria-busy="false"] [data-testid="task-chat-scroller"]'); + if (root) { + const rows = root.querySelectorAll('[id^="comment-"]'); + frames.push({ height: root.scrollHeight, latest: rows[rows.length - 1]?.getBoundingClientRect().top ?? 0, rows: rows.length }); + } + requestAnimationFrame(sample); + }; + requestAnimationFrame(sample); + }); + await page.goto(chatPath); + await ready(page); + await expect(page.locator('[id^="comment-"]')).toHaveCount(20); + await expect(page.locator(`[id="comment-${latestId}"]`)).toBeVisible(); + const reply = await page.locator(`[id="comment-${latestId}"]`).boundingBox(); + const composer = await page.getByTestId("task-chat-composer-dock").boundingBox(); + expect(reply!.y + reply!.height).toBeLessThanOrEqual(composer!.y + 2); + await expect(page.getByRole("button", { name: /^(More messages|Load earlier (comments|decisions|tasks))$/ })).toHaveCount(0); + await page.waitForTimeout(500); + expect(reads.filter(url => /\/comments\?/.test(url))).toEqual([]); + expect(reads.filter(url => /\/(activity|runs)$/.test(url))).toEqual([]); + expect(failures).toEqual([]); + const frames = await page.evaluate(() => (window as unknown as { chatFrames: { height: number; latest: number; rows: number }[] }).chatFrames); + expect(new Set(frames.map(frame => frame.rows)).size).toBe(1); + expect(Math.max(...frames.map(frame => frame.height)) - Math.min(...frames.map(frame => frame.height))).toBeLessThanOrEqual(2); + expect(Math.max(...frames.map(frame => frame.latest)) - Math.min(...frames.map(frame => frame.latest))).toBeLessThanOrEqual(2); + await fs.writeFile(info.outputPath("first-reveal.json"), JSON.stringify({ reads, frames })); + await page.screenshot({ path: info.outputPath("recent-chat.png") }); +}); + +test("upward scrolling loads one delayed page and preserves the row, viewport and draft", async ({ page }, info) => { + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let pageReads = 0; + await page.route(`**/api/issues/${fixture.issue.id}/comments?**`, async route => { + if (new URL(route.request().url()).searchParams.get("order") === "desc") { pageReads++; await held; } + await route.continue(); + }); + await page.goto(chatPath); await ready(page); + await editor(page).fill("Keep this draft while I read earlier messages."); + await editor(page).evaluate(element => element.setAttribute("data-retained-editor", "yes")); + try { + await scrollUp(page); + await expect(page.getByTestId("chat-history-status")).toHaveText("Loading earlier messages…"); + expect(pageReads).toBe(1); + const saved = await anchor(page); + await scrollUp(page, 100); + expect(pageReads).toBe(1); + release(); + await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + await expect(page.getByTestId("chat-history-status")).toHaveText("Scroll up for earlier messages"); + await expect.poll(() => offset(page, saved.id)).toBeCloseTo(saved.top, 0); + await page.waitForTimeout(300); + expect(Math.abs(await offset(page, saved.id) - saved.top)).toBeLessThanOrEqual(2); + await expect(editor(page)).toHaveAttribute("data-retained-editor", "yes"); + await expect(editor(page)).toContainText("Keep this draft"); + await page.screenshot({ path: info.outputPath("older-page.png") }); + } finally { release(); } +}); + +test("failed history stops automatic retries and leaves a working inline retry", async ({ page }) => { + let attempts = 0; + await page.route(`**/api/issues/${fixture.issue.id}/comments?**`, async route => { + if (++attempts === 1) await route.fulfill({ status: 500, json: { error: "Synthetic older-page failure" } }); + else await route.continue(); + }); + await page.goto(chatPath); await ready(page); + await editor(page).fill("A draft survives a history outage."); + await scrollUp(page); + await expect(page.getByTestId("chat-history-status")).toContainText("Earlier messages couldn’t load."); + const saved = await anchor(page); + await scrollUp(page, 100); + await page.waitForTimeout(300); + expect(attempts).toBe(1); + await expect(page.locator('[id^="comment-"]')).toHaveCount(20); + await page.getByTestId("chat-history-status").getByRole("button", { name: "Retry" }).click(); + await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + expect(Math.abs(await offset(page, saved.id) - saved.top)).toBeLessThanOrEqual(2); + await expect(editor(page)).toContainText("A draft survives"); +}); + +test("Back restores paged history and the reading position", async ({ page }) => { + await page.goto(chatPath); await ready(page); + await scrollUp(page); + await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + await scrollUp(page, 400); + const saved = await anchor(page); + const navigation = page.viewportSize()!.width < 768 + ? page.getByRole("navigation", { name: "Mobile navigation" }) + : page.getByRole("navigation", { name: "Workspace navigation" }); + await navigation.getByRole("link", { name: "Tasks", exact: true }).click(); + await expect(page).toHaveURL(/\/issues$/); + await page.goBack(); await ready(page); + await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + expect(Math.abs(await offset(page, saved.id) - saved.top)).toBeLessThanOrEqual(2); +}); + +test("an older comment link keeps compact loading and a usable target", async ({ page }) => { + const fullReads: string[] = []; + page.on("request", request => { if (/\/(comments|activity|runs)$/.test(new URL(request.url()).pathname) && !new URL(request.url()).search) fullReads.push(request.url()); }); + await page.goto(`${chatPath}#comment-${fixture.comments[5].id}`); await ready(page); + await expect(page.locator(`[id="comment-${fixture.comments[5].id}"]`)).toBeVisible(); + await expect(page.locator('[id^="comment-"]')).toHaveCount(21); + expect(fullReads).toEqual([]); +}); + +test("live updates keep an unpinned reader and draft, and the saved message appears once", async ({ page, request }) => { + await page.goto(chatPath); await ready(page); + await editor(page).fill("An unsent draft stays mounted during live refresh."); + await editor(page).evaluate(element => element.setAttribute("data-retained-editor", "yes")); + await scrollUp(page, 400); + const saved = await anchor(page); + const keyResponse = await request.post(`/api/agents/${fixture.agent.id}/keys`, { data: { name: "Disposable chat history regression" } }); + expect(keyResponse.status()).toBe(201); + const key = await keyResponse.json(); + const recentResponse = await request.get(`/api/issues/${fixture.issue.id}/recent-comments`); + const recentRuns = new Set((await recentResponse.json()).comments.map((row: { createdByRunId: string | null }) => row.createdByRunId)); + let sourceRunId: string | undefined; + for (const source of fixture.comments.filter((_, index) => index % 2 === 1).slice(0, 25)) { + const sourceResponse = await request.get(`/api/issues/${fixture.issue.id}/comments/${source.id}`); + const runId = (await sourceResponse.json()).createdByRunId; + if (runId && !recentRuns.has(runId)) { sourceRunId = runId; break; } + } + expect(sourceRunId).toBeTruthy(); + const body = `Live saved message ${randomUUID()}`; + const response = await request.post(`/api/issues/${fixture.issue.id}/comments`, { + headers: { authorization: `Bearer ${key.token}`, "x-paperclip-run-id": sourceRunId! }, data: { body }, + }); + await request.delete(`/api/agents/${fixture.agent.id}/keys/${key.id}`); + expect(response.ok(), await response.text()).toBeTruthy(); + const message = await response.json(); + await expect(page.locator(`[id="comment-${message.id}"]`)).toBeAttached({ timeout: 15_000 }); + expect(Math.abs(await offset(page, saved.id) - saved.top)).toBeLessThanOrEqual(2); + await expect(editor(page)).toHaveAttribute("data-retained-editor", "yes"); + await expect(editor(page)).toContainText("An unsent draft stays mounted"); + const persisted = await request.get(`/api/issues/${fixture.issue.id}/comments?order=desc&limit=50`); + expect((await persisted.json()).filter((row: { body: string }) => row.body === body)).toHaveLength(1); + await page.getByRole("button", { name: "Scroll to latest", exact: true }).click(); + await expect(page.locator(`[id="comment-${message.id}"]`)).toBeVisible(); +}); + +test("sending before the first upward scroll leaves earlier history reachable", async ({ page, request }, info) => { + const own = await longChatFixture(request, `send-first-${info.project.name}`); + const snapshot = await request.get(`/api/issues/${own.issue.id}/recent-comments`); + const oldest = (await snapshot.json()).oldestCommentId; + await page.goto(`/${own.company.issuePrefix}/chats/${own.agent.id}`); await ready(page); + const body = `Saved before paging ${randomUUID()}`; + await editor(page).fill(body); + const posted = page.waitForResponse(response => response.request().method() === "POST" + && new URL(response.url()).pathname === `/api/issues/${own.issue.id}/comments`); + await page.getByTestId("task-chat-composer-send").click(); + const receipt = await posted; + expect(receipt.ok(), await receipt.text()).toBeTruthy(); + await expect(page.getByTestId("chat-history-status")).toHaveText("Scroll up for earlier messages"); + const older = page.waitForResponse(response => new URL(response.url()).pathname === `/api/issues/${own.issue.id}/comments` + && new URL(response.url()).searchParams.get("order") === "desc"); + await scrollUp(page); + const pageResponse = await older; + expect(new URL(pageResponse.url()).searchParams.get("after")).toBe(oldest); + await expect(page.locator(`[id="comment-${own.comments.at(-21)!.id}"]`)).toBeAttached(); +}); + +test("a receipts outage leaves message paging available with one independent retry", async ({ page }) => { + await page.route("**/decision-receipts?*", route => route.fulfill({ status: 500, contentType: "application/json", body: '{"error":"Receipt fixture unavailable"}' })); + await page.goto(chatPath); await ready(page); + await expect(page.getByText("Decision updates couldn’t load.")).toBeVisible(); + await scrollUp(page); + await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + await expect(page.getByTestId("chat-history-status")).toHaveText("Scroll up for earlier messages"); + await expect(page.getByRole("button", { name: "Retry", exact: true })).toHaveCount(1); +}); + +test("successive older pages hydrate each message and run id only once", async ({ page, request }) => { + const fixture = await longChatFixture(request, "metadata"); + await request.put(`/api/companies/${fixture.company.id}/workspace-preferences/me`, { data: { coordinatorWorkspaceEnabled: true } }); + const chatPath = `/${fixture.company.issuePrefix}/chats/${fixture.agent.id}`; + const nativeIds: string[] = []; + page.on("response", response => { + const match = new URL(response.url()).pathname.match(/^\/api\/heartbeat-runs\/([^/]+)\/events$/); + if (match && response.status() === 200) nativeIds.push(match[1]); + }); + const seen: Record = { commentIds: [], runIds: [] }; + page.on("request", request => { + const url = new URL(request.url()); + if (!url.pathname.startsWith(`/api/issues/${fixture.issue.id}/`)) return; + for (const field of ["commentIds", "runIds"]) seen[field].push(...(url.searchParams.get(field)?.split(",") ?? [])); + }); + await page.goto(chatPath); await ready(page); + await page.waitForLoadState("networkidle"); + const initialNative = new Set(nativeIds); + await scrollUp(page); await expect(page.locator('[id^="comment-"]')).toHaveCount(70); + await expect.poll(() => new Set(nativeIds.filter(id => !initialNative.has(id))).size).toBe(25); + await page.waitForLoadState("networkidle"); + const secondPageStart = Object.fromEntries(Object.entries(seen).map(([key, ids]) => [key, ids.length])); + const firstPageNative = new Set(nativeIds); + const firstPageReads = nativeIds.length; + await scrollUp(page); await expect(page.locator('[id^="comment-"]')).toHaveCount(120); + await expect.poll(() => seen.commentIds.length - secondPageStart.commentIds).toBe(50); + await expect.poll(() => new Set(nativeIds.filter(id => !firstPageNative.has(id))).size).toBe(25); + await page.waitForLoadState("networkidle"); + expect(nativeIds.slice(firstPageReads).filter(id => firstPageNative.has(id))).toEqual([]); + for (const [key, ids] of Object.entries(seen)) { + const secondPage = ids.slice(secondPageStart[key]); + const previous = new Set(ids.slice(0, secondPageStart[key])); + expect(new Set(secondPage).size).toBe(secondPage.length); + expect(secondPage.filter(id => previous.has(id))).toEqual([]); + } +}); + +for (const pinned of [true, false]) test(`a delayed image preserves ${pinned ? "latest following" : "the reading anchor"}`, async ({ page, request }, info) => { + const own = await longChatFixture(request, `image-v3-${pinned}-${info.project.name}`, { delayedImage: true }); + const preference = await request.put(`/api/companies/${own.company.id}/workspace-preferences/me`, { data: { coordinatorWorkspaceEnabled: true } }); + expect(preference.ok()).toBeTruthy(); + let release!: () => void, requested!: () => void; + const held = new Promise(resolve => { release = resolve; }); + const pendingImage = new Promise(resolve => { requested = resolve; }); + await page.route("**/__chat-history-image.svg", async route => { + requested(); await held; + await route.fulfill({ contentType: "image/svg+xml", body: '' }); + }); + try { + await page.goto(`/${own.company.issuePrefix}/chats/${own.agent.id}`); await ready(page); + await scrollUp(page, 1200); + const images = await page.locator('img[alt="Delayed evidence"]').evaluateAll(nodes => nodes.map(node => ({ src: (node as HTMLImageElement).src, complete: (node as HTMLImageElement).complete, rect: node.getBoundingClientRect().toJSON() }))); + await fs.writeFile(info.outputPath("image-before-release.json"), JSON.stringify({ images })); + await pendingImage; + await page.getByRole("button", { name: "Scroll to latest", exact: true }).click(); + await expect.poll(() => scroller(page).evaluate(root => root.scrollHeight - root.scrollTop - root.clientHeight)).toBeLessThanOrEqual(2); + if (!pinned) await scrollUp(page, 200); + const saved = await anchor(page); + release(); + await expect(page.locator('img[alt="Delayed evidence"]')).toHaveJSProperty("complete", true); + await expect(page.locator('img[alt="Delayed evidence"]')).toHaveJSProperty("naturalWidth", 640); + await page.waitForTimeout(200); + if (pinned) { + const latest = await page.locator(`[id="comment-${own.comments.at(-1)!.id}"]`).boundingBox(); + const composer = await page.getByTestId("task-chat-composer-dock").boundingBox(); + expect(latest!.y + latest!.height).toBeLessThanOrEqual(composer!.y + 2); + } else expect(Math.abs(await offset(page, saved.id) - saved.top)).toBeLessThanOrEqual(2); + } finally { release(); } +}); diff --git a/tests/perf/agent-chat/playwright.config.ts b/tests/perf/agent-chat/playwright.config.ts new file mode 100644 index 00000000000..73582281dea --- /dev/null +++ b/tests/perf/agent-chat/playwright.config.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "@playwright/test"; +import base from "../issue-detail/playwright.config"; + +export default defineConfig({ + ...base, + testDir: ".", + testMatch: "agent-chat-startup.spec.ts", + timeout: 60_000, + workers: 1, + use: { ...base.use, serviceWorkers: "block" }, + projects: [ + { name: "desktop", use: { viewport: { width: 1440, height: 900 } } }, + { name: "mobile", use: { viewport: { width: 390, height: 844 }, isMobile: true, hasTouch: true } }, + ], + outputDir: "../../../test-results/agent-chat/playwright", +}); diff --git a/tests/perf/issue-detail/README.md b/tests/perf/issue-detail/README.md index 1077987582c..f344845ad8a 100644 --- a/tests/perf/issue-detail/README.md +++ b/tests/perf/issue-detail/README.md @@ -120,3 +120,7 @@ per-frame positions, layout shifts, browser timing counters, and reading-anchor measurement. The live script rejects non-local URLs and uses real provider capacity. Review the video as well as the assertions: layout-shift scores alone do not capture programmatic scrolling or replacement flashes. + +## Long-running agent chats + +The separate [agent-chat performance rig](../agent-chat/README.md) covers recent-only startup and scrolling history with long-lived conversations. It is opt-in and does not run in this task-detail rig. diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index a70961c74d9..3d237a790c1 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -29,7 +29,8 @@ and removes it with the isolated instance. The local native OpenCode profile uses the current production-qualified model and `OPENROUTER_API_KEY` through the existing encrypted company-secret fixture. -Each of the two journeys has a cell for each profile. All four cells are manual. +Each of the two journeys has a cell for each original profile. These four cells +remain unchanged; the TestOps profile below adds one separately named backlog cell. ```sh pnpm test:e2e:runner:typecheck @@ -40,13 +41,41 @@ pnpm test:e2e:runner -- --id chat-features.runner-codex-subscription.local.deleg pnpm test:e2e:runner -- --id chat-features.runner-opencode.local.delegate-announcement --max-automatic-retries 0 ``` -Run creation first and inspect its packaged report before delegation. For initial +Run creation first and inspect its packaged report before delegation. For the original Codex acceptance, run each cell three times against fresh fixtures on the same definition revision. Keep every attempt, including failures; a retry does not erase one. A fixture or product correction starts a new three-sample measurement. All cells are explicit-only and excluded from `--all`; this does not implement or qualify the rest of the YAML inventory. +## OpenCode TestOps integration acceptance + +The additional explicit cell +`chat-features.runner-opencode-deepseek-1017.local.create-backlog` runs the same backlog +query, public fixture and saved-state oracle on native OpenCode with +`openrouter/deepseek/deepseek-v4.1-flash-1017`. It requires only the selected +`OPENROUTER_API_KEY` profile credential. The launcher stores an encrypted secret +reference for the disposable agents; no key belongs in catalog data or chat. +This selected model is a qualification candidate. Other OpenCode suites keep +their existing model, and delegation is not registered for this profile. + +```sh +pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-automatic-retries 0 +``` + +Use a separately named TestOps integration-acceptance plan and launch for this +single cell. It retains one human request, one accounted controller work run, +120-second turns, a seven-minute attempt, the five-minute stable-state bound +and zero automatic retries. Morgan is created as fixture context but must not +execute the saved backlog task. Existing setup probes and unknown usage remain +reported separately. The original 244-scenario campaign, failed attempts and +Codex measurements are not reset or regraded by this measurement. + +The existing normalized result, JUnit and packaged report carry the outcome to +the execution integration. Catalog selection and provider-free oracle checks +do not establish an AWS model pass or a successful TestOps upload. Accept this +integration only after the real AWS execution and its TestOps result agree. + ## Bounds and grading Creation expects one accounted native work run; delegation expects three (CEO @@ -151,3 +180,71 @@ in the history. Across the original final-definition batch and this recheck, backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. + +## TestOps-backed AWS backlog cell + +The dedicated `Chat feature TestOps` workflow accepts only the shipped default +branch and the fixed OpenCode backlog cell. `evaluate` is the default operation +and requires TestOps' `ALLURE_JOB_RUN_ID`. A protected reporting job verifies the +project, singleton plan, assigned public workflow, selected case and current +GitHub run before any build or provider access. The evaluated process receives +no TestOps token. It retains the original one-query, one-work-run, 120-second +turn, 420-second attempt, 300-second settled-state and zero-retry limits. + +Two metadata operations prepare that route. `bootstrap` registers the actual +public CI job through pinned `allurectl` with an empty results directory and +reconciles only the dedicated case and singleton plan. It produces no passing +test result. After the registered job and protected case/plan/job identifiers +are verified, `launch-plan` sends one TestOps plan-run request and exits. +The TestOps job then dispatches `evaluate`; an operator must not dispatch a +second evaluation manually. Missing or ambiguous registration and selection +metadata stops before inference. + +The workflow reuses the existing AWS fleet and strict private evidence storage. +Its shipped-main authority is separate from manual candidate approval. Original +logs, provider records and failure evidence remain private. The reporting adapter +exports only a source/run-bound normalized result, JUnit and screenshots already +marked by the owned fixture screenshot producer with matching file hashes. +A separate protected reporting job converts that projection to the exact TestOps +case. Failed grades and cleanup failures remain failed. + +Reporting setup uses the master-only `runner-e2e-testops-reporting` environment, +with the existing reporting token delivered through the reviewed sealed-box +bootstrap. `CHAT_TESTOPS_CASE_ID`, `CHAT_TESTOPS_PLAN_ID` and +`CHAT_TESTOPS_JOB_ID` must name the observed dedicated mapping; the numeric +`CHAT_TESTOPS_GITHUB_WORKFLOW_ID` supports TestOps' registered workflow URL. +No arbitrary source, model, case, retry or budget input is accepted. + +The initial `launch-plan` administrator operation also requires a protected +`CHAT_TESTOPS_LAUNCH_RUN_NUMBER`. After registration, the trusted operator creates +this variable once with the reviewed next workflow run number and reads it back. +The launcher compares GitHub's actual run number, attempt, workflow, source and +actors before any TestOps mutation. A later dispatch or replay cannot reuse that +approved number. Missing or uncertain metadata stops; an uncertain request needs +reconciliation rather than a new send. The nonce and launch search provide +additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of +this one-time administrator fence. The operation parameter identifier and launch +nonce are also protected configuration, not dispatch inputs. + +`CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub +integration. The launcher verifies its project, type and enabled state, then binds +only the newly registered public job to it. It never changes integration settings +or credentials. Since the job response does not expose that association, only the +real TestOps-to-GitHub dispatch can qualify it. + +After `allurectl watch` finishes, the same protected reporter checks the actual +TestOps job-run, launch, selected case, result source/status and attachment +metadata. Processing reads are bounded to 12 polls, five-second gaps and a +120-second processing deadline. Unknown or missing results remain unverified. +The check downloads no attachment content and claims no server-side attachment +byte hash. Its closed receipt is retained separately. An upload success alone +is not a passing feature result, and verifying an original failure cannot turn +that failure into a pass. + +Build and worker installs use the existing reviewed report-lock policy. Only its +committed input or output lock is accepted; its authenticated correction is the +only allowed normalization. Dependency manifests, patch inputs and policy must +remain unchanged. Installs are frozen with lifecycle scripts disabled. The +credential-free build rechecks that boundary before packaging, and the worker +matches the source/lock receipt before installing. Arbitrary resolver drift is +rejected before provider access. diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index a335e98f910..77ece897599 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -5,7 +5,10 @@ only company, agents and project context through public APIs. Both agents use production role instructions; evaluator outcomes are never injected. Real browser submission, task audits, worker document provenance, completion delivery and reload readback establish the outcome. The existing fixture registry, subscription -credential staging and whole-company cleanup own setup and teardown. +credential staging and whole-company cleanup own setup and teardown. The explicit +OpenCode backlog cell instead uses an encrypted OpenRouter profile secret +reference. It preserves the same fixture and oracle without registering +OpenCode delegation or changing the other suites' model. [Deployment-targeted journeys](PRODUCT-JOURNEYS.md) resolve only their requested fixture graph through `FixtureRegistry.setupAll(requested)`. Omitting the diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 2eaea14febc..ef7abc55ee9 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -24,6 +24,7 @@ never start a server or depend on a tenant filesystem. | Journey | Required starting state | Observed outcome | | --- | --- | --- | | `task-create` | Company and signed-in board user | Create a task through the shared production composer, assign it to yourself, reopen and reload it, verify the saved prompt and no agent work | +| `project-task-context` | Company, signed-in board user, task-write capability; no model runs | Create a project and a self-assigned task with the project selected through the production dialogs. Reopen the project, open its task, reload both, and verify exact saved company, project, prompt, assignment, standard work mode and zero runs | | `ask-question` | Company, configured agent capability, positive model allowance and exactly one permitted work run | Reuse the existing cross-model Ask question prompt; require one successful run with matching company/task/agent/model/environment, one saved exact answer, and visible readback after reload | | `question-answer-resume` | Native agent; up to three work runs | Reuse the classic two-question story: answer a choice and a free-text question through the browser, verify each native continuation, then read the exact saved answers in an agent-authored task document after reload | | `task-properties` | Company, signed-in board user, task-write capability, prepared run-owned backlog task | Open its properties, assign it to the signed-in user, reload, check both UI and saved assignment; verify no agent run | @@ -52,6 +53,18 @@ The shared Studio helper waits for the original saved file body before typing. An editor control can mount before the file query completes; editing that empty draft can otherwise lose the original content when the file arrives. +The project/task journey records durable project and task ownership before each +creation and immediately saves the response IDs. Its project must retain its +planned state. Independent project-filtered task reads must contain the exact +new task. The final task view must show the same project after reload. Browser +readback uses the main project heading and visible task content rather +than a duplicate sidebar label. It binds navigation to the saved project URL key +and task identifier. Closed failure codes identify project-list, project-open, +task-open and property-readback failures without retaining Playwright page text. A Cloud +provider must clean only these owned resources, settle owned tasks before +archiving the project, and preserve their history. Catalog registration does not +admit this candidate to a saved daily plan; a real deployment pass is required. + The search/filter journey creates only campaign-owned, board-assigned tasks through the public API. The target and completed control share a unique search marker; a second Backlog control has a different marker. Search and status selection use diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index df1b315bccd..1e98adc848b 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -59,7 +59,7 @@ describe("runner E2E catalog", () => { }); it("exports deployment journeys separately from the model matrix and default selection", () => { - expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); + expect(productJourneys.map(journey => journey.id)).toEqual(["task-create", "project-task-context", "ask-question", "question-answer-resume", "task-properties", "task-search-filter", "plan-revise-accept", "build-revise", "protected-actions", "create-skill-studio", "stop-redirect", "cloud-onboarding"]); expect(runnerSuites.some(suite => suite.id === "product-journeys")).toBe(false); expect(runnerMatrix.some(execution => execution.suite.id === "product-journeys")).toBe(false); expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(execution => execution.suite.id === "product-journeys")).toBe(false); @@ -219,10 +219,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 4, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 2, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 2, 6, 8, 46, 2, 23, 15, 52, 6, 6, 20, 26, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(774); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(774); + expect(validateRunnerCatalog()).toHaveLength(775); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(775); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index c419d76d1bf..6c554afe028 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -26,7 +26,7 @@ import { copilotProviderDeathTasks } from "./copilot-active-stop-tasks.js"; import { instructionPersistenceTask } from "./instruction-persistence.js"; import { apiResponseReadingTask } from "./api-response-reading.js"; import { taskTitleTasks, taskTitleDefinitionDigest, TASK_TITLE_BUDGET_CENTS } from "./task-titles.js"; -import { CHAT_FEATURE_SUITE, CHAT_FEATURE_BUDGET_CENTS, chatFeatureTasks, chatFeatureProfile, chatFeatureScenarios, chatFeatureDefinitionDigest } from "./chat-feature-cases.js"; +import { CHAT_FEATURE_SUITE, CHAT_FEATURE_BUDGET_CENTS, CHAT_FEATURE_OPENCODE_MODEL, chatFeatureTasks, chatFeatureProfile, chatFeatureScenarios, chatFeatureDefinitionDigest } from "./chat-feature-cases.js"; import { blockerTasks, blockerProfile } from "./blocker-cases.js"; import { accountingTasks } from "./accounting-cases.js"; import { continuationTasks, questionResumeTask } from "./continuation-cases.js"; @@ -1609,10 +1609,14 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ id: CHAT_FEATURE_SUITE, label: "Chat feature journeys", manualOnly: true, description: "Natural catalog requests create backlog work or delegate an announcement and return its saved result.", groups: ["chat", "native"], - profiles: [nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription", - provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] }), - runnerProfiles.find(profile => profile.id === "runner-opencode")!].map(chatFeatureProfile), - environments: [localEnvironment], tasks: chatFeatureTasks, expectedMatrixSize: 4, + profiles: [chatFeatureProfile(nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription", + provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] })), + chatFeatureProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), + chatFeatureProfile(nativeProfile({ id: "runner-opencode-deepseek-1017", label: "Native OpenCode DeepSeek 1017", + provider: "opencode", model: CHAT_FEATURE_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"], + modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-feature-testops:deepseek-v4.1-flash-1017" } }))], + environments: [localEnvironment], tasks: chatFeatureTasks, expectedMatrixSize: 5, + excludedExecutionIds: ["chat-features.runner-opencode-deepseek-1017.local.delegate-announcement"], definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, scheduling: "explicit-only", instructions: "production-role-defaults", companyAndAgentBudgetCents: CHAT_FEATURE_BUDGET_CENTS, unpricedUsagePolicy: "allow-through-public-budget-policy", taskRunLimits: { ceo: 3, worker: 1 }, diff --git a/tests/runner-e2e/chat-feature-cases.ts b/tests/runner-e2e/chat-feature-cases.ts index fb2337bb447..3d5c66287dc 100644 --- a/tests/runner-e2e/chat-feature-cases.ts +++ b/tests/runner-e2e/chat-feature-cases.ts @@ -6,6 +6,7 @@ import type { RunnerProfileFixture, RunnerTaskFixture } from "./types.js"; export const CHAT_FEATURE_SUITE = "chat-features"; export const CHAT_FEATURE_BUDGET_CENTS = 500; export const CHAT_FEATURE_GRADER = "paperclip.chat-features.v2"; +export const CHAT_FEATURE_OPENCODE_MODEL = "openrouter/deepseek/deepseek-v4.1-flash-1017"; export const CHAT_FEATURE_CASES = ["create-backlog", "delegate-announcement"] as const; export type ChatFeatureCase = typeof CHAT_FEATURE_CASES[number]; export interface ChatScenario { @@ -48,7 +49,7 @@ export function chatFeatureProfile(profile: RunnerProfileFixture): RunnerProfile return { ...payload, name: "CEO", title: "CEO", role: "ceo", capabilities: "Coordinates company work and delegates tasks to the team.", budgetMonthlyCents: CHAT_FEATURE_BUDGET_CENTS, - adapterConfig: { ...adapterConfig, timeoutSec: 120, modelReasoningEffort: "low" }, + adapterConfig: { ...adapterConfig, timeoutSec: 120, ...(profile.id === "runner-opencode-deepseek-1017" ? {} : { modelReasoningEffort: "low" }) }, runtimeConfig: { ...(payload.runtimeConfig as Record), heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: CHAT_FEATURE_BUDGET_CENTS } }, }; diff --git a/tests/runner-e2e/chat-feature-scoring.test.ts b/tests/runner-e2e/chat-feature-scoring.test.ts index f76bec3ca6b..0eca4a8a9a1 100644 --- a/tests/runner-e2e/chat-feature-scoring.test.ts +++ b/tests/runner-e2e/chat-feature-scoring.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { chatFeatureExecutionRuns, gradeChatFeature, type ChatFeatureObservation } from "./chat-feature-scoring.js"; import { chatFeatureScenarios, readChatFeatureScenarios } from "./chat-feature-cases.js"; -import { runnerMatrix } from "./catalog.js"; +import { runnerMatrix, runnerProfiles } from "./catalog.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; const announcement = "Welcome to Acorn Notes, launching November 12! Join us with invitation code ACORN123."; @@ -107,9 +107,15 @@ describe("chat feature saved-state oracle", () => { }); describe("chat feature selection", () => { - it("registers the two authored queries as explicit local subscription and OpenCode cells", () => { - const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features"); + it("preserves the four authored subscription and production OpenCode cells", () => { + const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.profile.id !== "runner-opencode-deepseek-1017"); expect(cells).toHaveLength(4); + const stock = runnerProfiles.find(profile => profile.id === "runner-opencode")!; + expect(cells.filter(cell => cell.profile.id === stock.id).map(cell => cell.profile.model)).toEqual([stock.model, stock.model]); + for (const cell of cells.filter(cell => cell.profile.id === stock.id)) { + const payload = cell.profile.buildAgent({ environmentId: "env", environmentFixtureId: "local", workspacePath: "/tmp/fixture", executionId: cell.id, secretRefs: { OPENROUTER_API_KEY: { type: "secret_ref", secretId: "fixture-openrouter", version: "latest" } } }); + expect(payload.adapterConfig).toMatchObject({ model: stock.model, modelReasoningEffort: "low", timeoutSec: 120 }); + } for (const cell of cells) { expect(cell.environment.id).toBe("local"); expect(cell.requiredCredentials).toEqual([cell.profile.id === "runner-opencode" ? "OPENROUTER_API_KEY" : "CODEX_AUTH_JSON"]); @@ -119,6 +125,30 @@ describe("chat feature selection", () => { expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.id === cell.id)).toBe(false); } }); + it("selects one bounded OpenCode backlog cell with the requested model and secret reference", () => { + const cells = runnerMatrix.filter(cell => cell.suite.id === "chat-features" && cell.profile.id === "runner-opencode-deepseek-1017"); + expect(cells.map(cell => cell.id)).toEqual(["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]); + const cell = cells[0]!; + expect(cell.profile).toMatchObject({ provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", expectedRuntimeMode: "native" }); + expect(cell.profile.modelQualification.source).toBe("candidate_runner_profile"); + expect(cell.requiredCredentials).toEqual(["OPENROUTER_API_KEY"]); + expect(cell.task).toMatchObject({ expectedRunCount: 1, turnTimeoutMs: 120_000, + attemptTimeoutMs: { local: 420_000 }, automaticRetryPolicy: "single_attempt" }); + expect(cell.task.buildPrompt("nonce")).toBe(chatFeatureScenarios[0]!.query); + const secret = { type: "secret_ref", secretId: "fixture-openrouter", version: "latest" } as const; + const input = { environmentId: "env", environmentFixtureId: "local", workspacePath: "/tmp/fixture", executionId: "backlog" } as const; + const agent = cell.profile.buildAgent({ ...input, secretRefs: { OPENROUTER_API_KEY: secret } }); + expect(agent).toMatchObject({ role: "ceo", adapterType: "paperclip_runner", budgetMonthlyCents: 500, + adapterConfig: { provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", timeoutSec: 120, + env: { OPENROUTER_API_KEY: secret } }, + runtimeConfig: { heartbeat: { enabled: false, maxConcurrentRuns: 1, maxDailyRuns: 3, maxDailyCostCents: 500 } } }); + expect(agent).not.toHaveProperty("instructionsBundle"); + expect(agent.adapterConfig).not.toHaveProperty("modelReasoningEffort"); + expect(agent.adapterConfig).not.toHaveProperty("codexPermissionMode"); + expect(() => cell.profile.buildAgent({ ...input, secretRefs: {} })).toThrow("Missing fixture secret reference OPENROUTER_API_KEY"); + expect(selectRunnerExecutions(parseRunnerSelectors(["--id", cell.id])).map(row => row.id)).toEqual([cell.id]); + expect(selectRunnerExecutions(parseRunnerSelectors(["--all"])).some(row => row.id === cell.id)).toBe(false); + }); it("refuses missing, malformed or duplicate catalog entries", () => { for (const yaml of ["{", "schema_version: 2", "schema_version: 1\nfeature: tasks\nstatus: planned\nscenarios: []"]) expect(() => readChatFeatureScenarios(yaml)).toThrow(); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs new file mode 100644 index 00000000000..e77bbc4edac --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -0,0 +1,342 @@ +export const endpoint = 'https://paperclip.testops.cloud'; +export const planName = 'Chat feature evals'; +export const fullName = 'paperclip.Product E2E.chat-features.create-backlog'; +export const aliases = [fullName, 'paperclip.Product E2E.chat-features.tasks.create-backlog']; +class SetupFailure extends Error { + constructor(code, status = null) { super(code); this.code = code; this.status = status; } +} +const fail = code => { + const match = /^(testops_(?:api|auth)|chat_testops_github)_http_([1-5][0-9]{2})$/.exec(code); + throw new SetupFailure(match ? `${match[1]}_http` : code, match ? Number(match[2]) : null); +}; +const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_token_missing', 'testops_auth_http', + 'testops_api_http', 'testops_auth_invalid', 'testops_route_invalid', 'testops_response_too_large', + 'testops_response_invalid', 'testops_metadata_page_incomplete', 'testops_case_identity_invalid', + 'testops_case_alias_conflict', 'testops_plan_name_conflict', 'testops_plan_identity_invalid', + 'testops_integration_identity_invalid', 'testops_bootstrap_identity_invalid', 'testops_plan_without_matching_case', + 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', + 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', + 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', + 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); +export function failureProjection(error, stage) { + const stages = ['context', 'output_admission', 'authentication', 'metadata_inspection', 'plan_bootstrap', 'selection_validation', 'launch_admission', 'launch_preflight', 'plan_launch', 'bootstrap_job_metadata', 'result_export', 'upload_verification', 'receipt']; + return { stage: stages.includes(stage) ? stage : 'unknown', + code: error instanceof SetupFailure && diagnosticCodes.has(error.code) ? error.code : 'unknown_failure', + httpStatus: error instanceof SetupFailure && Number.isInteger(error.status) && error.status >= 100 && error.status < 600 ? error.status : null }; +} +const id = value => Number.isSafeInteger(value) && value > 0; +const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); + +export async function testopsClient(token, fetchImpl = fetch) { + if (!token) fail('testops_token_missing'); + const auth = await fetchImpl(`${endpoint}/api/uaa/oauth/token`, { method: 'POST', redirect: 'error', + headers: { Accept: 'application/json' }, body: new URLSearchParams({ grant_type: 'apitoken', scope: 'openid', token }), + signal: AbortSignal.timeout(30_000) }); + if (!auth.ok) fail(`testops_auth_http_${auth.status}`); + const value = await auth.json(); + if (typeof value.access_token !== 'string' || !value.access_token) fail('testops_auth_invalid'); + return async (route, method = 'GET', body, timeoutMs = 30_000) => { + if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30_000) fail('chat_testops_upload_unverified'); + if (!route.startsWith('/api/') || route.includes('://')) fail('testops_route_invalid'); + const response = await fetchImpl(`${endpoint}${route}`, { method, redirect: 'error', + headers: { Accept: 'application/json', Authorization: `Bearer ${value.access_token}`, ...(body ? { 'Content-Type': 'application/json' } : {}) }, + ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(timeoutMs) }); + if (!response.ok) fail(`testops_api_http_${response.status}`); + if (response.status === 204) return null; + const text = await response.text(); + if (Buffer.byteLength(text) > 2_000_000) fail('testops_response_too_large'); + try { return JSON.parse(text); } catch { fail('testops_response_invalid'); } + }; +} + +const rows = value => { + if (!Array.isArray(value?.content) || value.content.length > 100 || value.totalPages > 1) fail('testops_metadata_page_incomplete'); + return value.content; +}; +export async function inspectChatMetadata(api) { + const cases = new Map(), catalogIds = new Set(); + let totalPages, totalElements; + for (let page = 0; page < 10; page++) { + const catalog = await api(`/api/testcase?${new URLSearchParams({ projectId: '1', page: String(page), size: '100', sort: 'id,asc' })}`); + if (!Array.isArray(catalog?.content) || catalog.content.length > 100 || catalog.number !== page + || !Number.isSafeInteger(catalog.totalPages) || catalog.totalPages < 0 || catalog.totalPages > 10 + || !Number.isSafeInteger(catalog.totalElements) || catalog.totalElements < 0 || catalog.totalElements > 1000 + || totalPages !== undefined && (totalPages !== catalog.totalPages || totalElements !== catalog.totalElements)) fail('chat_testops_catalog_incomplete'); + totalPages = catalog.totalPages; totalElements = catalog.totalElements; + const ids = catalog.content.map(row => { + if (!id(row.id) || catalogIds.has(row.id)) fail('chat_testops_catalog_incomplete'); + catalogIds.add(row.id); return row.id; + }); + for (let offset = 0; offset < ids.length; offset += 4) { + const batch = ids.slice(offset, offset + 4); + const details = await Promise.all(batch.map(value => api(`/api/testcase/${value}`))); + for (const [index, value] of details.entries()) { + if (value?.id !== batch[index] || value.projectId !== 1 || value.fullName != null && typeof value.fullName !== 'string') fail('testops_case_identity_invalid'); + if (!aliases.includes(value.fullName)) continue; + if (value.deleted === true || typeof value.automated !== 'boolean') fail('testops_case_identity_invalid'); + cases.set(value.id, { id: value.id, fullName: value.fullName, automated: value.automated }); + } + } + if (page + 1 >= totalPages) break; + if (page === 9) fail('chat_testops_catalog_incomplete'); + } + if (catalogIds.size !== totalElements) fail('chat_testops_catalog_incomplete'); + const plans = rows(await api(`/api/testplan?${new URLSearchParams({ projectId: '1', name: planName, page: '0', size: '100' })}`)) + .filter(value => value.name === planName); + if (plans.length > 1) fail('testops_plan_name_conflict'); + if (plans.some(value => !id(value.id) || value.projectId !== 1)) fail('testops_plan_identity_invalid'); + const integrations = rows(await api('/api/integration/project/1?page=0&size=100')) + .filter(value => value.info?.type === 'github') + .map(value => ({ id: value.id, type: 'github', disabled: typeof value.disabled === 'boolean' ? value.disabled : null })); + if (integrations.some(value => !id(value.id))) fail('testops_integration_identity_invalid'); + return { projectId: 1, caseConflict: cases.size > 1, cases: [...cases.values()], plans: plans.map(value => ({ id: value.id, name: planName, count: value.testCasesCount })), integrations }; +} + +export async function ensureChatPlan(api, inspected, markOnce) { + if (inspected.projectId !== 1 || inspected.plans.length > 1) fail('testops_bootstrap_identity_invalid'); + if (inspected.caseConflict === true || inspected.cases.length > 1) fail('testops_case_alias_conflict'); + let selected = inspected.cases[0], plan = inspected.plans[0]; + if (plan && !selected) fail('testops_plan_without_matching_case'); + if (!selected || !plan) await markOnce(); + if (!selected) { + const created = await api('/api/testcase', 'POST', { projectId: 1, name: 'Create a task backlog through chat', fullName, + automated: true, description: 'Canonical scenario: tasks.create-backlog. Readiness and recorded outcomes remain separate.', + expectedResult: 'The real agent creates the requested backlog; public Product E2E assertions and cleanup must pass.' }); + if (!id(created?.id) || created.projectId !== 1 || created.fullName !== fullName) fail('testops_created_case_identity_invalid'); + selected = { id: created.id, fullName, automated: created.automated === true }; + } + if (!plan) { + const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, + treeSelection: { inverted: false, leafsInclude: [selected.id] } }); + if (!id(created?.id) || created.projectId !== 1 || created.name !== planName) fail('testops_created_plan_identity_invalid'); + plan = { id: created.id, name: planName }; + } + const actual = await api(`/api/testplan/${plan.id}`); + if (actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== 1 + || actual.treeSelection?.inverted === true || !same(actual.treeSelection?.leafsInclude, [selected.id]) + || (actual.treeSelection?.groupsInclude?.length ?? 0) > 0 || (actual.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('testops_plan_scope_mismatch'); + return { projectId: 1, case: selected, plan: { id: plan.id, name: planName, count: 1 }, + caseUrl: `${endpoint}/project/1/test-cases/${selected.id}`, planUrl: `${endpoint}/testplan/${plan.id}`, + actualResult: null, note: 'Plan preparation is not an executed measurement or a passing result.' }; +} + + +export const executionId = 'chat-features.runner-opencode-deepseek-1017.local.create-backlog'; +export const scenario = 'tasks.create-backlog'; +const positive = value => typeof value === 'string' && /^[1-9][0-9]{0,15}$/.test(value) && Number.isSafeInteger(Number(value)); +export function publicContext(env) { + if (env.GITHUB_REPOSITORY !== 'paperclipai/paperclip' || env.GITHUB_REF !== 'refs/heads/master' + || env.GITHUB_EVENT_NAME !== 'workflow_dispatch' || env.GITHUB_RUN_ATTEMPT !== '1' + || env.GITHUB_WORKFLOW_REF !== 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? '') || !positive(env.GITHUB_RUN_ID)) fail('untrusted_public_testops_context'); + return { runId: env.GITHUB_RUN_ID, attempt: '1', workflowSha: env.GITHUB_SHA, targetSha: env.GITHUB_SHA }; +} + +export async function validateChatSelection(api, env) { + const context = publicContext(env); + const names = ['ALLURE_JOB_RUN_ID', 'CHAT_TESTOPS_CASE_ID', 'CHAT_TESTOPS_PLAN_ID', 'CHAT_TESTOPS_JOB_ID']; + if (names.some(name => !positive(env[name]))) fail('chat_testops_identity_missing'); + const [caseId, planId, jobId, jobRunId] = ['CHAT_TESTOPS_CASE_ID', 'CHAT_TESTOPS_PLAN_ID', 'CHAT_TESTOPS_JOB_ID', 'ALLURE_JOB_RUN_ID'].map(name => Number(env[name])); + const selected = await api(`/api/testcase/${caseId}`); + if (selected.id !== caseId || selected.projectId !== 1 || !aliases.includes(selected.fullName) || selected.deleted === true || selected.automated !== true) fail('chat_testops_case_mismatch'); + const plan = await api(`/api/testplan/${planId}`); + if (plan.id !== planId || plan.projectId !== 1 || plan.name !== planName || plan.testCasesCount !== 1 + || plan.treeSelection?.inverted === true || !same(plan.treeSelection?.leafsInclude, [caseId]) + || (plan.treeSelection?.groupsInclude?.length ?? 0) > 0 || (plan.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('chat_testops_plan_mismatch'); + const assignments = await api(`/api/testplan/${planId}/job`); + if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0].job?.id !== jobId || assignments[0].testCasesCount !== 1) fail('chat_testops_plan_job_mismatch'); + const job = await api(`/api/job/${jobId}`); + const jobUrls = ['https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml']; + if (positive(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID)) jobUrls.push(`https://github.com/paperclipai/paperclip/actions/workflows/${env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID}`); + if (job.id !== jobId || job.type !== 'github' || !jobUrls.includes(job.url)) fail('chat_testops_job_mismatch'); + const run = await api(`/api/jobrun/${jobRunId}`); + if (run.id !== jobRunId || run.job?.id !== jobId || run.job?.type !== 'github' + || !['scheduled', 'in_progress'].includes(run.stage) || !id(run.launchId) + || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`) fail('chat_testops_jobrun_mismatch'); + const launch = await api(`/api/launch/${run.launchId}`); + if (launch.id !== run.launchId || launch.projectId !== 1 || launch.closed === true) fail('chat_testops_launch_mismatch'); + const selection = await api(`/api/jobrun/${jobRunId}/plan?expected=true`); + if (!Array.isArray(selection) || selection.length !== 1 || selection[0].id !== caseId || selection[0].selector !== selected.fullName) fail('chat_testops_selection_mismatch'); + return { schema: 'paperclip.chat-feature-testops-selection/v1', ...context, allureJobRunId: String(jobRunId), + executionId, canonicalScenario: scenario, projectId: 1, planId, jobId, caseId, fullName: selected.fullName, launchId: run.launchId }; +} + +function configuredIds(env) { + const names = ['CHAT_TESTOPS_CASE_ID', 'CHAT_TESTOPS_PLAN_ID', 'CHAT_TESTOPS_JOB_ID', 'CHAT_TESTOPS_OPERATION_PARAMETER_ID', 'CHAT_TESTOPS_INTEGRATION_ID']; + if (names.some(name => !positive(env[name]))) fail('chat_testops_identity_missing'); + return Object.fromEntries(names.map((name, index) => [['caseId', 'planId', 'jobId', 'parameterId', 'integrationId'][index], Number(env[name])])); +} +function publicJobUrl(url, env) { + return url === 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' + || positive(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID) && url === `https://github.com/paperclipai/paperclip/actions/workflows/${env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID}`; +} +async function registeredPlan(api, env) { + const ids = configuredIds(env); + const selected = await api(`/api/testcase/${ids.caseId}`); + if (selected.id !== ids.caseId || selected.projectId !== 1 || !aliases.includes(selected.fullName) || selected.deleted === true || selected.automated !== true) fail('chat_testops_case_mismatch'); + const plan = await api(`/api/testplan/${ids.planId}`); + if (plan.id !== ids.planId || plan.projectId !== 1 || plan.name !== planName || plan.testCasesCount !== 1 + || plan.treeSelection?.inverted === true || !same(plan.treeSelection?.leafsInclude, [ids.caseId]) + || (plan.treeSelection?.groupsInclude?.length ?? 0) > 0 || (plan.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('chat_testops_plan_mismatch'); + const job = await api(`/api/job/${ids.jobId}`); + if (job.id !== ids.jobId || job.type !== 'github' || !publicJobUrl(job.url, env)) fail('chat_testops_job_mismatch'); + const integration = await api(`/api/integration/${ids.integrationId}/project/1`); + if (integration.id !== ids.integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) fail('chat_testops_integration_mismatch'); + const definition = await api(`/api/environment/${ids.parameterId}`); + if (definition.id !== ids.parameterId || definition.name !== 'operation' || !Array.isArray(job.parameters) || job.parameters.length !== 1 + || job.parameters[0].id !== ids.parameterId || !['bootstrap', 'evaluate'].includes(job.parameters[0].value)) fail('chat_testops_parameter_mismatch'); + return { ...ids, selected, plan, job }; +} + +export async function bootstrapJobMetadata(api, env) { + if (!positive(env.ALLURE_JOB_RUN_ID)) return { status: 'official_job_run_id_missing', actualResult: null }; + const run = await api(`/api/jobrun/${env.ALLURE_JOB_RUN_ID}`); + if (!id(run?.job?.id)) fail('chat_testops_jobrun_mismatch'); + const job = await api(`/api/job/${run.job.id}`); + if (job.id !== run.job.id || job.type !== 'github' || !Array.isArray(job.parameters) || job.parameters.length > 4) fail('chat_testops_job_mismatch'); + const parameters = []; + for (const parameter of job.parameters) { + if (!id(parameter.id)) fail('chat_testops_parameter_mismatch'); + const definition = await api(`/api/environment/${parameter.id}`); + const known = ['operation', 'ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME'].includes(definition.name); + parameters.push({ id: parameter.id, name: known ? definition.name : 'UNKNOWN', + value: definition.name === 'operation' && ['bootstrap', 'evaluate', 'launch-plan'].includes(parameter.value) ? parameter.value : 'NOT_PROJECTED' }); + } + return { status: 'observed', projectId: 1, jobId: job.id, allureJobRunId: env.ALLURE_JOB_RUN_ID, + publicWorkflowMatches: publicJobUrl(job.url, env), currentRunUrlMatches: run.url === `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`, + launchId: id(run.launchId) ? run.launchId : null, parameters, actualResult: null }; +} + +const launchAdmissions = new WeakSet(); +export async function admitLaunchPlan(env, fetchImpl = fetch) { + const source = publicContext(env); + const expected = env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER, actual = env.GITHUB_RUN_NUMBER; + if (!positive(expected) || !positive(actual) || actual !== expected || !positive(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID) + || !positive(env.GITHUB_ACTOR_ID) || !positive(env.TESTOPS_TRIGGERING_ACTOR_ID) || !env.GH_TOKEN) fail('chat_testops_launch_admission'); + let allowed; + try { allowed = JSON.parse(env.TESTOPS_ALLOWED_ACTOR_IDS ?? 'null'); } catch { fail('chat_testops_launch_admission'); } + if (!Array.isArray(allowed) || !allowed.length || allowed.some(value => !id(value)) || !allowed.includes(Number(env.GITHUB_ACTOR_ID)) || !allowed.includes(Number(env.TESTOPS_TRIGGERING_ACTOR_ID))) fail('chat_testops_launch_admission'); + const read = async route => { + const response = await fetchImpl(`https://api.github.com/repos/paperclipai/paperclip${route ? `/${route}` : ''}`, { + method: 'GET', redirect: 'error', headers: { Accept: 'application/vnd.github+json', Authorization: `Bearer ${env.GH_TOKEN}` }, + signal: AbortSignal.timeout(30_000) }); + if (!response.ok) fail(`chat_testops_github_http_${response.status}`); + const text = await response.text(); + if (Buffer.byteLength(text) > 512_000) fail('chat_testops_launch_admission'); + try { return JSON.parse(text); } catch { fail('chat_testops_launch_admission'); } + }; + const repository = await read(''); + const workflow = await read('actions/workflows/chat-feature-testops.yml'); + const run = await read(`actions/runs/${source.runId}`); + if (repository.full_name !== 'paperclipai/paperclip' || repository.private !== false || repository.default_branch !== 'master' + || workflow.id !== Number(env.CHAT_TESTOPS_GITHUB_WORKFLOW_ID) || workflow.path !== '.github/workflows/chat-feature-testops.yml' || workflow.state !== 'active' + || run.id !== Number(source.runId) || run.workflow_id !== workflow.id || run.run_attempt !== 1 || run.run_number !== Number(expected) + || run.head_sha !== source.workflowSha || run.head_branch !== 'master' || run.event !== 'workflow_dispatch' + || run.repository?.full_name !== 'paperclipai/paperclip' || run.repository?.private !== false + || run.actor?.id !== Number(env.GITHUB_ACTOR_ID) || run.triggering_actor?.id !== Number(env.TESTOPS_TRIGGERING_ACTOR_ID) || !allowed.includes(run.triggering_actor.id)) fail('chat_testops_launch_admission'); + const receipt = Object.freeze({ schema: 'paperclip.chat-feature-testops-launch-admission/v1', ...source, + workflowId: workflow.id, runNumber: run.run_number, expectedRunNumber: Number(expected), actorId: run.actor.id, triggeringActorId: run.triggering_actor.id }); + launchAdmissions.add(receipt); return receipt; +} + +export async function launchChatPlan(api, env, markOnce, admission) { + const source = publicContext(env); + if (!admission || !launchAdmissions.has(admission) || admission.runId !== source.runId || admission.workflowSha !== source.workflowSha + || admission.targetSha !== source.targetSha || String(admission.expectedRunNumber) !== env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER + || String(admission.runNumber) !== env.GITHUB_RUN_NUMBER) fail('chat_testops_launch_admission'); + const nonce = env.CHAT_TESTOPS_LAUNCH_NONCE; + if (!/^[a-f0-9]{32}$/.test(nonce ?? '')) fail('chat_testops_nonce_invalid'); + const known = await registeredPlan(api, env); + const assignments = await api(`/api/testplan/${known.planId}/job`); + if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row.job?.id !== known.jobId || row.testCasesCount !== 1)) fail('chat_testops_plan_job_mismatch'); + const launchName = `${planName} · ${nonce}`; + const existing = rows(await api(`/api/launch?${new URLSearchParams({ projectId: '1', search: launchName, page: '0', size: '100' })}`)); + if (existing.length) { + if (existing.length !== 1 || existing[0].name !== launchName || !id(existing[0].id)) fail('chat_testops_nonce_conflict'); + return { schema: 'paperclip.chat-feature-testops-launch/v1', ...source, nonce, planId: known.planId, + jobId: known.jobId, caseId: known.caseId, launchId: existing[0].id, status: 'EXISTING_NO_RESEND', actualResult: null }; + } + await markOnce({ ...source, launchAdmission: admission, nonce, planId: known.planId, jobId: known.jobId, caseId: known.caseId }); + const parameters = [{ id: known.parameterId, value: 'evaluate' }]; + await api(`/api/job/${known.jobId}`, 'PATCH', { integrationId: known.integrationId, canRun: true, parameters }); + const patched = await api(`/api/job/${known.jobId}`); + if (patched.id !== known.jobId || patched.type !== 'github' || !publicJobUrl(patched.url, env) || !same(patched.parameters, parameters)) fail('chat_testops_parameter_mismatch'); + if (!assignments.length) await api(`/api/testplan/${known.planId}/assign`, 'POST', { jobId: known.jobId, + selection: { inverted: false, manual: false, leafsInclude: [known.caseId] } }); + await api(`/api/testplan/${known.planId}/jobparameter`, 'POST', { jobId: known.jobId, parameters }); + const bound = await api(`/api/testplan/${known.planId}/job`); + if (!Array.isArray(bound) || bound.length !== 1 || bound[0].job?.id !== known.jobId || bound[0].testCasesCount !== 1 + || !same(bound[0].job.parameters, parameters)) fail('chat_testops_plan_job_mismatch'); + const launched = await api(`/api/testplan/${known.planId}/run`, 'POST', { launchName }); + if (!id(launched?.id) || launched.projectId !== 1 || launched.name !== launchName) fail('chat_testops_launch_mismatch'); + return { schema: 'paperclip.chat-feature-testops-launch/v1', ...source, nonce, projectId: 1, + planId: known.planId, jobId: known.jobId, caseId: known.caseId, launchId: launched.id, + status: 'REQUESTED', integrationId: known.integrationId, integrationBindingRequested: true, integrationAssociationReadback: 'not_exposed_by_job_dto', executionId, canonicalScenario: scenario, actualRunId: null, actualResult: null }; +} + + +export async function verifyChatUpload(api, env, selected, exported, { wait = ms => new Promise(resolve => setTimeout(resolve, ms)), now = Date.now } = {}) { + const context = publicContext(env); + if (selected.runId !== context.runId || selected.attempt !== '1' || selected.workflowSha !== context.workflowSha + || selected.targetSha !== context.targetSha || selected.allureJobRunId !== env.ALLURE_JOB_RUN_ID + || selected.executionId !== executionId || selected.canonicalScenario !== scenario || selected.projectId !== 1 + || !aliases.includes(selected.fullName) || ['planId', 'jobId', 'caseId', 'launchId'].some(key => !id(selected[key])) + || exported.caseId !== selected.caseId || exported.campaignId !== `gha-${context.runId}-1` || exported.count !== 1 + || !['passed', 'failed'].includes(exported.status) || !['passed', 'failed', 'not_started'].includes(exported.cleanup) + || exported.status === 'passed' && exported.cleanup !== 'passed' || !Array.isArray(exported.attachmentMetadata) + || exported.attachmentMetadata.length < 2 || exported.attachmentMetadata.length > 10) fail('chat_testops_upload_mismatch'); + const allowedAttachments = new Map([['Synthetic task and source conversation', 'image/png'], + ['Execution provenance and checks', 'application/json'], ['Sanitized Product E2E record and accounting', 'application/json']]); + if (exported.attachmentMetadata.some(row => allowedAttachments.get(row.name) !== row.contentType + || !Number.isSafeInteger(row.contentLength) || row.contentLength <= 0 || row.contentLength > 12 * 1024 * 1024) + || exported.attachmentMetadata.filter(row => row.name === 'Execution provenance and checks').length !== 1 + || exported.attachmentMetadata.filter(row => row.name === 'Sanitized Product E2E record and accounting').length !== 1 + || exported.status === 'passed' && !exported.attachmentMetadata.some(row => row.contentType === 'image/png')) fail('chat_testops_upload_mismatch'); + const signature = rows => rows.map(row => JSON.stringify([row.name, row.contentType, row.contentLength])).sort(); + const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`; + const sourceUrl = `https://github.com/paperclipai/paperclip/commit/${context.targetSha}`; + const deadline = now() + 120_000; + const get = async route => { + const remaining = deadline - now(); + if (remaining < 1) fail('chat_testops_upload_unverified'); + const value = await api(route, 'GET', undefined, Math.min(30_000, Math.floor(remaining))); + if (now() >= deadline) fail('chat_testops_upload_unverified'); + return value; + }; + for (let poll = 0; poll < 12 && now() < deadline; poll++) { + const run = await get(`/api/jobrun/${selected.allureJobRunId}`); + if (run.id !== Number(selected.allureJobRunId) || run.job?.id !== selected.jobId || run.job?.type !== 'github' + || run.launchId !== selected.launchId || run.url !== runUrl || !['in_progress', 'finished'].includes(run.stage)) fail('chat_testops_jobrun_mismatch'); + const launch = await get(`/api/launch/${selected.launchId}`); + if (launch.id !== selected.launchId || launch.projectId !== 1) fail('chat_testops_launch_mismatch'); + const listed = await get(`/api/testresult?${new URLSearchParams({ launchId: String(selected.launchId), page: '0', size: '100' })}`); + if (!Array.isArray(listed?.content) || listed.content.length > 1 || !Number.isSafeInteger(listed.totalElements) + || listed.totalElements !== listed.content.length || !Number.isSafeInteger(listed.totalPages) || listed.totalPages < 0 || listed.totalPages > 1) fail('chat_testops_upload_mismatch'); + if (listed.content.length) { + const row = listed.content[0]; + if (!id(row.id) || row.testCaseId !== selected.caseId) fail('chat_testops_upload_mismatch'); + const result = await get(`/api/testresult/${row.id}`); + if (result.id !== row.id || result.projectId !== 1 || result.launchId !== selected.launchId || result.testCaseId !== selected.caseId + || result.fullName !== selected.fullName || result.jobRun?.id !== Number(selected.allureJobRunId) + || result.jobRun?.job?.id !== selected.jobId || result.jobRun?.launchId !== selected.launchId || result.jobRun?.url !== runUrl + || !Array.isArray(result.links) || !result.links.some(link => link.url === runUrl) + || !result.links.some(link => link.url === sourceUrl)) fail('chat_testops_upload_mismatch'); + if (!['passed', 'failed', 'broken', 'skipped', 'unknown'].includes(result.status)) fail('chat_testops_upload_mismatch'); + if (result.status !== 'unknown' && result.status !== exported.status) fail('chat_testops_upload_mismatch'); + const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(row.id), page: '0', size: '100' })}`); + if (!Array.isArray(attached?.content) || attached.content.length > 10 || !Number.isSafeInteger(attached.totalElements) + || attached.totalElements !== attached.content.length || !Number.isSafeInteger(attached.totalPages) || attached.totalPages < 0 || attached.totalPages > 1 + || new Set(attached.content.map(value => value.id)).size !== attached.content.length + || attached.content.some(value => !id(value.id) || value.missed === true || value.fromTestCase === true)) fail('chat_testops_upload_mismatch'); + if (now() < deadline && result.status === exported.status && same(signature(attached.content), signature(exported.attachmentMetadata))) { + return { schema: 'paperclip.chat-feature-testops-upload-verification/v1', ...context, projectId: 1, planId: selected.planId, + jobId: selected.jobId, allureJobRunId: selected.allureJobRunId, launchId: selected.launchId, caseId: selected.caseId, + resultId: result.id, executionId, status: result.status, cleanup: exported.cleanup, verification: 'VERIFIED', + attachments: exported.attachmentMetadata.map(value => ({ ...value })), + launchUrl: `${endpoint}/launch/${selected.launchId}`, resultUrl: `${endpoint}/testresult/${result.id}` }; + } + } + if (poll < 11 && now() + 5000 < deadline) await wait(5000); + } + fail('chat_testops_upload_unverified'); +} diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs new file mode 100644 index 00000000000..a64344d25cb --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -0,0 +1,392 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { endpoint, planName, fullName, inspectChatMetadata, ensureChatPlan, testopsClient, failureProjection } from './chat-feature-testops-api.mjs'; + +const caseRow = { id: 91, projectId: 1, fullName, automated: true }; +const plan = { id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { inverted: false, leafsInclude: [91] } }; +const page = content => ({ content, totalPages: 1 }); +const inspectApi = (cases = [caseRow], plans = [plan]) => async route => { + if (route.startsWith('/api/testcase?')) return { content: cases.map(row => ({ id: row.id, name: 'Not exported', automated: row.automated })), totalElements: cases.length, totalPages: cases.length ? 1 : 0, number: 0 }; + if (/^\/api\/testcase\/[0-9]+$/.test(route)) return cases.find(row => row.id === Number(route.split('/').at(-1))); + if (route.startsWith('/api/testplan?')) return page(plans); + if (route.startsWith('/api/integration/project/1')) return page([{ id: 2, info: { type: 'github' }, disabled: false, settings: { secret: 'DO_NOT_EXPORT' } }]); + assert.fail(route); +}; + +test('inspect reconciles exact historical case identity and discards integration settings', async () => { + const result = await inspectChatMetadata(inspectApi()); + assert.deepEqual(result.cases, [{ id: 91, fullName, automated: true }]); + assert.deepEqual(result.integrations, [{ id: 2, type: 'github', disabled: false }]); + assert.ok(!JSON.stringify(result).includes('DO_NOT_EXPORT')); +}); + +test('multiple alias identities, foreign project and incomplete metadata stop before mutation', async () => { + const conflicts = await inspectChatMetadata(inspectApi([caseRow, { ...caseRow, id: 92, fullName: 'paperclip.Product E2E.chat-features.tasks.create-backlog' }])); + assert.equal(conflicts.caseConflict, true); assert.deepEqual(conflicts.cases.map(row => row.id), [91, 92]); + await assert.rejects(ensureChatPlan(() => assert.fail('no ambiguous write'), conflicts, () => assert.fail('no ambiguous marker'))); + await assert.rejects(inspectChatMetadata(inspectApi([{ ...caseRow, projectId: 2 }])), /identity_invalid/); + await assert.rejects(inspectChatMetadata(async () => ({ content: [], totalPages: 11, totalElements: 1100, number: 0 })), /catalog_incomplete/); +}); + +test('existing exact plan reuses case and plan without a write', async () => { + const result = await ensureChatPlan(async route => { assert.equal(route, '/api/testplan/8'); return plan; }, + await inspectChatMetadata(inspectApi()), () => assert.fail('must not mark or write')); + assert.equal(result.case.id, 91); + assert.equal(result.plan.id, 8); + assert.equal(result.actualResult, null); +}); + +test('bootstrap creates only the missing exact plan after exclusive marker and reads back scope', async () => { + const calls = []; + const result = await ensureChatPlan(async (route, method, body) => { + calls.push({ route, method, body }); + if (method === 'POST') { assert.equal(calls[0].route, 'MARK'); assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [91] } }); } + return plan; + }, { projectId: 1, cases: [caseRow], plans: [] }, async () => calls.push({ route: 'MARK' })); + assert.equal(calls.filter(row => row.method === 'POST').length, 1); + assert.equal(result.plan.count, 1); +}); + +test('empty project bootstrap links the genuinely created case into only the new plan', async () => { + const calls = []; + const createdCase = { ...caseRow, id: 92 }; + const createdPlan = { ...plan, treeSelection: { inverted: false, leafsInclude: [92] } }; + const result = await ensureChatPlan(async (route, method, body) => { + calls.push({ route, method, body }); + assert.equal(calls[0].route, 'MARK'); + if (route === '/api/testcase') { + assert.equal(body.projectId, 1); assert.equal(body.fullName, fullName); assert.equal(body.automated, true); + return createdCase; + } + if (route === '/api/testplan') { + assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [92] } }); + return createdPlan; + } + assert.equal(route, '/api/testplan/8'); assert.equal(method, undefined); return createdPlan; + }, { projectId: 1, cases: [], plans: [] }, async () => calls.push({ route: 'MARK' })); + assert.equal(result.case.id, 92); assert.equal(result.plan.id, 8); + assert.deepEqual(calls.map(call => call.route), ['MARK', '/api/testcase', '/api/testplan', '/api/testplan/8']); +}); + +test('known conflicting plan stops before creating an otherwise missing case or marker', async () => { + await assert.rejects(ensureChatPlan(() => assert.fail('no API write or read needed'), + { projectId: 1, cases: [], plans: [plan] }, () => assert.fail('must not mark')), /plan_without_matching_case/); +}); + +test('new case is created only after alias reconciliation; an uncertain mutation is never resent', async () => { + let writes = 0, markers = 0; + await assert.rejects(ensureChatPlan(async (_route, method) => { + assert.equal(method, 'POST'); writes++; throw new Error('uncertain transport'); + }, { projectId: 1, cases: [], plans: [] }, async () => { markers++; }), /uncertain/); + assert.equal(writes, 1); assert.equal(markers, 1); +}); + +test('existing broad or conflicting plan never expands or changes its scope', async () => { + for (const actual of [{ ...plan, testCasesCount: 2 }, { ...plan, treeSelection: { leafsInclude: [92] } }, + { ...plan, treeSelection: { leafsInclude: [91], inverted: true } }, { ...plan, projectId: 2 }]) { + await assert.rejects(ensureChatPlan(async (_route, method) => { assert.equal(method, undefined); return actual; }, + { projectId: 1, cases: [caseRow], plans: [plan] }, () => assert.fail('no writes')), /scope_mismatch/); + } +}); + +test('API credential stays in memory, redirects fail closed and response text is not an error', async () => { + const calls = []; + const api = await testopsClient('PRIVATE_API_TOKEN', async (url, options) => { + calls.push({ url, options }); + return calls.length === 1 ? { ok: true, json: async () => ({ access_token: 'PRIVATE_BEARER' }) } + : { ok: false, status: 403, text: () => assert.fail('must not inspect arbitrary error body') }; + }); + await assert.rejects(api('/api/testplan?projectId=1'), error => { assert.deepEqual(failureProjection(error, 'metadata_inspection'), { stage: 'metadata_inspection', code: 'testops_api_http', httpStatus: 403 }); return true; }); + assert.equal(calls[0].url, `${endpoint}/api/uaa/oauth/token`); + assert.equal(calls[1].options.redirect, 'error'); + await assert.rejects(api('https://foreign.invalid'), /route_invalid/); + assert.equal(calls.length, 2); +}); + +test('unknown exception text is never copied into closed diagnostics', () => { + const value = failureProjection(new Error('PRIVATE_RESPONSE_TOKEN_BODY'), 'untrusted raw stage'); + assert.deepEqual(value, { stage: 'unknown', code: 'unknown_failure', httpStatus: null }); + assert.ok(!JSON.stringify(value).includes('PRIVATE')); +}); + + +import { validateChatSelection } from './chat-feature-testops-api.mjs'; +const publicEnv = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', + GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master', + ALLURE_JOB_RUN_ID: '9', CHAT_TESTOPS_CASE_ID: '91', CHAT_TESTOPS_PLAN_ID: '8', CHAT_TESTOPS_JOB_ID: '7' }; +const selectedApi = replacements => async route => { + const values = { '/api/testcase/91': caseRow, '/api/testplan/8': plan, + '/api/testplan/8/job': [{ job: { id: 7 }, testCasesCount: 1 }], + '/api/job/7': { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml' }, + '/api/jobrun/9': { id: 9, job: { id: 7, type: 'github' }, stage: 'scheduled', launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }, + '/api/launch/10': { id: 10, projectId: 1, closed: false }, + '/api/jobrun/9/plan?expected=true': [{ id: 91, selector: fullName }] }; + assert.ok(Object.hasOwn(values, route)); + return replacements?.[route] ?? values[route]; +}; + +test('real API selection is exact singleton/project/job/launch and current source bound before model access', async () => { + const selected = await validateChatSelection(selectedApi(), publicEnv); + assert.equal(selected.runId, '123'); assert.equal(selected.workflowSha, 'a'.repeat(40)); + assert.equal(selected.executionId, 'chat-features.runner-opencode-deepseek-1017.local.create-backlog'); + assert.equal(selected.canonicalScenario, 'tasks.create-backlog'); assert.equal(selected.allureJobRunId, '9'); + assert.equal(selected.launchId, 10); +}); + +test('selection refuses missing correlation, a different case, broad plan, foreign job or closed launch', async () => { + await assert.rejects(validateChatSelection(selectedApi(), { ...publicEnv, ALLURE_JOB_RUN_ID: '' }), /identity_missing/); + for (const [route, value] of [ + ['/api/testcase/91', { ...caseRow, projectId: 2 }], + ['/api/testplan/8', { ...plan, testCasesCount: 2 }], + ['/api/job/7', { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip-cloud/actions/workflows/qa-staging-smoke.yml' }], + ['/api/testplan/8/job', [{ job: { id: 70 }, testCasesCount: 1 }]], + ['/api/jobrun/9', { id: 9, job: { id: 70, type: 'github' }, stage: 'scheduled', launchId: 10 }], + ['/api/jobrun/9', { id: 9, job: { id: 7, type: 'github' }, stage: 'scheduled', launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/456' }], + ['/api/launch/10', { id: 10, projectId: 1, closed: true }], + ['/api/jobrun/9/plan?expected=true', [{ id: 92, selector: fullName }]], + ]) await assert.rejects(validateChatSelection(selectedApi({ [route]: value }), publicEnv), /mismatch/); +}); + +test('selection rejects every non-master, foreign and second-attempt execution before API access', async () => { + for (const change of [{ GITHUB_REF: 'refs/heads/feature' }, { GITHUB_REPOSITORY: 'foreign' }, + { GITHUB_EVENT_NAME: 'pull_request' }, { GITHUB_RUN_ATTEMPT: '2' }, { GITHUB_WORKFLOW_REF: 'other' }]) { + await assert.rejects(validateChatSelection(() => assert.fail('must stop before API'), { ...publicEnv, ...change }), /untrusted/); + } +}); + +import { launchChatPlan, bootstrapJobMetadata, admitLaunchPlan } from './chat-feature-testops-api.mjs'; +const launchEnv = { ...publicEnv, ALLURE_JOB_RUN_ID: '', CHAT_TESTOPS_OPERATION_PARAMETER_ID: '17', CHAT_TESTOPS_INTEGRATION_ID: '2', CHAT_TESTOPS_LAUNCH_NONCE: '1'.repeat(32), CHAT_TESTOPS_LAUNCH_RUN_NUMBER: '2', GITHUB_RUN_NUMBER: '2', + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: '99', GITHUB_ACTOR_ID: '34892728', TESTOPS_TRIGGERING_ACTOR_ID: '34892728', TESTOPS_ALLOWED_ACTOR_IDS: '[34892728]', GH_TOKEN: 'SYNTHETIC_GH_TOKEN' }; +const ownRun = { id: 123, workflow_id: 99, run_attempt: 1, run_number: 2, head_sha: 'a'.repeat(40), head_branch: 'master', event: 'workflow_dispatch', + repository: { full_name: 'paperclipai/paperclip', private: false }, actor: { id: 34892728 }, triggering_actor: { id: 34892728 } }; +async function fence(env = launchEnv, changes = {}, workflowChanges = {}, repoChanges = {}) { + return admitLaunchPlan(env, async url => ({ ok: true, text: async () => JSON.stringify(url.endsWith('/repos/paperclipai/paperclip') ? { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master', ...repoChanges } + : url.includes('/actions/runs/') ? { ...ownRun, ...changes } + : { id: 99, path: '.github/workflows/chat-feature-testops.yml', state: 'active', ...workflowChanges }) })); +} +function launchServer({ wrongParameter = false, wrongAssignment = false, uncertainRun = false, integrationChange = {} } = {}) { + const calls = [], parameters = [{ id: 17, value: 'bootstrap' }]; let assigned = false, launched = false, requests = 0; + const api = async (route, method, body) => { + calls.push({ route, method, body }); + if (route.startsWith('/api/launch?')) return { totalPages: 1, content: launched ? [{ id: 10, name: `Chat feature evals · ${'1'.repeat(32)}` }] : [] }; + if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false, ...integrationChange }; + if (route === '/api/environment/17') return { id: 17, name: wrongParameter ? 'OTHER_PRIVATE_PARAMETER' : 'operation' }; + if (route === '/api/job/7') { + if (method === 'PATCH') { assert.deepEqual(body, { integrationId: 2, canRun: true, parameters: [{ id: 17, value: 'evaluate' }] }); parameters[0].value = 'evaluate'; } + return { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters }; + } + if (route === '/api/testplan/8/job') return wrongAssignment ? [{ job: { id: 70 }, testCasesCount: 1 }] + : assigned ? [{ job: { id: 7, parameters }, testCasesCount: 1 }] : []; + if (route === '/api/testplan/8/assign') { assert.equal(method, 'POST'); assert.deepEqual(body, { jobId: 7, selection: { inverted: false, manual: false, leafsInclude: [91] } }); assigned = true; return null; } + if (route === '/api/testplan/8/jobparameter') { assert.deepEqual(body, { jobId: 7, parameters: [{ id: 17, value: 'evaluate' }] }); return null; } + if (route === '/api/testplan/8/run') { requests++; assert.equal(method, 'POST'); if (uncertainRun) throw new Error('synthetic uncertain transport'); launched = true; return { id: 10, projectId: 1, name: body.launchName }; } + return selectedApi()(route); + }; + return { api, calls, requests: () => requests }; +} + +test('launch configures only the exact registered public job/plan then issues one run request and exits', async () => { + const server = launchServer(); let markers = 0; + const result = await launchChatPlan(server.api, launchEnv, async () => { markers++; assert.equal(server.calls.some(row => row.method), false); }, await fence()); + assert.equal(markers, 1); assert.equal(server.requests(), 1); assert.equal(result.status, 'REQUESTED'); + assert.equal(result.launchId, 10); assert.equal(result.actualRunId, null); assert.equal(result.actualResult, null); + assert.ok(!server.calls.some(row => /jobrun.*rerun|dispatches/.test(row.route))); +}); + +test('observed launch nonce prevents a second run request across separate local entry attempts', async () => { + const server = launchServer(); await launchChatPlan(server.api, launchEnv, async () => {}, await fence()); + const result = await launchChatPlan(server.api, launchEnv, () => assert.fail('existing launch must not mark or write'), await fence()); + assert.equal(result.status, 'EXISTING_NO_RESEND'); assert.equal(server.requests(), 1); +}); + +test('foreign assignment and unobserved operation parameter stop before any write', async () => { + for (const options of [{ wrongAssignment: true }, { wrongParameter: true }]) { + const server = launchServer(options); + await assert.rejects(launchChatPlan(server.api, launchEnv, () => assert.fail('no marker'), await fence())); + assert.equal(server.calls.some(row => row.method), false); assert.equal(server.requests(), 0); + } +}); + +test('uncertain plan-run transport is sent once and never followed by another mutation', async () => { + const server = launchServer({ uncertainRun: true }); + await assert.rejects(launchChatPlan(server.api, launchEnv, async () => {}, await fence()), /uncertain/); + assert.equal(server.requests(), 1); + assert.equal(server.calls.at(-1).route, '/api/testplan/8/run'); +}); + +test('official bootstrap job metadata projects only fixed parameter names and operation enums', async () => { + const job = await bootstrapJobMetadata(async route => { + if (route === '/api/jobrun/9') return { id: 9, job: { id: 7 }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }; + if (route === '/api/job/7') return { id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters: [{ id: 17, value: 'bootstrap' }, { id: 18, value: 'SYNTHETIC_PRIVATE_VALUE' }] }; + if (route === '/api/environment/17') return { id: 17, name: 'operation' }; + return { id: 18, name: 'UNRELATED_NAME' }; + }, publicEnv); + assert.equal(job.publicWorkflowMatches, true); assert.equal(job.parameters[0].value, 'bootstrap'); + assert.deepEqual(job.parameters[1], { id: 18, name: 'UNKNOWN', value: 'NOT_PROJECTED' }); + assert.ok(!JSON.stringify(job).includes('SYNTHETIC_PRIVATE_VALUE')); +}); + +test('documented stable-ID list pagination is complete before locally matching exact alias detail metadata', async () => { + const calls = []; + const api = async route => { + calls.push(route); + if (route.startsWith('/api/testcase?')) { + const query = new URLSearchParams(route.split('?')[1]); + assert.equal(query.get('projectId'), '1'); assert.equal(query.get('sort'), 'id,asc'); + const number = Number(query.get('page')); + return { number, totalPages: 2, totalElements: 2, content: [{ id: number ? 92 : 91 }] }; + } + if (route === '/api/testcase/91') return { ...caseRow, description: 'SYNTHETIC_PRIVATE_DESCRIPTION' }; + if (route === '/api/testcase/92') return { id: 92, projectId: 1, fullName: 'paperclip.Product E2E.unrelated.case', automated: true, expectedResult: 'DO_NOT_EXPORT' }; + return inspectApi([], [])(route); + }; + const result = await inspectChatMetadata(api); + assert.deepEqual(result.cases, [{ id: 91, fullName, automated: true }]); + assert.ok(calls.some(route => route.includes('page=1'))); + assert.ok(!calls.some(route => route.includes('__search'))); + assert.ok(!JSON.stringify(result).includes('SYNTHETIC_PRIVATE_DESCRIPTION')); + assert.ok(!JSON.stringify(result).includes('DO_NOT_EXPORT')); +}); + +test('changing catalog totals or repeated page IDs cannot become a not-found result', async () => { + for (const duplicate of [false, true]) { + await assert.rejects(inspectChatMetadata(async route => { + if (route.startsWith('/api/testcase?')) { + const number = Number(new URLSearchParams(route.split('?')[1]).get('page')); + return { number, totalPages: 2, totalElements: number && !duplicate ? 3 : 2, content: [{ id: number && !duplicate ? 92 : 91 }] }; + } + return caseRow; + }), /catalog_incomplete/); + } +}); + +test('catalog detail reads have a finite four-request concurrency bound', async () => { + let active = 0, maximum = 0; + const api = async route => { + if (route.startsWith('/api/testcase?')) return { number: 0, totalPages: 1, totalElements: 7, content: Array.from({ length: 7 }, (_, index) => ({ id: index + 1 })) }; + if (/^\/api\/testcase\/[0-9]+$/.test(route)) { + active++; maximum = Math.max(maximum, active); await new Promise(resolve => queueMicrotask(resolve)); active--; + return { id: Number(route.split('/').at(-1)), projectId: 1, fullName: null, automated: false }; + } + return inspectApi([], [])(route); + }; + assert.deepEqual((await inspectChatMetadata(api)).cases, []); + assert.equal(maximum, 4); +}); + +test('fresh dispatch or replay cannot acquire the fixed initial launch counter even with stale TestOps search', async () => { + for (const env of [{ ...launchEnv, GITHUB_RUN_NUMBER: '3' }, { ...launchEnv, GITHUB_RUN_ATTEMPT: '2' }, + { ...launchEnv, CHAT_TESTOPS_LAUNCH_RUN_NUMBER: '' }]) { + await assert.rejects(admitLaunchPlan(env, () => assert.fail('counter/refusal precedes all remote reads'))); + } +}); + +test('real own-run metadata rejects counter, source, workflow, actor and default-ref contradictions', async () => { + for (const changes of [{ run_number: 3 }, { head_sha: 'b'.repeat(40) }, { workflow_id: 100 }, + { actor: { id: 1 } }, { triggering_actor: { id: 1 } }, { run_attempt: 2 }, { head_branch: 'feature' }]) { + await assert.rejects(fence(launchEnv, changes), /launch_admission/); + } + await assert.rejects(fence(launchEnv, {}, {}, { default_branch: 'other' }), /launch_admission/); + await assert.rejects(fence(launchEnv, {}, { state: 'disabled_manually' }), /launch_admission/); +}); + +test('no caller-supplied JSON can replace the source-owned GitHub launch admission receipt', async () => { + const actual = await fence(); + await assert.rejects(launchChatPlan(() => assert.fail('no API mutation'), launchEnv, () => assert.fail('no marker'), JSON.parse(JSON.stringify(actual))), /launch_admission/); +}); + +test('GitHub metadata denial retains only its fixed code and HTTP status', async () => { + await assert.rejects(admitLaunchPlan(launchEnv, async () => ({ ok: false, status: 403, text: () => assert.fail('must not read denial body') })), error => { + assert.deepEqual(failureProjection(error, 'launch_admission'), { stage: 'launch_admission', code: 'chat_testops_github_http', httpStatus: 403 }); return true; + }); +}); + +test('new job receives only the reviewed enabled GitHub integration; disabled/foreign identities stop before writes', async () => { + for (const integrationChange of [{ id: 3 }, { projectId: 2 }, { disabled: true }, { disabled: null }, { info: { type: 'gitlab' } }]) { + const server = launchServer({ integrationChange }); + await assert.rejects(launchChatPlan(server.api, launchEnv, () => assert.fail('no marker'), await fence()), /integration_mismatch/); + assert.equal(server.calls.some(row => row.method), false); + } +}); + +const uploadContext = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', ALLURE_JOB_RUN_ID: '9', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; +const uploadSelection = { runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: 'a'.repeat(40), allureJobRunId: '9', + executionId: 'chat-features.runner-opencode-deepseek-1017.local.create-backlog', canonicalScenario: 'tasks.create-backlog', projectId: 1, + planId: 8, jobId: 7, caseId: 91, fullName, launchId: 10 }; +const uploadReceipt = { status: 'passed', cleanup: 'passed', caseId: 91, campaignId: 'gha-123-1', count: 1, + attachmentMetadata: [{ name: 'Synthetic task and source conversation', contentType: 'image/png', contentLength: 400 }, + { name: 'Execution provenance and checks', contentType: 'application/json', contentLength: 300 }, + { name: 'Sanitized Product E2E record and accounting', contentType: 'application/json', contentLength: 800 }] }; +const uploadRun = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123', stage: 'finished' }; +function uploadApi(change = {}, absent = 0) { + let polls = 0; + return async (route, method) => { + assert.equal(method, 'GET'); + if (route === '/api/jobrun/9') return uploadRun; + if (route === '/api/launch/10') return { id: 10, projectId: 1 }; + if (route.startsWith('/api/testresult?')) { polls++; return { content: polls <= absent ? [] : [{ id: 45, testCaseId: 91 }], totalElements: polls <= absent ? 0 : 1, totalPages: polls <= absent ? 0 : 1 }; } + if (route === '/api/testresult/45') return { id: 45, projectId: 1, launchId: 10, testCaseId: 91, fullName, jobRun: uploadRun, status: 'passed', + links: [{ url: uploadRun.url }, { url: `https://github.com/paperclipai/paperclip/commit/${'a'.repeat(40)}` }], + get trace() { assert.fail('must not inspect trace'); }, ...change }; + if (route.startsWith('/api/testresult/attachment?')) return { content: uploadReceipt.attachmentMetadata.map((row, i) => ({ ...row, id: i + 1, missed: false, fromTestCase: false })), totalElements: 3, totalPages: 1 }; + assert.fail(route); + }; +} + +test('readback verifies exact current launch/case/source/status and safe attachment metadata after processing lag', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + let waits = 0; + const result = await verifyChatUpload(uploadApi({}, 1), uploadContext, uploadSelection, uploadReceipt, { wait: async () => { waits++; }, now: () => 0 }); + assert.equal(waits, 1); assert.equal(result.verification, 'VERIFIED'); assert.equal(result.resultId, 45); + assert.equal(result.status, 'passed'); assert.deepEqual(result.attachments, uploadReceipt.attachmentMetadata); + assert.ok(!JSON.stringify(result).includes('trace')); +}); + +test('readback rejects wrong launch, case, current jobrun, source and contradictory result status', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + for (const change of [{ launchId: 11 }, { testCaseId: 92 }, { jobRun: { ...uploadRun, id: 11 } }, { links: [] }, { status: 'failed' }]) { + await assert.rejects(verifyChatUpload(uploadApi(change), uploadContext, uploadSelection, uploadReceipt), /upload_mismatch/); + } +}); + +test('readback missing result or screenshot metadata is bounded unverified, never a fabricated pass', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + let waits = 0; + await assert.rejects(verifyChatUpload(uploadApi({}, 100), uploadContext, uploadSelection, uploadReceipt, + { wait: async () => { waits++; }, now: () => 0 }), /upload_unverified/); + assert.equal(waits, 11); + const base = uploadApi(); + await assert.rejects(verifyChatUpload(async (route, ...args) => route.startsWith('/api/testresult/attachment?') + ? { content: [], totalElements: 0, totalPages: 0 } : base(route, ...args), uploadContext, uploadSelection, uploadReceipt, + { wait: async () => {}, now: () => 0 }), /upload_unverified/); +}); + +test('verified failed Product E2E remains failed and cleanup cannot be promoted', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + const result = await verifyChatUpload(uploadApi({ status: 'failed' }), uploadContext, uploadSelection, { ...uploadReceipt, status: 'failed', cleanup: 'failed' }); + assert.equal(result.status, 'failed'); assert.equal(result.cleanup, 'failed'); + await assert.rejects(verifyChatUpload(uploadApi(), uploadContext, uploadSelection, { ...uploadReceipt, cleanup: 'failed' }), /upload_mismatch/); +}); + + +test('readback stops immediately after a late GET and gives each client only the remaining deadline', async () => { + const { verifyChatUpload } = await import('./chat-feature-testops-api.mjs'); + let time = 0, calls = 0; + await assert.rejects(verifyChatUpload(async (route, method, body, timeout) => { + calls++; assert.equal(route, '/api/jobrun/9'); assert.equal(timeout, 30_000); time = 120_001; return uploadRun; + }, uploadContext, uploadSelection, uploadReceipt, { now: () => time }), /upload_unverified/); + assert.equal(calls, 1); + time = 0; calls = 0; + await assert.rejects(verifyChatUpload(async (route, method, body, timeout) => { + calls++; + if (calls === 1) { assert.equal(timeout, 30_000); time = 119_990; return uploadRun; } + assert.equal(route, '/api/launch/10'); assert.equal(timeout, 10); time = 120_001; return { id: 10, projectId: 1 }; + }, uploadContext, uploadSelection, uploadReceipt, { now: () => time }), /upload_unverified/); + assert.equal(calls, 2); +}); diff --git a/tests/runner-e2e/chat-feature-testops-cli.mjs b/tests/runner-e2e/chat-feature-testops-cli.mjs new file mode 100644 index 00000000000..60768ac420f --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-cli.mjs @@ -0,0 +1,99 @@ +import { lstat, mkdir, open, readFile, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { publicContext, testopsClient, inspectChatMetadata, ensureChatPlan, validateChatSelection, bootstrapJobMetadata, launchChatPlan, admitLaunchPlan, verifyChatUpload, failureProjection } from './chat-feature-testops-api.mjs'; +import { exportChatAllure, readBoundChatExport, ExportFailure } from './chat-feature-testops-results.mjs'; + +export async function run(env = process.env, fetchImpl = fetch) { + let stage = 'context', stateRoot; + try { + const expected = publicContext(env), operation = env.CHAT_TESTOPS_OPERATION; + if (!['bootstrap', 'launch-plan', 'validate-selection', 'export', 'verify-upload'].includes(operation)) throw new Error('unknown_operation'); + if (operation === 'bootstrap' || operation === 'launch-plan') { + const eventPath = env.GITHUB_EVENT_PATH; + if (!eventPath) throw new Error('bootstrap_original_event_missing'); + const stat = await lstat(eventPath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 65_536) throw new Error('bootstrap_original_event_invalid'); + const event = JSON.parse(await readFile(eventPath, 'utf8')); + if (event.inputs?.operation !== operation || (event.inputs?.ALLURE_JOB_RUN_ID ?? '') !== '') throw new Error('bootstrap_cannot_run_from_testops'); + } + stage = 'output_admission'; + const requested = env.CHAT_TESTOPS_STATE_ROOT; + if (!env.RUNNER_TEMP || !requested || !path.isAbsolute(requested) + || path.dirname(path.resolve(requested)) !== path.resolve(env.RUNNER_TEMP)) throw new Error('invalid_output'); + await mkdir(requested, { mode: 0o700 }); stateRoot = requested; + let result; + if (operation === 'export' || operation === 'verify-upload') { + stage = 'result_export'; + const selectionPath = env.CHAT_TESTOPS_SELECTION; + if (!selectionPath || !path.isAbsolute(selectionPath) || !selectionPath.startsWith(`${path.resolve(env.RUNNER_TEMP)}${path.sep}`)) throw new Error('invalid_selection_path'); + const stat = await lstat(selectionPath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 16_384) throw new Error('invalid_selection_file'); + const selected = JSON.parse(await readFile(selectionPath, 'utf8')); + if (operation === 'verify-upload') { + stage = 'upload_verification'; + const receiptRoot = path.join(env.RUNNER_TEMP, 'chat-testops-upload-state'); + const rootStat = await lstat(receiptRoot), file = path.join(receiptRoot, 'export.json'); + const receiptStat = await lstat(file); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink() || !receiptStat.isFile() || receiptStat.isSymbolicLink() + || receiptStat.nlink !== 1 || receiptStat.size > 16_384) throw new Error('invalid_export_receipt'); + const exported = JSON.parse(await readFile(file, 'utf8')); + const bound = await readBoundChatExport(env.CHAT_TESTOPS_EXPORT_ROOT, expected); + if (exported.status !== (bound.result.status === 'passed' && bound.result.cleanup === 'passed' ? 'passed' : 'failed') + || exported.cleanup !== bound.result.cleanup) throw new Error('export_verdict_mismatch'); + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + result = await verifyChatUpload(api, env, selected, exported); + await writeFile(path.join(stateRoot, 'verification.json'), JSON.stringify(result, null, 2), { flag: 'wx', mode: 0o600 }); + } else { + result = await exportChatAllure({ root: env.CHAT_TESTOPS_EXPORT_ROOT, output: env.ALLURE_RESULTS, expected, selected }); + await writeFile(path.join(stateRoot, 'export.json'), JSON.stringify(result, null, 2), { flag: 'wx', mode: 0o600 }); + } + } else { + let launchAdmission; + if (operation === 'launch-plan') { stage = 'launch_admission'; launchAdmission = await admitLaunchPlan(env, fetchImpl); } + stage = 'authentication'; + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + if (operation === 'launch-plan') { + stage = 'launch_preflight'; + result = await launchChatPlan(api, env, async identity => { + const handle = await open(path.join(stateRoot, 'once.json'), 'wx', 0o600); + await handle.writeFile(JSON.stringify({ ...identity, status: 'STARTED_NO_AUTOMATIC_RESEND' })); + await handle.close(); stage = 'plan_launch'; + }, launchAdmission); + await writeFile(path.join(stateRoot, 'launch.json'), JSON.stringify(result, null, 2), { flag: 'wx', mode: 0o600 }); + } else if (operation === 'validate-selection') { + stage = 'selection_validation'; + result = await validateChatSelection(api, env); + await writeFile(path.join(stateRoot, 'selection.json'), JSON.stringify(result, null, 2), { flag: 'wx', mode: 0o600 }); + } else { + stage = 'metadata_inspection'; + const inspected = await inspectChatMetadata(api); + await writeFile(path.join(stateRoot, 'metadata.json'), JSON.stringify(inspected, null, 2), { flag: 'wx', mode: 0o600 }); + let marked = false; + const once = async () => { + if (marked) return; + const handle = await open(path.join(stateRoot, 'once.json'), 'wx', 0o600); + await handle.writeFile(JSON.stringify({ ...expected, operation: 'plan_bootstrap', status: 'STARTED_NO_AUTOMATIC_RESEND' })); + await handle.close(); marked = true; + }; + stage = 'plan_bootstrap'; + result = await ensureChatPlan(api, inspected, once); + await writeFile(path.join(stateRoot, 'plan.json'), JSON.stringify(result, null, 2), { flag: 'wx', mode: 0o600 }); + stage = 'bootstrap_job_metadata'; + const job = await bootstrapJobMetadata(api, env); + await writeFile(path.join(stateRoot, 'job.json'), JSON.stringify(job, null, 2), { flag: 'wx', mode: 0o600 }); + } + } + return operation === 'export' && result.status !== 'passed' || operation === 'launch-plan' && result.status !== 'REQUESTED' ? 1 : 0; + } catch (error) { + const safe = failureProjection(error, stage); + if (error instanceof ExportFailure) safe.code = 'chat_testops_export_rejected'; + if (stateRoot) await writeFile(path.join(stateRoot, 'failure.json'), JSON.stringify(safe, null, 2), { flag: 'wx', mode: 0o600 }).catch(() => {}); + process.stderr.write(`Chat TestOps stopped: ${JSON.stringify(safe)}\n`); + return 1; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + run().then(code => { process.exitCode = code; }).catch(() => { process.stderr.write('Chat TestOps stopped: unknown failure\n'); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/chat-feature-testops-cli.test.mjs b/tests/runner-e2e/chat-feature-testops-cli.test.mjs new file mode 100644 index 00000000000..107f046b109 --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-cli.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import { run } from './chat-feature-testops-cli.mjs'; +import { fullName, planName } from './chat-feature-testops-api.mjs'; +const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; + +test('actual metadata bootstrap caller creates no result or agent and reuses exact case/plan', async t => { + const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-bootstrap-'))); + t.after(() => rm(temporary, { recursive: true })); + const state = path.join(temporary, 'state'), requests = []; + const eventPath = path.join(temporary, 'event.json'); await writeFile(eventPath, JSON.stringify({ inputs: { operation: 'bootstrap', ALLURE_JOB_RUN_ID: '' } })); + const response = value => ({ ok: true, status: 200, json: async () => value, text: async () => JSON.stringify(value) }); + const fetchImpl = async (url, options) => { + requests.push({ url, method: options.method }); + if (url.endsWith('/api/uaa/oauth/token')) return response({ access_token: 'SYNTHETIC_BEARER' }); + const route = new URL(url).pathname; + if (route === '/api/testcase') return response({ number: 0, totalPages: 1, totalElements: 1, content: [{ id: 91, name: 'Not projected', automated: true }] }); + if (route === '/api/testcase/91') return response({ id: 91, projectId: 1, fullName, automated: true }); + if (route === '/api/testplan') return response({ totalPages: 1, content: [{ id: 8, projectId: 1, name: planName, testCasesCount: 1 }] }); + if (route === '/api/integration/project/1') return response({ totalPages: 1, content: [] }); + if (route === '/api/jobrun/9') return response({ id: 9, job: { id: 7 }, launchId: 10, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }); + if (route === '/api/job/7') return response({ id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters: [{ id: 17, value: 'bootstrap' }] }); + if (route === '/api/environment/17') return response({ id: 17, name: 'operation' }); + assert.equal(route, '/api/testplan/8'); + return response({ id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { leafsInclude: [91], inverted: false } }); + }; + assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 0); + const plan = JSON.parse(await readFile(path.join(state, 'plan.json'))); + assert.equal(plan.actualResult, null); + assert.equal(requests.filter(value => value.method === 'POST').length, 1, 'only auth token exchange; no case/plan write'); + await assert.rejects(readFile(path.join(state, 'allure-results.json')), { code: 'ENOENT' }); +}); + +test('actual public caller retains only safe auth status on failure before discovery', async t => { + const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-failure-'))); + t.after(() => rm(temporary, { recursive: true })); + const messages = [], original = process.stderr.write; + process.stderr.write = value => { messages.push(String(value)); return true; }; + try { + const state = path.join(temporary, 'state'); + const eventPath = path.join(temporary, 'event.json'); await writeFile(eventPath, JSON.stringify({ inputs: { operation: 'bootstrap' } })); + assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_TOKEN: 'SYNTHETIC_PRIVATE_TOKEN' }, + async () => ({ ok: false, status: 403, text: () => assert.fail('must not read denial body') })), 1); + assert.deepEqual(JSON.parse(await readFile(path.join(state, 'failure.json'))), { stage: 'authentication', code: 'testops_auth_http', httpStatus: 403 }); + assert.ok(!messages.join('').includes('SYNTHETIC_PRIVATE_TOKEN')); + } finally { process.stderr.write = original; } +}); + + +test('original TestOps-triggered bootstrap is rejected even if its official environment is populated', async t => { + const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-wrong-bootstrap-'))); + t.after(() => rm(temporary, { recursive: true })); + const event = path.join(temporary, 'event.json'); await writeFile(event, JSON.stringify({ inputs: { operation: 'bootstrap', ALLURE_JOB_RUN_ID: '9' } })); + const original = process.stderr.write; + process.stderr.write = () => true; + try { + assert.equal(await run({ ...context, GITHUB_EVENT_PATH: event, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: path.join(temporary, 'state'), + CHAT_TESTOPS_OPERATION: 'bootstrap', ALLURE_JOB_RUN_ID: '9' }, () => assert.fail('no API access')), 1); + } finally { process.stderr.write = original; } +}); diff --git a/tests/runner-e2e/chat-feature-testops-results.mjs b/tests/runner-e2e/chat-feature-testops-results.mjs new file mode 100644 index 00000000000..b20f8fc6337 --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-results.mjs @@ -0,0 +1,134 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { lstat, mkdir, readFile, readdir, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { aliases, executionId } from './chat-feature-testops-api.mjs'; + +export class ExportFailure extends Error {} +const fail = () => { throw new ExportFailure('chat_testops_export_rejected'); }; +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const pngMagic = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +const model = 'openrouter/deepseek/deepseek-v4.1-flash-1017'; +const pngPath = new RegExp(`^evidence/${executionId.replaceAll('.', '\\.')}/attempt-1/[A-Za-z0-9][A-Za-z0-9._-]{0,199}\\.png$`); + +async function inventory(root, relative = '') { + const output = []; + const entries = await readdir(path.join(root, relative)); + if (entries.length > 12) fail(); + for (const name of entries) { + if (!/^[A-Za-z0-9._-]+$/.test(name) || name === '.' || name === '..') fail(); + const file = relative ? `${relative}/${name}` : name; + const stat = await lstat(path.join(root, file)); + if (stat.isSymbolicLink()) fail(); + if (stat.isDirectory()) { + if (!['evidence', `evidence/${executionId}`, `evidence/${executionId}/attempt-1`].includes(file)) fail(); + output.push(...await inventory(root, file)); + } + else if (stat.isFile() && stat.nlink === 1 && stat.size <= 12 * 1024 * 1024) output.push(file); + else fail(); + if (output.length > 12) fail(); + } + return output; +} + +export async function readBoundChatExport(root, expected) { + if (!/^[1-9][0-9]{0,19}$/.test(expected.runId ?? '') || expected.attempt !== '1' + || !/^[a-f0-9]{40}$/.test(expected.workflowSha ?? '') || expected.targetSha !== expected.workflowSha) fail(); + const stat = await lstat(root); + if (!stat.isDirectory() || stat.isSymbolicLink() || await realpath(root) !== path.resolve(root)) fail(); + const files = await inventory(root); + if (!files.includes('binding.json')) fail(); + const bindingBytes = await readFile(path.join(root, 'binding.json')); + if (bindingBytes.length > 128_000) fail(); + const binding = JSON.parse(bindingBytes); + if (binding.schema !== 'paperclip.chat-feature-testops-export/v1' || binding.attempt !== '1' + || binding.runId !== expected.runId || binding.workflowSha !== expected.workflowSha || binding.targetSha !== expected.targetSha + || binding.executionId !== executionId || binding.campaignId !== `gha-${expected.runId}-1` + || !Array.isArray(binding.files) || binding.files.length < 2 || binding.files.length > 10) fail(); + const names = binding.files.map(row => row.path); + if (new Set(names).size !== names.length || !names.includes('normalized-results.json') || !names.includes('junit.xml') + || names.some(name => !['normalized-results.json', 'junit.xml'].includes(name) && !pngPath.test(name)) + || JSON.stringify([...names, 'binding.json'].sort()) !== JSON.stringify(files.sort())) fail(); + const retained = new Map(); + let total = 0; + for (const row of binding.files) { + if (!/^[a-f0-9]{64}$/.test(row.sha256 ?? '') || !Number.isSafeInteger(row.bytes) || row.bytes < 0) fail(); + const bytes = await readFile(path.join(root, row.path)); + total += bytes.length; + if (total > 32 * 1024 * 1024 || bytes.length !== row.bytes || hash(bytes) !== row.sha256) fail(); + retained.set(row.path, bytes); + } + const bytes = retained.get('normalized-results.json'); + if (bytes.length > 2_000_000) fail(); + const campaign = JSON.parse(bytes); + if (campaign.schema !== 'paperclip.runner-e2e.campaign/v2' || campaign.campaignId !== binding.campaignId + || !Array.isArray(campaign.results) || campaign.results.length !== 1) fail(); + const result = campaign.results[0]; + if (result.executionId !== executionId || result.suiteId !== 'chat-features' || result.caseId !== 'create-backlog' + || result.profileId !== 'runner-opencode-deepseek-1017' || result.provider !== 'opencode' || result.environmentId !== 'local' + || result.model !== model || result.runtimeMode !== 'native' || result.attempt !== 1 || result.source?.sha !== expected.targetSha + || !['passed', 'failed'].includes(result.status) || result.status !== binding.resultStatus + || !['passed', 'failed', 'not_started'].includes(result.cleanup) || result.cleanup !== binding.cleanup + || !Array.isArray(result.matcherResults) || result.matcherResults.length > 200 || !Array.isArray(result.screenshots) || result.screenshots.length > 8) fail(); + for (const check of result.matcherResults) if (check.matcher?.kind !== 'json_path' || !/^[A-Za-z0-9._-]{1,150}$/.test(check.matcher.path) + || check.matcher.expected !== true || typeof check.passed !== 'boolean') fail(); + for (const shot of result.screenshots) { + const name = `evidence/${executionId}/attempt-1/${shot.file}`; + const png = retained.get(name); + if (!png || !png.subarray(0, 8).equals(pngMagic) || png.subarray(12, 16).toString('ascii') !== 'IHDR' || hash(png) !== shot.sha256) fail(); + } + if (names.filter(name => pngPath.test(name)).length !== result.screenshots.length) fail(); + if (result.status === 'passed' && (result.cleanup !== 'passed' || result.matcherResults.length === 0 + || result.matcherResults.some(check => !check.passed) || result.runIds?.length !== 1 + || !result.screenshots.some(shot => shot.id === 'final-state' && shot.file === 'final-state.png'))) fail(); + return { binding, result, retained }; +} + +export async function exportChatAllure({ root, output, expected, selected }) { + if (selected.projectId !== 1 || !Number.isSafeInteger(selected.caseId) || selected.caseId < 1 || !aliases.includes(selected.fullName) + || selected.runId !== expected.runId || selected.attempt !== '1' || selected.workflowSha !== expected.workflowSha || selected.targetSha !== expected.targetSha + || selected.executionId !== executionId) fail(); + const { binding, result, retained } = await readBoundChatExport(root, expected); + await mkdir(output, { mode: 0o700 }).catch(error => { if (error.code !== 'EEXIST') throw error; }); + const outputStat = await lstat(output); + if (!outputStat.isDirectory() || outputStat.isSymbolicLink() || (process.getuid && outputStat.uid !== process.getuid()) || (await readdir(output)).length !== 0) fail(); + const status = result.status === 'passed' && result.cleanup === 'passed' ? 'passed' : 'failed'; + const attachments = []; + const attachmentMetadata = []; + for (const shot of result.screenshots) { + const source = `${randomUUID()}-attachment.png`; + await writeFile(path.join(output, source), retained.get(`evidence/${executionId}/attempt-1/${shot.file}`), { flag: 'wx', mode: 0o600 }); + attachments.push({ name: 'Synthetic task and source conversation', source, type: 'image/png' }); + attachmentMetadata.push({ name: 'Synthetic task and source conversation', contentType: 'image/png', contentLength: retained.get(`evidence/${executionId}/attempt-1/${shot.file}`).length }); + } + const projection = { campaignId: binding.campaignId, executionId, runId: expected.runId, attempt: 1, + workflowSha: expected.workflowSha, sourceSha: expected.targetSha, status: result.status, cleanup: result.cleanup, + profile: result.profileId, model: result.model, environment: result.environmentId, runtime: result.runtimeMode, + durationMs: Number.isFinite(result.durationMs) ? result.durationMs : null, + failureClass: typeof result.failureClass === 'string' && /^[a-z_]{1,80}$/.test(result.failureClass) ? result.failureClass : null, + billingComplete: typeof result.billing?.complete === 'boolean' ? result.billing.complete : null, + checks: result.matcherResults.map(check => ({ path: check.matcher.path, passed: check.passed })), + note: 'Setup usage and AWS billing are not qualified by this result; missing accounting remains unknown.' }; + const source = `${randomUUID()}-attachment.json`; + const projectionBytes = Buffer.from(JSON.stringify(projection, null, 2)); + await writeFile(path.join(output, source), projectionBytes, { flag: 'wx', mode: 0o600 }); + attachmentMetadata.push({ name: 'Execution provenance and checks', contentType: 'application/json', contentLength: projectionBytes.length }); + attachments.push({ name: 'Execution provenance and checks', source, type: 'application/json' }); + const normalizedSource = `${randomUUID()}-attachment.json`; + await writeFile(path.join(output, normalizedSource), retained.get('normalized-results.json'), { flag: 'wx', mode: 0o600 }); + attachments.push({ name: 'Sanitized Product E2E record and accounting', source: normalizedSource, type: 'application/json' }); + attachmentMetadata.push({ name: 'Sanitized Product E2E record and accounting', contentType: 'application/json', contentLength: retained.get('normalized-results.json').length }); + const value = { uuid: randomUUID(), name: 'Create a task backlog through chat', fullName: selected.fullName, + testCaseId: hash(`${JSON.stringify(selected.fullName)}\n`), + historyId: hash(JSON.stringify([selected.fullName, result.profileId, result.model, result.environmentId])), status, stage: 'finished', + statusDetails: { message: status === 'passed' ? 'Real Product E2E assertions and cleanup passed' : 'Real Product E2E evaluation or cleanup failed; original verdict preserved' }, + labels: [{ name: 'AS_ID', value: String(selected.caseId) }, { name: 'parentSuite', value: 'Product E2E' }, + { name: 'suite', value: 'chat-features' }, { name: 'area', value: 'Chat and collaboration' }, + { name: 'framework', value: 'Paperclip Evals' }, { name: 'tag', value: 'chat-feature-testops' }], + parameters: [{ name: 'canonicalScenario', value: 'tasks.create-backlog' }, { name: 'model', value: model }, { name: 'environment', value: 'GitHub Actions / AWS' }], + links: [{ name: 'GitHub Actions run', type: 'custom', url: `https://github.com/paperclipai/paperclip/actions/runs/${expected.runId}` }, + { name: 'Evaluated source', type: 'custom', url: `https://github.com/paperclipai/paperclip/commit/${expected.targetSha}` }], + steps: result.matcherResults.map(check => ({ name: check.matcher.path, status: check.passed ? 'passed' : 'failed', stage: 'finished' })), attachments }; + for (const [key, time] of [['start', result.startedAt], ['stop', result.finishedAt]]) if (Number.isFinite(Date.parse(time))) value[key] = Date.parse(time); + await writeFile(path.join(output, `${value.uuid}-result.json`), JSON.stringify(value), { flag: 'wx', mode: 0o600 }); + return { status, cleanup: result.cleanup, caseId: selected.caseId, campaignId: binding.campaignId, count: 1, attachmentMetadata }; +} diff --git a/tests/runner-e2e/chat-feature-testops-results.test.mjs b/tests/runner-e2e/chat-feature-testops-results.test.mjs new file mode 100644 index 00000000000..84034e0b879 --- /dev/null +++ b/tests/runner-e2e/chat-feature-testops-results.test.mjs @@ -0,0 +1,73 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, mkdir, readFile, readdir, realpath, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { exportChatAllure, readBoundChatExport } from './chat-feature-testops-results.mjs'; +import { executionId, fullName } from './chat-feature-testops-api.mjs'; +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const expected = { runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: 'a'.repeat(40) }; +const selected = { ...expected, projectId: 1, caseId: 91, fullName, executionId }; +async function fixture(t, changes = {}) { + const directory = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-testops-export-'))); + t.after(() => rm(directory, { recursive: true })); + const root = path.join(directory, 'export'); await mkdir(root); + const png = Buffer.from([137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82]); + const shot = { id: 'final-state', file: 'final-state.png', sha256: hash(png) }; + const result = { executionId, suiteId: 'chat-features', caseId: 'create-backlog', profileId: 'runner-opencode-deepseek-1017', provider: 'opencode', + environmentId: 'local', model: 'openrouter/deepseek/deepseek-v4.1-flash-1017', runtimeMode: 'native', attempt: 1, + source: { sha: expected.targetSha }, status: 'passed', cleanup: 'passed', runIds: ['synthetic-run'], durationMs: 1234, + matcherResults: [{ matcher: { kind: 'json_path', path: 'chatFeature.createdTask', expected: true }, passed: true }], screenshots: [shot], ...changes }; + const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2', campaignId: 'gha-123-1', complete: false, results: [result] }))], + ['junit.xml', Buffer.from('')], [`evidence/${executionId}/attempt-1/final-state.png`, png]]); + const binding = { schema: 'paperclip.chat-feature-testops-export/v1', ...expected, executionId, campaignId: 'gha-123-1', resultStatus: result.status, cleanup: result.cleanup, + files: [...files].map(([name, bytes]) => ({ path: name, sha256: hash(bytes), bytes: bytes.length })) }; + for (const [name, bytes] of files) { await mkdir(path.dirname(path.join(root, name)), { recursive: true }); await writeFile(path.join(root, name), bytes); } + await writeFile(path.join(root, 'binding.json'), JSON.stringify(binding)); + return { directory, root, output: path.join(directory, 'allure'), result, binding }; +} + +test('bound synthetic selected result maps to existing Allure case, independent of whole-suite complete', async t => { + const f = await fixture(t); + const value = await exportChatAllure({ ...f, expected, selected }); + assert.equal(value.status, 'passed'); assert.equal(value.count, 1); + const name = (await readdir(f.output)).find(name => name.endsWith('-result.json')); + const result = JSON.parse(await readFile(path.join(f.output, name))); + assert.equal(result.fullName, fullName); assert.deepEqual(result.labels[0], { name: 'AS_ID', value: '91' }); + assert.equal(result.attachments.filter(row => row.type === 'image/png').length, 1); +}); + +test('real failed outcome stays failed and no absence can synthesize a pass', async t => { + const f = await fixture(t, { status: 'failed', cleanup: 'failed' }); + assert.equal((await exportChatAllure({ ...f, expected, selected })).status, 'failed'); + await assert.rejects(exportChatAllure({ root: path.join(f.directory, 'absent'), output: path.join(f.directory, 'missing'), expected, selected })); +}); + +test('foreign source or selection and tampered file hash reject', async t => { + const f = await fixture(t); + await assert.rejects(readBoundChatExport(f.root, { ...expected, runId: '456' })); + await assert.rejects(exportChatAllure({ ...f, expected, selected: { ...selected, projectId: 2 } })); + await writeFile(path.join(f.root, 'junit.xml'), 'tampered'); + await assert.rejects(readBoundChatExport(f.root, expected)); +}); + +test('an unmanifested file, unknown directory or symlink is rejected before upload', async t => { + const f = await fixture(t); + await writeFile(path.join(f.root, 'provider-raw.txt'), 'SYNTHETIC_PRIVATE_NEVER_READ'); + await assert.rejects(readBoundChatExport(f.root, expected)); + await rm(path.join(f.root, 'provider-raw.txt')); + await mkdir(path.join(f.root, 'unknown-empty')); + await assert.rejects(readBoundChatExport(f.root, expected)); + await rm(path.join(f.root, 'unknown-empty'), { recursive: true }); + await rm(path.join(f.root, 'junit.xml')); await symlink(path.join(f.root, 'normalized-results.json'), path.join(f.root, 'junit.xml')); + await assert.rejects(readBoundChatExport(f.root, expected)); +}); + +test('passing result cannot omit independent checks, cleanup or actual feature capture', async t => { + for (const changes of [{ matcherResults: [] }, { cleanup: 'failed' }, { runIds: [] }, { screenshots: [] }, + { matcherResults: [{ matcher: { kind: 'json_path', path: 'chatFeature.createdTask', expected: true }, passed: false }] }]) { + const f = await fixture(t, changes); + await assert.rejects(exportChatAllure({ ...f, expected, selected })); + } +}); diff --git a/tests/runner-e2e/journey-features.ts b/tests/runner-e2e/journey-features.ts index 630e59d1c05..683f170c1e7 100644 --- a/tests/runner-e2e/journey-features.ts +++ b/tests/runner-e2e/journey-features.ts @@ -19,6 +19,7 @@ function associate(suites: readonly string[], cases: Record { - if (context.private !== true || !context.executionIds.includes(executionId) || !/^chat-features\.runner-opencode\.local\.[a-z0-9][a-z0-9.-]{0,159}$/.test(executionId) + if (context.private !== true || !context.executionIds.includes(executionId) || (!/^chat-features\.runner-opencode\.local\.[a-z0-9][a-z0-9.-]{0,159}$/.test(executionId) && executionId !== "chat-features.runner-opencode-deepseek-1017.local.create-backlog") || executionId.includes("..") || !/^[1-9][0-9]{0,19}$/.test(context.runId) || context.attempt !== "1" || !context.cellReportProducer) throw new Error("Invalid private cell report owner"); const root = await realpath(context.root), metadata = await lstat(context.root); @@ -116,8 +116,9 @@ export function cellTraceAsset(relative: string, owner: PrivateCellEvidenceOwner const segments = relative.split("/"); const reserved = segments.some((part, index) => part === "html-report" && segments[index + 1] === "trace"); if (!reserved) return { reserved: false }; - const scenario = owner.executionId.slice("chat-features.runner-opencode.local.".length); - const final = `gha-${owner.runId}-${owner.attempt}-${owner.executionId}/chat-features/runner-opencode/local/${scenario}/attempt-${owner.attempt}/html-report/trace/`; + const profile = owner.executionId === "chat-features.runner-opencode-deepseek-1017.local.create-backlog" ? "runner-opencode-deepseek-1017" : "runner-opencode"; + const scenario = owner.executionId.slice(`chat-features.${profile}.local.`.length); + const final = `gha-${owner.runId}-${owner.attempt}-${owner.executionId}/chat-features/${profile}/local/${scenario}/attempt-${owner.attempt}/html-report/trace/`; const prefix = layout === "original" ? "html-report/trace/" : layout === "final" ? final : `html-report/${final}`; const canonical = relative.startsWith(prefix) ? canonicalGeneratedReportAsset(`html/trace/${relative.slice(prefix.length)}`) : undefined; return { reserved: true, canonical }; diff --git a/tests/runner-e2e/manual-private-retention.ts b/tests/runner-e2e/manual-private-retention.ts index 94f8215b5eb..a5851fd1a8a 100644 --- a/tests/runner-e2e/manual-private-retention.ts +++ b/tests/runner-e2e/manual-private-retention.ts @@ -300,7 +300,7 @@ export function manualPrivateSessionPolicy(input: { bucket: string; runId: strin if (!/^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/.test(input.bucket) || !/^[1-9][0-9]{0,19}$/.test(input.runId)) throw new Error("Invalid manual private storage identity"); const ids = canonicalPrivateManualIds(input.executionIds); - if (ids.some(id => !MANUAL_PRIVATE_EXECUTIONS.includes(id)) || input.mode !== "report-write" && ids.length !== 1) + if (ids.some(id => !MANUAL_PRIVATE_EXECUTIONS.includes(id) && id !== "chat-features.runner-opencode-deepseek-1017.local.create-backlog") || input.mode !== "report-write" && ids.length !== 1) throw new Error("Private archive sessions require one exact cell or the report writer"); const prefix = `arn:aws:s3:::${input.bucket}/runner-e2e-private/bounded-v1/campaigns/gha-${input.runId}-1/`; const cells = ids.map(id => `${prefix}${manualCellKind(id)}.zip`); diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts index 04370a86ca9..3c70f966486 100644 --- a/tests/runner-e2e/manual-private-transfer.ts +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -10,11 +10,12 @@ import { isStandardEvidenceFile } from "./evidence.js"; import { BoundedHistoryPublisher } from "./history-budget.js"; import { privateArtifactKey, retainPrivateTransfer, transferPrivateArtifact, verifyPrivateHistory, type PrivateEvidenceCompleteness } from "./private-history.js"; import { authorizeManualPrivateEvidence, MANUAL_PRIVATE_PROTOCOL, manualCellKind, manualPrivateSessionPolicy } from "./manual-private-retention.js"; +import { shippedChatEvidenceAuthority } from "./testops-chat-admission.js"; const hash = (bytes: string | Buffer) => createHash("sha256").update(bytes).digest("hex"); function request(env: NodeJS.ProcessEnv) { const ids = JSON.parse(env.MANUAL_PRIVATE_IDS ?? "null") as string[]; - const approval = authorizeManualPrivateEvidence({ workflowSha: env.GITHUB_SHA ?? "", targetSha: env.TARGET_SHA ?? "", + const approval = env.PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION !== undefined ? shippedChatEvidenceAuthority(env) : authorizeManualPrivateEvidence({ workflowSha: env.GITHUB_SHA ?? "", targetSha: env.TARGET_SHA ?? "", runId: env.GITHUB_RUN_ID ?? "", attempt: env.GITHUB_RUN_ATTEMPT ?? "", executionIds: ids }, env.MANUAL_PRIVATE_APPROVALS_JSON ?? ""); return { ids, approval, bucket: env.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET ?? "" }; } diff --git a/tests/runner-e2e/playwright-support.config.ts b/tests/runner-e2e/playwright-support.config.ts index 19af159e9bb..f8358898b16 100644 --- a/tests/runner-e2e/playwright-support.config.ts +++ b/tests/runner-e2e/playwright-support.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test"; /** Browser-only harness regressions: no Paperclip instance or provider credentials. */ export default defineConfig({ testDir: ".", - testMatch: ["skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"], + testMatch: ["project-context-readback.spec.ts", "skill-studio-flow.spec.ts", "native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts", "task-search-filter.spec.ts"], workers: 1, retries: 0, timeout: 10_000, diff --git a/tests/runner-e2e/product-journeys/catalog.ts b/tests/runner-e2e/product-journeys/catalog.ts index 0bee75c084e..3d40c2cdc66 100644 --- a/tests/runner-e2e/product-journeys/catalog.ts +++ b/tests/runner-e2e/product-journeys/catalog.ts @@ -3,6 +3,7 @@ export { agentJourneyRuns } from "./success-contract.js"; import type { JourneyId, TargetConfig } from "./types.js"; export const productJourneys = [ { id: "task-create", label: "Create a task for yourself and reopen it", requires: ["company", "board-user", "task-write"], capabilities: ["tasks/create"], paid: false }, + { id: "project-task-context", label: "Create a project and reopen its self-assigned task", requires: ["company", "board-user", "task-write"], capabilities: ["projects/lifecycle", "projects/task-context", "tasks/create"], paid: false }, { id: "ask-question", label: "Ask an agent a question and read its answer", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, { id: "question-answer-resume", label: "Answer agent questions and verify the saved result", requires: ["company", "board-user", "task-write", "agent"], capabilities: ["questions-and-approvals/questions"], paid: true }, { id: "task-properties", label: "Assign a task to the signed-in user", requires: ["company", "board-user", "task-write", "backlog-task"], capabilities: ["tasks/properties"], paid: false }, diff --git a/tests/runner-e2e/product-journeys/contracts.test.ts b/tests/runner-e2e/product-journeys/contracts.test.ts index b02057281a3..f3c20ccb6f7 100644 --- a/tests/runner-e2e/product-journeys/contracts.test.ts +++ b/tests/runner-e2e/product-journeys/contracts.test.ts @@ -13,6 +13,7 @@ import type { RunnerE2EResult } from "../types.js"; import type { JourneyContext, JourneyEvidence, TargetSession } from "./types.js"; import { createConnectionProof } from "../connection-evidence.js"; import { retainOnboardingArtifact } from "./artifact-evidence.js"; +import { journeyRequiredChecks } from "./success-contract.js"; import type { OnboardingConnectEvidence } from "./onboarding-connect.js"; vi.mock("@playwright/test", () => ({ expect: Object.assign( @@ -220,12 +221,17 @@ describe("deployment journey contracts", () => { onboarding.productJourney!.operations.modelCostUsd=null; expect(aggregateCampaignBilling([onboarding,result()]).observedAndEstimatedCostUsd).toBeNull(); }); - it("requires real pass evidence and forbids model work in browser-only passes", () => { - expect(()=>validateRetainedRunnerResult(result())).not.toThrow(); - const missing = result(); missing.productJourney!.checks=[]; + it.each(["task-properties", "task-create", "project-task-context"])("requires real pass evidence and forbids model work in browser-only %s passes", caseId => { + const browser = result(); + browser.caseId = caseId; + browser.executionId = `product-journeys.local-qa.local.${caseId}`; + browser.productJourney!.checks = journeyRequiredChecks[caseId]!.map(id => ({ id, passed: true })); + expect(()=>validateRetainedRunnerResult(browser)).not.toThrow(); + const missing = structuredClone(browser); missing.productJourney!.checks=[]; expect(()=>validateRetainedRunnerResult(missing)).toThrow(); - const paid = result(); paid.runIds=["unexpected"]; - expect(()=>validateRetainedRunnerResult(paid)).toThrow(); + const forbidden: Array> = [{ runIds: ["unexpected"] }, { provider: "openai" }, { model: "unexpected" }, + { runtimeMode: "native" }, { runtimeMode: "legacy" }]; + for (const work of forbidden) expect(()=>validateRetainedRunnerResult({ ...browser, ...work })).toThrow("result.browserOnly"); }); it("keeps browser-only reporting free of invented model calls and rejects public publication", () => { const value=result(); diff --git a/tests/runner-e2e/product-journeys/launch.ts b/tests/runner-e2e/product-journeys/launch.ts index 6abd80fa42b..4be973f1e81 100644 --- a/tests/runner-e2e/product-journeys/launch.ts +++ b/tests/runner-e2e/product-journeys/launch.ts @@ -12,6 +12,7 @@ import { journeyFixtures } from "./fixtures.js"; import { taskProperties } from "./task-properties.js"; import { taskSearchFilter } from "./task-search-filter.js"; import { askQuestion, taskCreate } from "./basic-flows.js"; +import { projectTaskContext } from "./project-context-flow.js"; import { runAgentJourney } from "./agent-flows.js"; import { runClarificationJourney } from "./clarification-flow.js"; import { JourneyError, type JourneyContext, type JourneyEvidence, type JourneyOutcome, type TargetLifecycleInput, type TargetProvider, type TargetSession } from "./types.js"; @@ -105,7 +106,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri try { sha = execFileSync("git", ["rev-parse", "HEAD"], { cwd: repositoryRoot, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim(); } catch { /* Packaged harnesses may not include Git metadata. */ } const source = resolveRunnerE2ESource({ sha, ref: null, workflowRunUrl: null }); const definitionHash = createHash("sha256").update(JSON.stringify(productJourneys)); - for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); + for (const name of ["catalog.ts", "task-properties.ts", "task-search-filter.ts", "task-search-evidence.ts", "basic-flows.ts", "basic-evidence.ts", "project-context-flow.ts", "project-context-readback.ts", "project-context-evidence.ts", "agent-flows.ts", "clarification-flow.ts", "clarification-evidence.ts", "../question-resume-ui.ts", "../question-resume-scoring.ts", "../runtime-question-readiness.ts", "../chat-flow.ts", "../continuation-readiness.ts", "../continuation-cases.ts", "success-contract.ts", "product-screenshot.ts", "cloud-onboarding.ts", "onboarding-connect.ts", "onboarding-first-task.ts", "workspace-deferral.ts", "onboarding-pre-acceptance.ts", "artifact-evidence.ts", "fixtures.ts", "launch.ts", "types.ts", "collection.ts", "local-target.ts"]) definitionHash.update(await readFile(path.join(import.meta.dirname, name))); for (const file of ["catalog.ts", "api.ts", "run-observations.ts", "native-task-reply.ts", "native-event-envelope.ts", "user-actions.ts", "plan-revision-flow.ts", "skill-studio-flow.ts", "protected-action-flow.ts", "story-artifact.ts", "stop-redirect-flow.ts", "everyday-cases.ts", "everyday-delivery.ts", "everyday-decisions.ts", "connection-guidance-evidence.ts", "everyday-artifact.py"]) definitionHash.update(await readFile(path.join(import.meta.dirname, "..", file))); definitionHash.update(runnerSuites.flatMap(suite => suite.tasks).find(task => task.id === "ask-question")!.buildPrompt("definition")); const suiteDefinitionHash = definitionHash.digest("hex"); @@ -217,6 +218,7 @@ export async function launchProductJourneys(args: string[], repositoryRoot: stri await captureProductScreenshot(ready!.page, ready!.api.origin, path.join(directory, file), deadlineAt); } }); } else if (id === "task-create") await taskCreate(ready, session!, campaignId, evidence); + else if (id === "project-task-context") await projectTaskContext(ready, session!, campaignId, evidence); else if (id === "task-search-filter") await taskSearchFilter(ready, session!, campaignId, evidence); else await taskProperties(ready, session!, campaignId, evidence); } diff --git a/tests/runner-e2e/product-journeys/project-context-evidence.test.ts b/tests/runner-e2e/product-journeys/project-context-evidence.test.ts new file mode 100644 index 00000000000..1caa6986eb2 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-evidence.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { parseTargetConfig, preparationPreview, productJourneys } from "./catalog.js"; +import { projectContextEvidencePasses, type ProjectContextObservation } from "./project-context-evidence.js"; +import { journeyRequiredChecks } from "./success-contract.js"; + +function observation(): ProjectContextObservation { + const project = { id: "project", companyId: "company", name: "Journey project campaign", status: "planned", archivedAt: null }; + const task = { id: "task", companyId: "company", title: "Journey task campaign", description: "Review this myself.", projectId: "project", + assigneeUserId: "user", assigneeAgentId: null, workMode: "standard" }; + return { companyId: "company", userId: "user", projectId: "project", projectName: project.name, + taskId: "task", taskTitle: task.title, prompt: task.description, + projectBeforeReload: { ...project }, projectAfterReload: { ...project }, taskBeforeReload: { ...task }, taskAfterReload: { ...task }, + projectTasks: [{ ...task }], runsBeforeReload: [], runsAfterReload: [] }; +} + +describe("project/task context independent saved-state oracle", () => { + it("accepts exact persisted project, task, assignment and work mode with no work runs", () => { + expect(projectContextEvidencePasses(observation())).toBe(true); + }); + it.each(["projectBeforeReload", "projectAfterReload"] as const)("rejects a wrong or missing %s", key => { + for (const patch of [{ id: "foreign" }, { companyId: "foreign" }, { name: "another project" }, { status: "completed" }, { archivedAt: "2099-01-01T00:00:00Z" }]) { + const saved = observation(); + Object.assign(saved[key], patch); + expect(projectContextEvidencePasses(saved)).toBe(false); + } + const missing = observation(); + Reflect.deleteProperty(missing, key); + expect(projectContextEvidencePasses(missing)).toBe(false); + }); + it.each(["taskBeforeReload", "taskAfterReload"] as const)("rejects plausible false passes in %s", key => { + for (const patch of [{ id: "foreign" }, { companyId: "foreign" }, { title: "wrong" }, { description: "lost prompt" }, { projectId: null }, + { projectId: "foreign" }, { assigneeUserId: "foreign" }, { assigneeAgentId: "agent" }, { workMode: "ask" }]) { + const saved = observation(); + Object.assign(saved[key], patch); + expect(projectContextEvidencePasses(saved)).toBe(false); + } + const missing = observation(); + Reflect.deleteProperty(missing, key); + expect(projectContextEvidencePasses(missing)).toBe(false); + }); + it("requires exact project-filtered membership and rejects any native work before or after reload", () => { + const changes: Array<(saved: ProjectContextObservation) => void> = [ + saved => { saved.projectTasks = []; }, saved => { saved.projectTasks.push({ ...saved.projectTasks[0]! }); }, + saved => { saved.projectTasks[0]!.projectId = "foreign"; }, saved => { saved.projectTasks[0]!.companyId = "foreign"; }, + saved => { saved.projectTasks[0]!.id = "another task"; }, saved => { saved.runsBeforeReload = [{ id: "run" }]; }, + saved => { saved.runsAfterReload = [{ id: "run" }]; }, saved => { Reflect.deleteProperty(saved, "runsAfterReload"); }, + saved => { Reflect.deleteProperty(saved, "projectTasks"); }, saved => { saved.projectId = ""; }, + ]; + for (const change of changes) { const saved = observation(); change(saved); expect(projectContextEvidencePasses(saved)).toBe(false); } + }); + it("is discoverable as a browser journey with zero model budget or native work", () => { + const config = parseTargetConfig({ version: 1, id: "project-qa", mode: "managed-self-hosted", journeys: ["project-task-context"], + executionEnvironment: "local", baseline: { features: {}, connections: [] }, limits: { durationMs: 300_000, modelBudgetCents: 0, maxRuns: 0 } }); + const entry = productJourneys.find(journey => journey.id === "project-task-context")!; + expect(entry.paid).toBe(false); + expect(entry.requires).toEqual(["company", "board-user", "task-write"]); + expect(preparationPreview(config).journeys).toEqual([entry]); + expect(journeyRequiredChecks[entry.id]).toEqual(["browser-created-project-persisted", "browser-created-project-task-persisted", + "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"]); + }); +}); diff --git a/tests/runner-e2e/product-journeys/project-context-evidence.ts b/tests/runner-e2e/product-journeys/project-context-evidence.ts new file mode 100644 index 00000000000..393b9e82053 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-evidence.ts @@ -0,0 +1,51 @@ +interface ProjectSnapshot { + id: string; + companyId: string; + name: string; + status: string; + archivedAt: string | null; + urlKey?: string | null; +} +interface TaskSnapshot { + id: string; + companyId: string; + title: string; + description: string | null; + projectId: string | null; + assigneeUserId: string | null; + assigneeAgentId: string | null; + workMode: string; + identifier?: string | null; +} +export interface ProjectContextObservation { + companyId: string; + userId: string; + projectId: string; + projectName: string; + taskId: string; + taskTitle: string; + prompt: string; + projectBeforeReload: ProjectSnapshot; + projectAfterReload: ProjectSnapshot; + taskBeforeReload: TaskSnapshot; + taskAfterReload: TaskSnapshot; + projectTasks: TaskSnapshot[]; + runsBeforeReload: unknown[]; + runsAfterReload: unknown[]; +} + +/** Compare independent saved reads against the submitted UI values, not the POST echo. */ +export function projectContextEvidencePasses(observed: ProjectContextObservation): boolean { + const { companyId, userId, projectId, projectName, taskId, taskTitle, prompt } = observed; + if (![companyId, userId, projectId, projectName, taskId, taskTitle, prompt].every(value => typeof value === "string" && value.length)) return false; + const projectMatches = (project: ProjectSnapshot) => project?.id === projectId && project.companyId === companyId + && project.name === projectName && project.status === "planned" && project.archivedAt === null; + const taskMatches = (task: TaskSnapshot) => task?.id === taskId && task.companyId === companyId && task.title === taskTitle + && task.description === prompt && task.projectId === projectId && task.assigneeUserId === userId + && task.assigneeAgentId === null && task.workMode === "standard"; + return projectMatches(observed.projectBeforeReload) && projectMatches(observed.projectAfterReload) + && taskMatches(observed.taskBeforeReload) && taskMatches(observed.taskAfterReload) + && Array.isArray(observed.projectTasks) && observed.projectTasks.length === 1 && taskMatches(observed.projectTasks[0]!) + && Array.isArray(observed.runsBeforeReload) && observed.runsBeforeReload.length === 0 + && Array.isArray(observed.runsAfterReload) && observed.runsAfterReload.length === 0; +} diff --git a/tests/runner-e2e/product-journeys/project-context-flow.ts b/tests/runner-e2e/product-journeys/project-context-flow.ts new file mode 100644 index 00000000000..2dae9213854 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-flow.ts @@ -0,0 +1,53 @@ +import { expect } from "@playwright/test"; +import { createTaskThroughUi } from "../user-actions.js"; +import { projectContextEvidencePasses, type ProjectContextObservation } from "./project-context-evidence.js"; +import { openProjectTaskThroughList, reopenProjectThroughList } from "./project-context-readback.js"; +import { JourneyError, type JourneyContext, type JourneyEvidence, type TargetSession } from "./types.js"; + +export async function projectTaskContext(context: JourneyContext, session: TargetSession, campaignId: string, evidence: JourneyEvidence) { + const { page, api, company } = context; + const projectName = `Journey QA project ${campaignId}`; + const taskTitle = `Journey QA project task ${campaignId}`; + const prompt = `Keep this task assigned to me in ${projectName}. I will review it myself.`; + const journeyId = "project-task-context" as const; + await page.goto(`${api.origin}/${company.issuePrefix}/projects`); + await page.getByRole("button", { name: "Add Project", exact: true }).first().click(); + const dialog = page.getByRole("dialog"); + await dialog.getByRole("textbox", { name: "Project name", exact: true }).fill(projectName); + await session.own({ kind: "project", name: projectName, journeyId }); + const responsePromise = page.waitForResponse(response => response.request().method() === "POST" + && new URL(response.url()).pathname === `/api/companies/${company.id}/projects`); + await dialog.getByRole("button", { name: "Create project", exact: true }).click(); + const response = await responsePromise; + if (response.status() !== 201) throw new JourneyError("failed", "project_ui_creation_failed"); + const createdProject = await response.json(); + if (typeof createdProject.id !== "string" || !createdProject.id || createdProject.companyId !== company.id || createdProject.name !== projectName) { + throw new JourneyError("failed", "project_ui_creation_mismatch"); + } + await session.own({ kind: "project", name: projectName, id: createdProject.id, journeyId }); + await expect(dialog).not.toBeVisible(); + const projectRoute = `/api/projects/${createdProject.id}`; + const projectBeforeReload = await api.get(projectRoute); + const task = await createTaskThroughUi({ page, origin: api.origin, issuePrefix: company.issuePrefix, + agentName: "Me", title: taskTitle, prompt, workMode: "standard", projectName, requireExplicitTitle: true, + beforeDispatch: () => session.own({ kind: "issue", name: taskTitle, journeyId }) }); + await session.own({ kind: "issue", name: taskTitle, id: task.issueId, journeyId }); + const taskRoute = `/api/issues/${task.issueId}`; + const taskBeforeReload = await api.get(taskRoute); + const runsBeforeReload = await api.get(`${taskRoute}/runs`); + + // Open the project from its list, then open its task from that project's Tasks view. + await reopenProjectThroughList({ page, origin: api.origin, issuePrefix: company.issuePrefix, projectName, + projectRef: projectBeforeReload.urlKey ?? createdProject.id }); + const projectTasks = await api.get(`/api/companies/${company.id}/issues?projectId=${createdProject.id}&limit=100&offset=0`); + await openProjectTaskThroughList({ page, origin: api.origin, issuePrefix: company.issuePrefix, projectName, taskTitle, prompt, + taskRef: taskBeforeReload.identifier ?? task.issueId }); + const observed: ProjectContextObservation = { + companyId: company.id, userId: context.user.id, projectId: createdProject.id, projectName, + taskId: task.issueId, taskTitle, prompt, projectBeforeReload, taskBeforeReload, projectTasks, runsBeforeReload, + projectAfterReload: await api.get(projectRoute), taskAfterReload: await api.get(taskRoute), runsAfterReload: await api.get(`${taskRoute}/runs`), + }; + if (!projectContextEvidencePasses(observed)) throw new JourneyError("failed", "project_task_context_mismatch"); + evidence.checks.push(...["browser-created-project-persisted", "browser-created-project-task-persisted", + "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"].map(id => ({ id, passed: true }))); +} diff --git a/tests/runner-e2e/product-journeys/project-context-readback.ts b/tests/runner-e2e/product-journeys/project-context-readback.ts new file mode 100644 index 00000000000..498e9c7a5f3 --- /dev/null +++ b/tests/runner-e2e/product-journeys/project-context-readback.ts @@ -0,0 +1,48 @@ +import { expect, type Page } from "@playwright/test"; +import { JourneyError } from "./types.js"; + +type BrowserStage = "project_list" | "project_open" | "project_task_open" | "project_property"; +async function browserStep(stage: BrowserStage, action: () => Promise): Promise { + try { return await action(); } + catch (error) { + if (error instanceof JourneyError) throw error; + // Playwright errors can contain page content. Retain only the closed stage. + throw new JourneyError("failed", `project_context_${stage}_failed`); + } +} + +export async function reopenProjectThroughList(input: { page: Page; origin: string; issuePrefix: string; projectName: string; projectRef: string }) { + const { page, origin, issuePrefix, projectName, projectRef } = input; + await browserStep("project_list", () => page.goto(`${origin}/${issuePrefix}/projects`)); + await browserStep("project_open", async () => { + const link = page.locator("main#main-content").getByRole("link").filter({ hasText: projectName }); + await expect(link).toHaveCount(1); + await link.click(); + const expected = `${origin}/${issuePrefix}/projects/${encodeURIComponent(projectRef)}/issues`; + await expect.poll(() => { const url = new URL(page.url()); return `${url.origin}${url.pathname}`; }).toBe(expected); + await page.reload(); + await expect(page.locator("main#main-content").getByRole("heading", { name: projectName, exact: true })).toBeVisible(); + }); +} + +export async function openProjectTaskThroughList(input: { page: Page; origin: string; issuePrefix: string; projectName: string; taskTitle: string; taskRef: string; prompt: string }) { + const { page, origin, issuePrefix, projectName, taskTitle, taskRef, prompt } = input; + await browserStep("project_task_open", async () => { + const link = page.locator("main#main-content").getByRole("link").filter({ hasText: taskTitle }); + await expect(link).toHaveCount(1); + await link.click(); + const expected = `${origin}/${issuePrefix}/issues/${encodeURIComponent(taskRef)}`; + await expect.poll(() => { const url = new URL(page.url()); return `${url.origin}${url.pathname}`; }).toBe(expected); + await page.reload(); + const main = page.locator("main#main-content"); + await expect(main.getByText(taskTitle, { exact: true }).filter({ visible: true }).first()).toBeVisible(); + await expect(main.getByText(prompt, { exact: true }).filter({ visible: true }).first()).toBeVisible(); + }); + await browserStep("project_property", async () => { + const properties = page.getByRole("button", { name: /^(Show properties|Properties)$/ }).filter({ visible: true }).first(); + if (await properties.isVisible()) await properties.click(); + const projectProperty = page.locator("[data-property-row]").filter({ has: page.locator('[data-property-label="Project"]') }).filter({ visible: true }); + await expect(projectProperty).toHaveCount(1); + await expect(projectProperty.locator("[data-property-value]")).toContainText(projectName); + }); +} diff --git a/tests/runner-e2e/product-journeys/success-contract.ts b/tests/runner-e2e/product-journeys/success-contract.ts index 8c1ddb66b94..f49db32a938 100644 --- a/tests/runner-e2e/product-journeys/success-contract.ts +++ b/tests/runner-e2e/product-journeys/success-contract.ts @@ -11,6 +11,7 @@ export function agentJourneyRunCountValid(id: string, count: unknown): boolean { export const journeyRequiredChecks: Record = { "task-properties": ["assignment-persisted", "no-agent-work"], "task-create": ["browser-created-task-persisted", "no-agent-work"], + "project-task-context": ["browser-created-project-persisted", "browser-created-project-task-persisted", "project-task-association-reloaded", "project-task-context-visible", "no-agent-work"], "task-search-filter": ["task-search-results-exact", "task-status-filter-excludes-controls", "task-search-filter-reloaded", "task-search-result-reopened", "no-agent-work"], "ask-question": ["question-answer-persisted", "one-bound-successful-run", "configured-model-and-environment"], "question-answer-resume": ["question-resume.forms", "question-resume.choice.answer", "question-resume.choice.path", diff --git a/tests/runner-e2e/product-journeys/types.ts b/tests/runner-e2e/product-journeys/types.ts index 90f658debb0..0f899efca5e 100644 --- a/tests/runner-e2e/product-journeys/types.ts +++ b/tests/runner-e2e/product-journeys/types.ts @@ -2,7 +2,7 @@ import type { BrowserContext, Page } from "@playwright/test"; import type { ConnectionApi } from "../connection-target.js"; export type JourneyOutcome = "passed" | "failed" | "blocked" | "unsupported"; -export type JourneyId = "task-properties" | "task-create" | "ask-question" | "cloud-onboarding" +export type JourneyId = "task-properties" | "task-create" | "project-task-context" | "ask-question" | "cloud-onboarding" | "question-answer-resume" | "task-search-filter" | "plan-revise-accept" | "build-revise" | "protected-actions" | "create-skill-studio" | "stop-redirect"; export type TargetMode = "managed-self-hosted" | "existing-cloud" | "new-cloud"; export interface DeploymentIdentity { @@ -111,7 +111,7 @@ export interface TargetSession { /** Publish the observed finalization failure without releasing resources or leases. */ checkpointEvidence?(input?: TargetLifecycleInput): Promise; /** Durable intent before creation; receipt immediately after creation. */ - own(resource: { kind: "issue" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId }): Promise; + own(resource: { kind: "issue" | "project" | "agent" | "connection" | "tool-connection" | "skill"; name: string; id?: string; journeyId?: JourneyId }): Promise; } export interface TargetProvider { version: 1; diff --git a/tests/runner-e2e/project-context-readback.spec.ts b/tests/runner-e2e/project-context-readback.spec.ts new file mode 100644 index 00000000000..fb04df62bea --- /dev/null +++ b/tests/runner-e2e/project-context-readback.spec.ts @@ -0,0 +1,51 @@ +import { expect, test, type Page } from "@playwright/test"; +import { openProjectTaskThroughList, reopenProjectThroughList } from "./product-journeys/project-context-readback.js"; + +const origin = "https://project-context.test", projectName = "Journey QA project owned-campaign"; +const taskTitle = "Journey QA project task owned-campaign", prompt = "I will review this task myself."; +const projectPath = "/QA/projects/owned-project-f962db91/issues", taskPath = "/QA/issues/QA-201"; +const projectInput = { origin, issuePrefix: "QA", projectName, projectRef: "owned-project-f962db91" }; +const taskInput = { origin, issuePrefix: "QA", projectName, taskTitle, prompt, taskRef: "QA-201" }; + +async function fixture(page: Page, changes: { missingProject?: boolean; wrongProject?: boolean; missingTask?: boolean; wrongTask?: boolean; wrongProperty?: boolean } = {}) { + const requests: string[] = []; + await page.route(`${origin}/**`, async route => { + const path = new URL(route.request().url()).pathname; + requests.push(path); + // Layout can retain a hidden sidebar label. ProjectDetail renders its + // production InlineEditor as h2; IssueRow has an overlay link with sr-only text. + const sidebar = ``; + const body = path === "/QA/projects" ? `${sidebar}
${changes.missingProject ? "" : `${projectName}`}
` + : path.includes("/projects/") ? `${sidebar}

${projectName}

${changes.missingTask ? "" : ``}
` + : `${sidebar}

${taskTitle}

${prompt}

`; + await route.fulfill({ contentType: "text/html", body: `${body}` }); + }); + return requests; +} + +test("reopens and reloads the owned project and task despite hidden duplicate sidebar labels", async ({ page }) => { + const requests = await fixture(page); + await reopenProjectThroughList({ ...projectInput, page }); + await openProjectTaskThroughList({ ...taskInput, page }); + expect(requests.filter(path => path === projectPath)).toHaveLength(2); + expect(requests.filter(path => path === taskPath)).toHaveLength(2); + await expect(page.locator("main#main-content [data-property-value]")).toContainText(projectName); +}); + +for (const [name, changes, code] of [ + ["missing project list row", { missingProject: true }, "project_context_project_open_failed"], + ["wrong project route with a matching label", { wrongProject: true }, "project_context_project_open_failed"], +] as const) test(`rejects ${name} with a closed stage code`, async ({ page }) => { + await fixture(page, changes); + await expect(reopenProjectThroughList({ ...projectInput, page })).rejects.toMatchObject({ outcome: "failed", code, message: code }); +}); + +for (const [name, changes, code] of [ + ["missing task in the project", { missingTask: true }, "project_context_project_task_open_failed"], + ["wrong task route with a matching label", { wrongTask: true }, "project_context_project_task_open_failed"], + ["wrong saved project property", { wrongProperty: true }, "project_context_project_property_failed"], +] as const) test(`rejects ${name} with a closed stage code`, async ({ page }) => { + await fixture(page, changes); + await page.goto(`${origin}${projectPath}`); + await expect(openProjectTaskThroughList({ ...taskInput, page })).rejects.toMatchObject({ outcome: "failed", code, message: code }); +}); diff --git a/tests/runner-e2e/report-catalog.test.ts b/tests/runner-e2e/report-catalog.test.ts index 2560c28971a..28929783eb1 100644 --- a/tests/runner-e2e/report-catalog.test.ts +++ b/tests/runner-e2e/report-catalog.test.ts @@ -10,7 +10,7 @@ import { discoverReportCatalog } from "./report-catalog.js"; import { buildRunnerCampaign } from "./history.js"; import { renderRunnerE2EDashboard } from "./dashboard.js"; import { regenerateRunnerDashboard } from "./dashboard-regenerate.js"; -import { journeyCapabilities } from "./journey-features.js"; +import { journeyCapabilities, journeyFeatureIds } from "./journey-features.js"; import type { RunnerE2EResult } from "./types.js"; const temporary: string[] = []; @@ -91,6 +91,14 @@ describe("trusted report catalog discovery", () => { expect(journeyCapabilities({ ...product, task: { ...known.task, id: "task-search-filter" } }).map(capability => capability.id)).toEqual(["tasks/organize"]); }); + it("counts the project/task deployment journey toward projects and task creation coverage", () => { + const known = runnerMatrix.find(execution => execution.id === "core-compatibility.legacy-codex.local.ask-question")!; + const project = { ...known, suite: { ...known.suite, id: "product-journeys" }, task: { ...known.task, id: "project-task-context" } }; + expect(journeyCapabilities(project).map(capability => capability.id)).toEqual(["projects/lifecycle", "projects/task-context", "tasks/create"]); + expect(journeyFeatureIds(project)).toEqual(["projects", "tasks"]); + expect(journeyCapabilities({ ...project, suite: { ...project.suite, id: "unrecognized-suite" } })).toEqual([]); + }); + it("aggregates unknown suites and renders their cases, missing results, and screenshot gallery", () => { const summary = campaign(); expect(summary.suites).toEqual([ diff --git a/tests/runner-e2e/result-validation.ts b/tests/runner-e2e/result-validation.ts index be027c62bec..d85b195394e 100644 --- a/tests/runner-e2e/result-validation.ts +++ b/tests/runner-e2e/result-validation.ts @@ -434,7 +434,7 @@ export function validateRetainedRunnerResult( if (!evidence) return invalid("result.productJourney"); const required = journeyRequiredChecks[checked.caseId]; if (evidence.outcome !== "passed" || !evidence.deployment || !required || !required.every(id => evidence.checks.some(c => c.id === id && c.passed)) || evidence.checks.some(check => !check.passed) || evidence.phases.some(p => p.outcome !== "passed") || checked.cleanup !== "passed") invalid("result.productJourney"); - if (["task-properties", "task-create", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); + if (["task-properties", "task-create", "project-task-context", "task-search-filter"].includes(checked.caseId) && (checked.runtimeMode !== "none" || checked.provider !== null || checked.model !== null || checked.runIds?.length)) invalid("result.browserOnly"); if (Object.hasOwn(agentJourneyRuns, checked.caseId) && (!agentJourneyRunCountValid(checked.caseId, checked.runIds?.length) || !checked.model || !checked.provider || checked.runtimeMode !== "native")) invalid("result.agentJourneyRun"); if (checked.caseId === "cloud-onboarding" && (!checked.runIds?.length || !evidence.operations || !evidence.operations.requestCount || !evidence.operations.traceIds.length || evidence.operations.campaignId !== evidence.campaignId || evidence.operations.attemptId !== evidence.attemptId)) invalid("result.paidJourneyReceipt"); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index ee7e6dfd1f2..711798d1c71 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -21,7 +21,8 @@ delete env.NODE_OPTIONS; for (const [name, command] of [ ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], ["Node tests", ["--test", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", - "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs"]], + "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", + "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs"]], ]) { const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); if (result.error || result.signal) { diff --git a/tests/runner-e2e/testops-chat-admission.test.ts b/tests/runner-e2e/testops-chat-admission.test.ts new file mode 100644 index 00000000000..af7759b447c --- /dev/null +++ b/tests/runner-e2e/testops-chat-admission.test.ts @@ -0,0 +1,154 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterEach, expect, it, vi } from "vitest"; +import { runnerMatrix } from "./catalog.js"; +import { authorizeShippedChat, shippedChatEvidenceAuthority } from "./testops-chat-admission.js"; + +vi.mock("node:fs", async importOriginal => { + const actual = await importOriginal(); + return { ...actual, readFileSync: vi.fn(actual.readFileSync) }; +}); + +const wrapperDirectories: string[] = []; +afterEach(async () => { + vi.restoreAllMocks(); + const actual = await vi.importActual("node:fs"); + vi.mocked(readFileSync).mockImplementation(actual.readFileSync); + for (const directory of wrapperDirectories.splice(0)) rmSync(directory, { recursive: true, force: true }); +}); + +const id = "chat-features.runner-opencode-deepseek-1017.local.create-backlog"; +function fixture() { + const env: NodeJS.ProcessEnv = { PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: "v1", GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", + GITHUB_JOB: "chat", GITHUB_SHA: "a".repeat(40), TARGET_SHA: "a".repeat(40), TESTOPS_ADMITTED_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", + GITHUB_ACTOR_ID: "123", TESTOPS_TRIGGERING_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]", MANUAL_PRIVATE_IDS: JSON.stringify([id]), SELECTED_EXECUTION_ID: id }; + const event = { ref: "refs/heads/master", sender: { id: 123 }, repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, inputs: { ALLURE_JOB_RUN_ID: "789", operation: "evaluate" } as Record }; + const selected = runnerMatrix.find(row => row.id === id)!; + return { env, event, selected }; +} +it("binds the actual shipped selection to immutable main admission without candidate approval variables", () => { + const f = fixture(); const result = authorizeShippedChat(f.env, f.event, f.selected, "b".repeat(64)); + expect(result).toMatchObject({ authority: "paperclip.shipped-chat-testops/v1", private: true, runId: "12345", attempt: "1", workflowSha: f.env.GITHUB_SHA, targetSha: f.env.GITHUB_SHA, executionIds: [id] }); + expect(result.authoritySha256).toMatch(/^[a-f0-9]{64}$/); + expect(authorizeShippedChat({ ...f.env, LATEST_MASTER_SHA: "c".repeat(40), GITHUB_JOB: "report" }, f.event, f.selected, "b".repeat(64)).authoritySha256).toBe(result.authoritySha256); +}); +it.each([ + ["unknown mode", { PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: "unknown" }], + ["untrusted repo", { GITHUB_REPOSITORY: "foreign/repo" }], + ["nondefault ref", { GITHUB_REF: "refs/heads/feature" }], + ["wrong workflow caller", { GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master" }], + ["source substitution", { TARGET_SHA: "c".repeat(40) }], + ["admitted source mismatch", { TESTOPS_ADMITTED_SHA: "c".repeat(40) }], + ["replay", { GITHUB_RUN_ATTEMPT: "2" }], + ["unlisted actor", { TESTOPS_ALLOWED_ACTOR_IDS: "[456]" }], + ["untrusted job", { GITHUB_JOB: "build" }], + ["foreign selector", { SELECTED_EXECUTION_ID: "other" }], +])("rejects %s", (_name, patch) => { const f = fixture(); expect(() => authorizeShippedChat({ ...f.env, ...patch }, f.event, f.selected, "b".repeat(64))).toThrow(); }); +it.each(["target_branch", "profile", "model", "retries", "timeout"])("rejects the undeclared %s input", field => { + const f = fixture(); f.event.inputs[field] = "override"; expect(() => authorizeShippedChat(f.env, f.event, f.selected, "b".repeat(64))).toThrow(); +}); +it("rejects metadata bootstrap as an evaluation admission", () => { const f = fixture(); f.event.inputs.operation = "bootstrap"; expect(() => authorizeShippedChat(f.env, f.event, f.selected, "b".repeat(64))).toThrow(); }); +it("rejects forged event sender and foreign event source", () => { + const f = fixture(); f.event.sender.id = 456; expect(() => authorizeShippedChat(f.env, f.event, f.selected, "b".repeat(64))).toThrow(); + f.event.sender.id = 123; f.event.repository.full_name = "foreign/repo"; expect(() => authorizeShippedChat(f.env, f.event, f.selected, "b".repeat(64))).toThrow(); +}); +it("rejects the existing production OpenCode backlog cell", () => { + const f = fixture(); + const stock = runnerMatrix.find(row => row.id === "chat-features.runner-opencode.local.create-backlog")!; + expect(() => authorizeShippedChat(f.env, f.event, stock, "b".repeat(64))).toThrow("Unexpected shipped chat selection or bounds"); +}); +it.each(["model", "provider", "work", "clock", "retry", "budget"])("rejects changed selected %s", field => { + const f = fixture(); const selected = { ...f.selected, profile: { ...f.selected.profile }, task: { ...f.selected.task }, suite: { ...f.selected.suite, definitionMetadata: { ...f.selected.suite.definitionMetadata } } }; + if (field === "model") selected.profile.model = "other/model"; + if (field === "provider") selected.profile.provider = "codex"; + if (field === "work") selected.task.expectedRunCount = 2; + if (field === "clock") selected.task.turnTimeoutMs = 121000; + if (field === "retry") selected.task.automaticRetryPolicy = undefined; + if (field === "budget") selected.suite.definitionMetadata.companyAndAgentBudgetCents = 1000; + expect(() => authorizeShippedChat(f.env, f.event, selected, "b".repeat(64))).toThrow(); +}); + +async function wrapperFixture() { + const actual = await vi.importActual("node:fs"); + const directory = mkdtempSync(path.join(tmpdir(), "testops-chat-authority-")); + wrapperDirectories.push(directory); + const f = fixture(), eventPath = path.join(directory, "event.json"), launchPath = path.join(directory, "launch.ts"); + const canonicalLaunch = new URL("./launch.ts", import.meta.url); + writeFileSync(eventPath, JSON.stringify(f.event)); + writeFileSync(launchPath, actual.readFileSync(canonicalLaunch)); + vi.mocked(readFileSync).mockImplementation(actual.readFileSync).mockClear(); + const env: NodeJS.ProcessEnv = { ...f.env, GITHUB_EVENT_PATH: eventPath }; + return { ...f, env, directory, eventPath, launchPath, canonicalLaunch, actual }; +} + +it("admits the real filesystem wrapper with the actual catalog and canonical launcher without fixture callbacks", async () => { + const f = await wrapperFixture(); + const callbacks = [vi.spyOn(f.selected.profile, "buildAgent"), vi.spyOn(f.selected.task, "buildPrompt")]; + const result = shippedChatEvidenceAuthority(f.env); + expect(result).toMatchObject({ authority: "paperclip.shipped-chat-testops/v1", workflowSha: f.env.GITHUB_SHA, + targetSha: f.env.GITHUB_SHA, runId: f.env.GITHUB_RUN_ID, attempt: "1", executionIds: [id], private: true, + launchSha256: createHash("sha256").update(f.actual.readFileSync(f.canonicalLaunch)).digest("hex") }); + expect(Object.isFrozen(result)).toBe(true); + expect(callbacks.every(callback => callback.mock.calls.length === 0)).toBe(true); + expect(vi.mocked(readFileSync).mock.calls.map(([file]) => String(file))).toEqual([f.eventPath, String(f.canonicalLaunch)]); +}); + +it.each(["missing", "absent", "directory", "symlink", "oversize", "invalid-json"])("rejects %s event custody through the real wrapper", async kind => { + const f = await wrapperFixture(); + if (kind === "missing") delete f.env.GITHUB_EVENT_PATH; + if (kind === "absent") rmSync(f.eventPath); + if (kind === "directory") { rmSync(f.eventPath); mkdirSync(f.eventPath); } + if (kind === "symlink") { + const target = path.join(f.directory, "real-event.json"); writeFileSync(target, JSON.stringify(f.event)); + rmSync(f.eventPath); symlinkSync(target, f.eventPath); + } + if (kind === "oversize") writeFileSync(f.eventPath, " ".repeat(65_537)); + if (kind === "invalid-json") writeFileSync(f.eventPath, "{invalid"); + expect(() => shippedChatEvidenceAuthority(f.env)).toThrow(); + expect(vi.mocked(readFileSync).mock.calls.some(([file]) => String(file) === String(f.canonicalLaunch))).toBe(false); +}); + +it.each(["sender", "bootstrap", "model-input", "source"])("rejects a tampered %s through event parsing and wrapper admission", async kind => { + const f = await wrapperFixture(); + if (kind === "sender") f.event.sender.id = 456; + if (kind === "bootstrap") f.event.inputs.operation = "bootstrap"; + if (kind === "model-input") f.event.inputs.model = "foreign/model"; + if (kind === "source") f.env.TARGET_SHA = "c".repeat(40); + writeFileSync(f.eventPath, JSON.stringify(f.event)); + expect(() => shippedChatEvidenceAuthority(f.env)).toThrow("Invalid shipped chat authority"); +}); + +it.each(["missing", "model"])("rejects %s selected catalog metadata in the actual wrapper", async kind => { + const f = await wrapperFixture(); + const find = vi.spyOn(runnerMatrix, "find"); + if (kind === "missing") find.mockReturnValue(undefined); + else find.mockReturnValue({ ...f.selected, profile: { ...f.selected.profile, model: "foreign/model" } }); + expect(() => shippedChatEvidenceAuthority(f.env)).toThrow(kind === "missing" ? "Shipped chat execution is absent" : "Unexpected shipped chat selection or bounds"); +}); + +it.each(["absent", "directory", "denied"])("propagates a %s launcher read from a test-only filesystem port", async kind => { + const f = await wrapperFixture(); + if (kind === "absent") rmSync(f.launchPath); + if (kind === "directory") { rmSync(f.launchPath); mkdirSync(f.launchPath); } + vi.mocked(readFileSync).mockImplementation((file, options) => { + if (String(file) !== String(f.canonicalLaunch)) return f.actual.readFileSync(file, options); + if (kind === "denied") throw Object.assign(new Error("Owned launcher unreadable"), { code: "EACCES" }); + return f.actual.readFileSync(f.launchPath, options); + }); + expect(() => shippedChatEvidenceAuthority(f.env)).toThrow(expect.objectContaining({ code: kind === "absent" ? "ENOENT" : kind === "directory" ? "EISDIR" : "EACCES" })); +}); + +it("binds changed real launcher bytes to new authority rather than inventing an independent trusted hash", async () => { + const f = await wrapperFixture(); + vi.mocked(readFileSync).mockImplementation((file, options) => f.actual.readFileSync( + String(file) === String(f.canonicalLaunch) ? f.launchPath : file, options)); + const original = shippedChatEvidenceAuthority(f.env); + writeFileSync(f.launchPath, "source-only changed launcher bytes\n"); + const changed = shippedChatEvidenceAuthority(f.env); + expect(changed.launchSha256).toBe(createHash("sha256").update(f.actual.readFileSync(f.launchPath)).digest("hex")); + expect(changed.launchSha256).not.toBe(original.launchSha256); + expect(changed.authoritySha256).not.toBe(original.authoritySha256); +}); diff --git a/tests/runner-e2e/testops-chat-admission.ts b/tests/runner-e2e/testops-chat-admission.ts new file mode 100644 index 00000000000..e391e9af67e --- /dev/null +++ b/tests/runner-e2e/testops-chat-admission.ts @@ -0,0 +1,48 @@ +import { createHash } from "node:crypto"; +import { lstatSync, readFileSync } from "node:fs"; +import { runnerMatrix } from "./catalog.js"; +import { MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +import type { MatrixExecution } from "./types.js"; + +const executionId = "chat-features.runner-opencode-deepseek-1017.local.create-backlog"; +const authority = "paperclip.shipped-chat-testops/v1"; +const repository = "paperclipai/paperclip"; +const workflow = ".github/workflows/chat-feature-testops.yml"; + +export function authorizeShippedChat(env: NodeJS.ProcessEnv, event: unknown, selected: MatrixExecution, launchSha256: string) { + const payload = event as { ref?: unknown; sender?: { id?: unknown }; repository?: { full_name?: unknown; private?: unknown; default_branch?: unknown }; inputs?: Record }; + const branch = payload.repository?.default_branch; + const actor = env.GITHUB_ACTOR_ID, triggering = env.TESTOPS_TRIGGERING_ACTOR_ID; + const allowed = JSON.parse(env.TESTOPS_ALLOWED_ACTOR_IDS ?? "null"); + if (env.PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION !== "v1" || env.GITHUB_ACTIONS !== "true" || env.GITHUB_EVENT_NAME !== "workflow_dispatch" + || env.GITHUB_REPOSITORY !== repository || payload.repository?.full_name !== repository || payload.repository.private !== false + || typeof branch !== "string" || !/^[A-Za-z0-9._/-]+$/.test(branch) || env.GITHUB_REF !== `refs/heads/${branch}` || payload.ref !== env.GITHUB_REF + || env.GITHUB_WORKFLOW_REF !== `${repository}/${workflow}@refs/heads/${branch}` || !["chat", "report"].includes(env.GITHUB_JOB ?? "") + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? "") || env.GITHUB_SHA !== env.TARGET_SHA || env.GITHUB_SHA !== env.TESTOPS_ADMITTED_SHA + || !/^[1-9][0-9]{0,19}$/.test(env.GITHUB_RUN_ID ?? "") || env.GITHUB_RUN_ATTEMPT !== "1" + || !/^[1-9][0-9]{0,19}$/.test(actor ?? "") || !/^[1-9][0-9]{0,19}$/.test(triggering ?? "") || payload.sender?.id !== Number(actor) + || !Array.isArray(allowed) || !allowed.length || allowed.some(v => !Number.isSafeInteger(v) || v <= 0) || !allowed.includes(Number(actor)) || !allowed.includes(Number(triggering)) + || !payload.inputs || Object.keys(payload.inputs).some(k => !["ALLURE_JOB_RUN_ID", "ALLURE_USERNAME", "operation"].includes(k)) || (payload.inputs.operation ?? "evaluate") !== "evaluate" || !/^[1-9][0-9]{0,19}$/.test(String(payload.inputs.ALLURE_JOB_RUN_ID ?? "")) + || !/^[a-f0-9]{64}$/.test(launchSha256)) throw new Error("Invalid shipped chat authority"); + if (JSON.stringify(JSON.parse(env.MANUAL_PRIVATE_IDS ?? "null")) !== JSON.stringify([executionId]) || env.SELECTED_EXECUTION_ID && env.SELECTED_EXECUTION_ID !== executionId + || selected.id !== executionId || selected.suite.id !== "chat-features" || selected.profile.id !== "runner-opencode-deepseek-1017" || selected.environment.id !== "local" + || selected.profile.provider !== "opencode" || selected.profile.model !== "openrouter/deepseek/deepseek-v4.1-flash-1017" || selected.profile.expectedRuntimeMode !== "native" + || selected.requiredCredentials.length !== 1 || selected.requiredCredentials[0] !== "OPENROUTER_API_KEY" || selected.task.id !== "create-backlog" + || selected.task.expectedRunCount !== 1 || selected.task.turnTimeoutMs !== 120_000 || selected.task.attemptTimeoutMs.local !== 420_000 || selected.task.automaticRetryPolicy !== "single_attempt" + || selected.suite.definitionMetadata?.companyAndAgentBudgetCents !== 500 || selected.suite.definitionMetadata?.maximumAttemptsPerCell !== 1) throw new Error("Unexpected shipped chat selection or bounds"); + const identity = { workflowSha: env.GITHUB_SHA!, targetSha: env.GITHUB_SHA!, runId: env.GITHUB_RUN_ID!, attempt: "1", executionIds: Object.freeze([executionId]) }; + const authoritySha256 = createHash("sha256").update(JSON.stringify({ authority, ...identity, actor, triggering, jobRunId: payload.inputs.ALLURE_JOB_RUN_ID, launchSha256 })).digest("hex"); + return Object.freeze({ ...identity, authority, protocol: MANUAL_PRIVATE_PROTOCOL, launchSha256, authoritySha256, approvalSha256: authoritySha256, private: true as const }); +} + +export function shippedChatEvidenceAuthority(env: NodeJS.ProcessEnv) { + const eventPath = env.GITHUB_EVENT_PATH; + if (!eventPath) throw new Error("Missing shipped chat event"); + const stat = lstatSync(eventPath); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 65536) throw new Error("Invalid shipped chat event custody"); + const event = JSON.parse(readFileSync(eventPath, "utf8")); + const selected = runnerMatrix.find(row => row.id === executionId); + if (!selected) throw new Error("Shipped chat execution is absent"); + const launch = readFileSync(new URL("./launch.ts", import.meta.url)); + return authorizeShippedChat(env, event, selected, createHash("sha256").update(launch).digest("hex")); +} diff --git a/tests/runner-e2e/testops-chat-dispatch.ts b/tests/runner-e2e/testops-chat-dispatch.ts new file mode 100644 index 00000000000..29047e2cf6d --- /dev/null +++ b/tests/runner-e2e/testops-chat-dispatch.ts @@ -0,0 +1,31 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync, lstatSync, readFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export function validateChatTestOpsDispatch(env: NodeJS.ProcessEnv, event: unknown, triggeringActorId: number) { + const e = event as { ref?: unknown; sender?: { id?: unknown }; repository?: { full_name?: unknown; private?: unknown; default_branch?: unknown }; inputs?: Record }; + const branch = e.repository?.default_branch, allowed = JSON.parse(env.TESTOPS_ALLOWED_ACTOR_IDS ?? "null"); + const operation = e.inputs?.operation ?? "evaluate"; + if (env.GITHUB_ACTIONS !== "true" || env.GITHUB_EVENT_NAME !== "workflow_dispatch" || env.GITHUB_REPOSITORY !== "paperclipai/paperclip" + || e.repository?.full_name !== "paperclipai/paperclip" || e.repository.private !== false || typeof branch !== "string" || !/^[A-Za-z0-9._/-]+$/.test(branch) + || env.GITHUB_REF !== `refs/heads/${branch}` || e.ref !== env.GITHUB_REF || env.GITHUB_WORKFLOW_REF !== `paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/${branch}` + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? "") || env.GITHUB_RUN_ATTEMPT !== "1" || !/^[1-9][0-9]{0,19}$/.test(env.GITHUB_RUN_ID ?? "") + || !/^[1-9][0-9]{0,19}$/.test(env.GITHUB_ACTOR_ID ?? "") || e.sender?.id !== Number(env.GITHUB_ACTOR_ID) + || !Array.isArray(allowed) || !allowed.length || allowed.some(v => !Number.isSafeInteger(v) || v <= 0) || !allowed.includes(Number(env.GITHUB_ACTOR_ID)) || !allowed.includes(triggeringActorId) + || !e.inputs || Object.keys(e.inputs).some(k => !["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"].includes(k)) || !["bootstrap", "launch-plan", "evaluate"].includes(String(operation)) + || operation === "evaluate" && !/^[1-9][0-9]{0,19}$/.test(String(e.inputs.ALLURE_JOB_RUN_ID ?? "")) + || operation !== "evaluate" && (e.inputs.ALLURE_JOB_RUN_ID ?? "") !== "") throw new Error("Chat TestOps dispatch rejected"); + return { sha: env.GITHUB_SHA!, operation: String(operation), triggeringActorId: String(triggeringActorId) }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + try { + if (!/^[A-Za-z0-9-]{1,39}$/.test(process.env.GITHUB_TRIGGERING_ACTOR ?? "")) throw new Error("Invalid actor"); + const file = process.env.GITHUB_EVENT_PATH!; + const stat = lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size > 65536) throw new Error("Invalid event"); + const actor = Number(execFileSync("gh", ["api", `users/${process.env.GITHUB_TRIGGERING_ACTOR}`, "--jq", ".id"], { encoding: "utf8", timeout: 30_000, stdio: ["ignore", "pipe", "pipe"] }).trim()); + const checked = validateChatTestOpsDispatch(process.env, JSON.parse(readFileSync(file, "utf8")), actor); + appendFileSync(process.env.GITHUB_OUTPUT!, `sha=${checked.sha}\noperation=${checked.operation}\ntriggering_actor_id=${checked.triggeringActorId}\n`); + } catch { process.stderr.write("Fixed chat TestOps dispatch identity rejected before credential access\n"); process.exitCode = 1; } +} diff --git a/tests/runner-e2e/testops-chat-export.test.ts b/tests/runner-e2e/testops-chat-export.test.ts new file mode 100644 index 00000000000..29656c5f719 --- /dev/null +++ b/tests/runner-e2e/testops-chat-export.test.ts @@ -0,0 +1,84 @@ +import { createHash } from "node:crypto"; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, expect, it } from "vitest"; +import { exportTestOpsChat, TESTOPS_CHAT_EXECUTION } from "./testops-chat-export.js"; + +const directories: string[] = []; +afterEach(async () => { await Promise.all(directories.splice(0).map(d => rm(d, { recursive: true, force: true }))); }); +const binding = { runId: "123456", attempt: "1", workflowSha: "a".repeat(40), targetSha: "b".repeat(40) }; +const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/l9sAAAAASUVORK5CYII=", "base64"); +const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +async function fixture() { + const root = await mkdtemp(path.join(os.tmpdir(), "testops-chat-export-")); directories.push(root); + const report = path.join(root, "report"), output = path.join(root, "export"); + const screenshot = `evidence/${TESTOPS_CHAT_EXECUTION}/attempt-1/final-state.png`; + await mkdir(path.dirname(path.join(report, screenshot)), { recursive: true }); + await writeFile(path.join(report, screenshot), png); + const result = { + schema: "paperclip.runner-e2e.result/v2", executionId: TESTOPS_CHAT_EXECUTION, suiteId: "chat-features", suiteDefinitionHash: "c".repeat(64), + source: { sha: binding.targetSha, ref: "refs/heads/master", workflowRunUrl: `https://github.com/paperclipai/paperclip/actions/runs/${binding.runId}` }, + attempt: 1, status: "passed", cleanup: "passed", evidenceValid: true, runIds: ["synthetic-run"], + profileId: "runner-opencode-deepseek-1017", environmentId: "local", caseId: "create-backlog", provider: "opencode", model: "openrouter/deepseek/deepseek-v4.1-flash-1017", runtimeMode: "native", + startedAt: "2026-10-30T00:00:00.000Z", finishedAt: "2026-10-30T00:00:01.000Z", durationMs: 1000, + matcherResults: [{ matcher: { kind: "json_path", path: "chat.saved-backlog", expected: true }, passed: true, detail: "Private diagnostic body" }], + screenshots: [{ id: "final-state", label: "Fixture", file: "final-state.png", publication: "public-runner-fixture", sha256: digest(png) }], + billing: { llm: { runCount: 1, runsWithTokenUsage: 1, runsWithReportedCost: 1, inputTokens: 100, outputTokens: 20, cachedInputTokens: 0, totalTokens: 120, reportedCostUsd: 0.01, costStatus: "reported" }, + runtime: { provider: "local", agentRunDurationMs: 1000, leaseDurationMs: null, leaseCount: 0, costStatus: "not_metered", costSource: "local_not_metered" }, + reportedCostUsd: 0.01, estimatedRuntimeCostUsd: 0, observedAndEstimatedCostUsd: null, complete: false }, + usage: { hiddenReasoning: "Do not export this provider body" }, unknownProviderBody: "Do not export this extra result field", + }; + const campaign = { schema: "paperclip.runner-e2e.campaign/v2", campaignId: `gha-${binding.runId}-1`, expected: [TESTOPS_CHAT_EXECUTION], results: [result], retries: 0, generatedAt: result.finishedAt }; + const save = () => writeFile(path.join(report, "normalized-results.json"), JSON.stringify(campaign)); + await save(); return { report, output, result, campaign, save, screenshot }; +} + +it("exports the actual normalized result, marked PNG and bound hashes without opaque diagnostics", async () => { + const f = await fixture(); await writeFile(path.join(f.report, "unapproved.log"), "private original"); + const receipt = await exportTestOpsChat(f.report, f.output, binding); + const text = await readFile(path.join(f.output, "normalized-results.json"), "utf8"), projected = JSON.parse(text); + expect(projected.results[0]).toMatchObject({ status: "passed", cleanup: "passed", provider: "opencode", model: f.result.model, runtimeMode: "native" }); + expect(projected.billing.llm.inputTokens).toBe(100); expect(projected.billing.testsWithCompleteBilling).toBe(0); + expect(projected.complete).toBe(false); + expect(text).not.toMatch(/Private diagnostic|hiddenReasoning|unknownProviderBody|Do not export/); + expect(receipt.files.map(f => f.path).sort()).toEqual([f.screenshot, "junit.xml", "normalized-results.json"].sort()); + for (const file of receipt.files) expect(digest(await readFile(path.join(f.output, file.path)))).toBe(file.sha256); + expect(JSON.parse(await readFile(path.join(f.output, "binding.json"), "utf8"))).toMatchObject({ ...binding, executionId: TESTOPS_CHAT_EXECUTION }); + expect(await readFile(path.join(f.output, "junit.xml"), "utf8")).toContain('failures="0"'); +}); + +it("preserves a real failed grade and failed cleanup as failed JUnit", async () => { + const f = await fixture(); f.result.status = "failed"; f.result.cleanup = "failed"; await f.save(); + await exportTestOpsChat(f.report, f.output, binding); + expect(JSON.parse(await readFile(path.join(f.output, "normalized-results.json"), "utf8")).results[0]).toMatchObject({ status: "failed", cleanup: "failed" }); + expect(await readFile(path.join(f.output, "junit.xml"), "utf8")).toContain('failures="1"'); +}); + +it.each(["selection", "source", "model", "retry", "evidence", "oracle", "cleanup", "run-count"])("rejects inconsistent %s before exporting a PASS", async kind => { + const f = await fixture(); + if (kind === "selection") f.campaign.expected.push("other"); + if (kind === "source") f.result.source.sha = "d".repeat(40); + if (kind === "model") f.result.model = "other/model"; + if (kind === "retry") f.result.attempt = 2; + if (kind === "evidence") f.result.evidenceValid = false; + if (kind === "oracle") f.result.matcherResults[0]!.passed = false; + if (kind === "cleanup") f.result.cleanup = "failed"; + if (kind === "run-count") f.result.runIds.push("extra-run"); + await f.save(); await expect(exportTestOpsChat(f.report, f.output, binding)).rejects.toThrow(); + await expect(readFile(path.join(f.output, "binding.json"))).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it.each(["unmarked", "hash", "symlink"])("rejects %s screenshot evidence", async kind => { + const f = await fixture(); + if (kind === "unmarked") f.result.screenshots[0]!.publication = "private"; + if (kind === "hash") f.result.screenshots[0]!.sha256 = "d".repeat(64); + if (kind === "symlink") { const target = path.join(f.report, "original.png"); await writeFile(target, png); await rm(path.join(f.report, f.screenshot)); await symlink(target, path.join(f.report, f.screenshot)); } + await f.save(); await expect(exportTestOpsChat(f.report, f.output, binding)).rejects.toThrow(); +}); + +it("rejects a workflow replay and refuses to overwrite an existing export", async () => { + const f = await fixture(); await expect(exportTestOpsChat(f.report, f.output, { ...binding, attempt: "2" })).rejects.toThrow(); + await exportTestOpsChat(f.report, f.output, binding); + await expect(exportTestOpsChat(f.report, f.output, binding)).rejects.toMatchObject({ code: "EEXIST" }); +}); diff --git a/tests/runner-e2e/testops-chat-export.ts b/tests/runner-e2e/testops-chat-export.ts new file mode 100644 index 00000000000..cc67cff2e75 --- /dev/null +++ b/tests/runner-e2e/testops-chat-export.ts @@ -0,0 +1,104 @@ +import { createHash } from "node:crypto"; +import { lstat, mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { buildRunnerCampaign } from "./history.js"; +import { findSecretLeakInJsonValues, assertSecretFree } from "./redaction.js"; +import { validateRetainedRunnerResult } from "./result-validation.js"; +import { PUBLIC_RUNNER_SCREENSHOT_MARKER } from "./screenshot-policy.js"; +import type { RunnerE2EResult } from "./types.js"; + +export const TESTOPS_CHAT_EXECUTION = "chat-features.runner-opencode-deepseek-1017.local.create-backlog"; +const model = "openrouter/deepseek/deepseek-v4.1-flash-1017"; +const hash = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex"); +const sha = /^[a-f0-9]{40}$/; +const pngMagic = Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]); +export interface TestOpsChatBinding { runId: string; attempt: string; workflowSha: string; targetSha: string } + +function publicBilling(value: NonNullable) { + const pick = (source: object, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(source, key)).map(key => [key, (source as Record)[key]])); + const numeric = (source: object, keys: string[]) => { + const selected = pick(source, keys); + if (Object.values(selected).some(v => v !== null && (typeof v !== "number" || !Number.isFinite(v) || v < 0))) throw new Error("Invalid billing scalar"); + return selected; + }; + return { + llm: { ...numeric(value.llm, ["runCount", "runsWithTokenUsage", "runsWithReportedCost", "runsWithEstimatedCost", "runsWithPartialReportedCost", "inputTokens", "outputTokens", "cachedInputTokens", "totalTokens", "reportedCostUsd", "estimatedCostUsd"]), costStatus: value.llm.costStatus }, + runtime: { ...numeric(value.runtime, ["agentRunDurationMs", "leaseDurationMs", "leaseCount", "cpuCores", "memoryGiB", "diskGiB", "estimatedListCostUsd"]), provider: value.runtime.provider, costStatus: value.runtime.costStatus, costSource: value.runtime.costSource }, + ...numeric(value, ["reportedCostUsd", "estimatedRuntimeCostUsd", "observedAndEstimatedCostUsd"]), complete: value.complete, + } as RunnerE2EResult["billing"]; +} + +async function ownedRead(root: string, relative: string, limit: number) { + const segments = relative.split("/"); + if (segments.some(s => !/^[A-Za-z0-9._-]+$/.test(s) || s === "." || s === "..")) throw new Error("Unsafe export source"); + let current = root; + const rootStat = await lstat(current); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) throw new Error("Invalid report root"); + for (const [index, segment] of segments.entries()) { + current = path.join(current, segment); + const st = await lstat(current); + if (st.isSymbolicLink() || (index < segments.length - 1 ? !st.isDirectory() : !st.isFile() || st.nlink !== 1 || st.size > limit)) throw new Error("Invalid export source"); + } + if (!(await realpath(current)).startsWith(`${await realpath(root)}${path.sep}`)) throw new Error("Export source escapes report"); + const before = await lstat(current), bytes = await readFile(current), after = await lstat(current); + if (bytes.length > limit || before.ino !== after.ino || before.dev !== after.dev || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) throw new Error("Export source changed"); + return bytes; +} + +export async function exportTestOpsChat(reportRoot: string, output: string, binding: TestOpsChatBinding) { + if (!/^[1-9][0-9]{0,19}$/.test(binding.runId) || binding.attempt !== "1" || !sha.test(binding.workflowSha) || !sha.test(binding.targetSha)) throw new Error("Invalid TestOps run binding"); + const raw = JSON.parse((await ownedRead(reportRoot, "normalized-results.json", 8 * 1024 * 1024)).toString("utf8")); + const campaignId = `gha-${binding.runId}-${binding.attempt}`; + if (raw.schema !== "paperclip.runner-e2e.campaign/v2" || raw.campaignId !== campaignId || !Array.isArray(raw.expected) || raw.expected.length !== 1 || raw.expected[0] !== TESTOPS_CHAT_EXECUTION || !Array.isArray(raw.results) || raw.results.length !== 1 || raw.retries !== 0) throw new Error("Foreign or repeated TestOps selection"); + const original = raw.results[0]; + validateRetainedRunnerResult(original); + if (original.executionId !== TESTOPS_CHAT_EXECUTION || original.suiteId !== "chat-features" || original.profileId !== "runner-opencode-deepseek-1017" || original.environmentId !== "local" || original.caseId !== "create-backlog" || original.provider !== "opencode" || original.model !== model || original.runtimeMode !== "native" || original.attempt !== 1 || original.source?.sha !== binding.targetSha) throw new Error("Foreign TestOps result"); + if (!original.billing) throw new Error("Missing retained billing coverage"); + const checks = original.matcherResults ?? []; + if (original.status === "passed" && (original.cleanup !== "passed" || raw.results[0].evidenceValid !== true || checks.length === 0 || checks.some(c => !c.passed) || original.runIds?.length !== 1)) throw new Error("Incomplete passing TestOps evidence"); + const screenshots = (original.screenshots ?? []).filter(s => s.publication === PUBLIC_RUNNER_SCREENSHOT_MARKER); + if (screenshots.length > 8 || original.status === "passed" && !screenshots.some(s => s.id === "final-state" && s.file === "final-state.png")) throw new Error("Missing or excessive fixture screenshots"); + const files = new Map(); + for (const shot of screenshots) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,199}\.png$/.test(shot.file) || !/^[a-f0-9]{64}$/.test(shot.sha256 ?? "")) throw new Error("Unbound fixture screenshot"); + const relative = `evidence/${TESTOPS_CHAT_EXECUTION}/attempt-1/${shot.file}`; + const bytes = await ownedRead(reportRoot, relative, 12 * 1024 * 1024); + if (!bytes.subarray(0, 8).equals(pngMagic) || bytes.subarray(12, 16).toString("ascii") !== "IHDR" || hash(bytes) !== shot.sha256 || files.has(relative)) throw new Error("Invalid fixture PNG"); + assertSecretFree(bytes, [], "TestOps fixture PNG", { includeShapes: true }); + files.set(relative, bytes); + } + const result: RunnerE2EResult = { + schema: original.schema, executionId: TESTOPS_CHAT_EXECUTION, suiteId: "chat-features", suiteDefinitionHash: original.suiteDefinitionHash, + source: { sha: binding.targetSha, ref: null, workflowRunUrl: `https://github.com/paperclipai/paperclip/actions/runs/${binding.runId}` }, + attempt: 1, status: original.status, ...(original.failureClass ? { failureClass: original.failureClass } : {}), + ...(original.status === "failed" ? { error: "Retained chat feature evaluation failed; private originals preserve diagnostics" } : {}), + profileId: "runner-opencode-deepseek-1017", environmentId: "local", caseId: "create-backlog", provider: "opencode", model, runtimeMode: "native", + runIds: original.runIds?.map(id => { if (!/^[A-Za-z0-9._-]{1,100}$/.test(id)) throw new Error("Invalid task run identity"); return id; }), + startedAt: original.startedAt, finishedAt: original.finishedAt, durationMs: original.durationMs, cleanup: original.cleanup, + billing: publicBilling(original.billing), + matcherResults: checks.map(check => { + if (check.matcher.kind !== "json_path" || !/^[A-Za-z0-9._-]{1,150}$/.test(check.matcher.path) || check.matcher.expected !== true) throw new Error("Unsupported chat oracle check"); + return { matcher: { kind: "json_path", path: check.matcher.path, expected: true }, passed: check.passed, detail: "Retained independent chat-feature oracle check" }; + }), + screenshots: screenshots.map(s => ({ id: s.id, label: "Reviewed synthetic chat-feature fixture", file: s.file, publication: PUBLIC_RUNNER_SCREENSHOT_MARKER, sha256: s.sha256 })), + }; + validateRetainedRunnerResult(result); + const campaign = buildRunnerCampaign({ campaignId, generatedAt: raw.generatedAt, expected: [TESTOPS_CHAT_EXECUTION], results: [result] }); + if (findSecretLeakInJsonValues(campaign, [], { includeShapes: true })) throw new Error("Unsafe TestOps projection"); + files.set("normalized-results.json", Buffer.from(`${JSON.stringify(campaign, null, 2)}\n`)); + const failed = result.status !== "passed" || result.cleanup !== "passed"; + files.set("junit.xml", Buffer.from(`${failed ? '' : ""}\n`)); + const manifest = [...files].sort(([a], [b]) => a.localeCompare(b)).map(([file, bytes]) => ({ path: file, sha256: hash(bytes), bytes: bytes.length })); + const receipt = { schema: "paperclip.chat-feature-testops-export/v1", ...binding, executionId: TESTOPS_CHAT_EXECUTION, campaignId, resultStatus: result.status, cleanup: result.cleanup, files: manifest }; + await mkdir(output, { mode: 0o700 }); + for (const [relative, bytes] of files) { const target = path.join(output, relative); await mkdir(path.dirname(target), { recursive: true, mode: 0o700 }); await writeFile(target, bytes, { flag: "wx", mode: 0o600 }); } + await writeFile(path.join(output, "binding.json"), `${JSON.stringify(receipt, null, 2)}\n`, { flag: "wx", mode: 0o600 }); + return receipt; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + exportTestOpsChat(path.resolve(process.env.TESTOPS_CHAT_REPORT_ROOT ?? "runner-e2e-merged-report/normalized"), path.resolve(process.env.TESTOPS_CHAT_EXPORT_ROOT ?? "runner-e2e-testops"), { + runId: process.env.GITHUB_RUN_ID ?? "", attempt: process.env.GITHUB_RUN_ATTEMPT ?? "", workflowSha: process.env.GITHUB_SHA ?? "", targetSha: process.env.TESTOPS_CHAT_TARGET_SHA ?? "", + }).catch(() => { process.stderr.write("Chat feature TestOps export rejected; private originals retained\n"); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/testops-chat-lock.test.ts b/tests/runner-e2e/testops-chat-lock.test.ts new file mode 100644 index 00000000000..d60d227de8d --- /dev/null +++ b/tests/runner-e2e/testops-chat-lock.test.ts @@ -0,0 +1,46 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { expect, it } from "vitest"; +import { prepareChatBuildLock } from "./testops-chat-lock.js"; + +const hash = (value: string) => createHash("sha256").update(value).digest("hex"); +async function fixture(source: "input" | "output" | "unknown") { + const root = await mkdtemp(path.join(tmpdir(), "chat-reviewed-lock-")); + const input = "reviewed input\n", output = "reviewed output\n"; + const patch = "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -1 +1 @@\n-reviewed input\n+reviewed output\n"; + await mkdir(path.join(root, "tests/runner-e2e"), { recursive: true }); + await writeFile(path.join(root, "package.json"), "{\"name\":\"synthetic-lock-owner\"}\n"); + await writeFile(path.join(root, "pnpm-lock.yaml"), source === "input" ? input : source === "output" ? output : "unreviewed source\n"); + await writeFile(path.join(root, "tests/runner-e2e/report-lock.patch"), patch); + await writeFile(path.join(root, "tests/runner-e2e/report-lock.json"), JSON.stringify({ inputSha256: hash(input), outputSha256: hash(output), patchSha256: hash(patch) })); + execFileSync("git", ["init", "-q"], { cwd: root }); + execFileSync("git", ["add", "."], { cwd: root }); + execFileSync("git", ["-c", "user.name=Fixture", "-c", "user.email=fixture@example.invalid", "commit", "-qm", "reviewed fixture"], { cwd: root }); + return { root, output }; +} + +it.each(["input", "output"] as const)("uses the existing reviewed %s lock without resolving dependencies", async source => { + const f = await fixture(source); + try { + const receipt = await prepareChatBuildLock(f.root); + expect(receipt.runtimeLockSha256).toBe(hash(f.output)); + expect(receipt.sourceInputsUnchanged).toBe(true); + expect(await readFile(path.join(f.root, "pnpm-lock.yaml"), "utf8")).toBe(f.output); + expect(await prepareChatBuildLock(f.root)).toEqual(receipt); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); + +it.each(["unknown-source", "unknown-runtime", "manifest", "patch", "policy"])("rejects %s drift before any installer or artifact", async variant => { + const f = await fixture(variant === "unknown-source" ? "unknown" : "output"); + try { + if (variant === "unknown-source") await writeFile(path.join(f.root, "pnpm-lock.yaml"), f.output); + if (variant === "unknown-runtime") await writeFile(path.join(f.root, "pnpm-lock.yaml"), "unreviewed runtime\n"); + if (variant === "manifest") await writeFile(path.join(f.root, "package.json"), "{\"name\":\"unreviewed-graph\"}\n"); + if (variant === "patch") await writeFile(path.join(f.root, "tests/runner-e2e/report-lock.patch"), "unreviewed patch\n"); + if (variant === "policy") await writeFile(path.join(f.root, "tests/runner-e2e/report-lock.json"), "{}\n"); + await expect(prepareChatBuildLock(f.root)).rejects.toThrow(); + } finally { await rm(f.root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/testops-chat-lock.ts b/tests/runner-e2e/testops-chat-lock.ts new file mode 100644 index 00000000000..36054a6c158 --- /dev/null +++ b/tests/runner-e2e/testops-chat-lock.ts @@ -0,0 +1,32 @@ +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { prepareReportLock } from "./prepare-report-lock.mjs"; + +const hash = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); +const inputs = ["package.json", ":(glob)**/package.json", "pnpm-workspace.yaml", ".npmrc", ".pnpmfile.cjs", + ":(glob)**/*.patch", "tests/runner-e2e/report-lock.json"]; + +export async function prepareChatBuildLock(root = path.resolve(import.meta.dirname, "../..")) { + try { + execFileSync("git", ["diff", "--quiet", "HEAD", "--", ...inputs], { cwd: root, stdio: "pipe" }); + } catch { throw new Error("Source dependency inputs changed before install or packaging"); } + const committed = execFileSync("git", ["show", "HEAD:pnpm-lock.yaml"], { cwd: root }); + const policyBytes = await readFile(path.join(root, "tests/runner-e2e/report-lock.json")); + const policy = JSON.parse(policyBytes.toString("utf8")); + const sourceLockSha256 = hash(committed); + if (![policy.inputSha256, policy.outputSha256].includes(sourceLockSha256)) + throw new Error("Committed dependency lock is not reviewed"); + await prepareReportLock(root); + const runtimeLockSha256 = hash(await readFile(path.join(root, "pnpm-lock.yaml"))); + if (runtimeLockSha256 !== policy.outputSha256) throw new Error("Runtime dependency lock is not reviewed"); + return { sourceSha: execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(), + sourceLockSha256, runtimeLockSha256, policySha256: hash(policyBytes), sourceInputsUnchanged: true }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + prepareChatBuildLock().then(value => console.log(JSON.stringify(value))) + .catch(() => { process.stderr.write("Chat TestOps dependency graph is not source-reviewed\n"); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/testops-chat-private-path.test.ts b/tests/runner-e2e/testops-chat-private-path.test.ts new file mode 100644 index 00000000000..40dbb219c0c --- /dev/null +++ b/tests/runner-e2e/testops-chat-private-path.test.ts @@ -0,0 +1,36 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { authorizeManualPrivateEvidence, MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL, manualCellKind, manualPrivateSessionPolicy } from "./manual-private-retention.js"; +import { canonicalGeneratedReportAsset, cellTraceAsset, inspectCellReportProducer, privateCellEvidenceOwner } from "./manual-private-report-assets.js"; + +const id = "chat-features.runner-opencode-deepseek-1017.local.create-backlog"; +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +it.each(["cell-read", "cell-write", "report-write"] as const)("keeps the shipped chat %s session scoped to only its existing private archive", mode => { + const policy = manualPrivateSessionPolicy({ bucket: "private-evidence", runId: "123", executionIds: [id], mode }); + const objects = policy.Statement.filter(row => Array.isArray(row.Resource)).flatMap(row => row.Resource); + const filename = mode === "report-write" ? "manual-report" : manualCellKind(id); + const key = `arn:aws:s3:::private-evidence/runner-e2e-private/bounded-v1/campaigns/gha-123-1/${filename}.zip`; + expect(objects).toEqual([key, `${key}.upload-receipt.json`]); + expect(JSON.stringify(policy)).not.toContain("*"); + expect(() => manualPrivateSessionPolicy({ bucket: "private-evidence", runId: "123", executionIds: [id.replace("create-backlog", "delegate-announcement")], mode })).toThrow(); +}); +it("preserves the frozen244 candidate approval registry and refuses shipped authority through that old authorizer", () => { + expect(MANUAL_PRIVATE_EXECUTIONS).toHaveLength(244); + expect(MANUAL_PRIVATE_EXECUTIONS).not.toContain(id); + const identity = { workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [id] }; + expect(() => authorizeManualPrivateEvidence(identity, JSON.stringify({ version: 1, approvals: [{ ...identity, protocol: MANUAL_PRIVATE_PROTOCOL, launchSha256: "c".repeat(64) }] }))).toThrow("Unsupported private manual evidence selection"); +}); +it("binds canonical private trace assets to the exact shipped profile and rejects a rewritten profile or sibling case", async () => { + const root = await realpath(await mkdtemp(path.join(os.tmpdir(), "testops-private-path-"))); roots.push(root); + const context = { root, private: true as const, runId: "123", attempt: "1", executionIds: [id], cellReportProducer: await inspectCellReportProducer() }; + const owner = await privateCellEvidenceOwner(context, id); + const prefix = `gha-123-1-${id}/chat-features/runner-opencode-deepseek-1017/local/create-backlog/attempt-1/html-report/trace/`; + expect(cellTraceAsset(`${prefix}playwright-logo.svg`, owner, "final")).toEqual({ reserved: true, canonical: canonicalGeneratedReportAsset("html/trace/playwright-logo.svg") }); + expect(cellTraceAsset(`${prefix.replace("/runner-opencode-deepseek-1017/", "/runner-opencode/")}playwright-logo.svg`, owner, "final")).toEqual({ reserved: true, canonical: undefined }); + for (const foreign of [id.replace("create-backlog", "delegate-announcement"), id.replace("deepseek-1017", "unknown")]) { + await expect(privateCellEvidenceOwner({ ...context, executionIds: [foreign] }, foreign)).rejects.toThrow("Invalid private cell report owner"); + } +}); diff --git a/tests/runner-e2e/testops-chat-reporter.mjs b/tests/runner-e2e/testops-chat-reporter.mjs new file mode 100644 index 00000000000..75bd096a6d8 --- /dev/null +++ b/tests/runner-e2e/testops-chat-reporter.mjs @@ -0,0 +1,70 @@ +import { createHash } from 'node:crypto'; +import { appendFile, lstat, readFile, realpath, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { publicContext, aliases, executionId, scenario } from './chat-feature-testops-api.mjs'; +import { run } from './chat-feature-testops-cli.mjs'; +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const fail = () => { throw new Error('chat_testops_selection_rejected'); }; + +async function readVerifiedSelection(file, env) { + const expected = publicContext(env); + if (!/^[a-f0-9]{64}$/.test(env.EXPECTED_SELECTION_SHA256 ?? '') || !/^[1-9][0-9]{0,15}$/.test(env.ALLURE_JOB_RUN_ID ?? '')) fail(); + const root = await realpath(env.GITHUB_WORKSPACE ?? process.cwd()), absolute = path.resolve(file); + if (!absolute.startsWith(`${root}${path.sep}`)) fail(); + let current = root; + const parts = path.relative(root, absolute).split(path.sep); + for (const [index, part] of parts.entries()) { + if (!/^[A-Za-z0-9._-]+$/.test(part) || ['.', '..'].includes(part)) fail(); + current = path.join(current, part); + const stat = await lstat(current); + if (stat.isSymbolicLink() || (index < parts.length - 1 ? !stat.isDirectory() : !stat.isFile() || stat.nlink !== 1 || stat.size > 16_384)) fail(); + } + const bytes = await readFile(absolute); + if (hash(bytes) !== env.EXPECTED_SELECTION_SHA256) fail(); + const value = JSON.parse(bytes); + if (value.schema !== 'paperclip.chat-feature-testops-selection/v1' || value.runId !== expected.runId || value.attempt !== '1' + || value.workflowSha !== expected.workflowSha || value.targetSha !== expected.targetSha || value.allureJobRunId !== env.ALLURE_JOB_RUN_ID + || value.executionId !== executionId || value.canonicalScenario !== scenario || value.projectId !== 1 || !aliases.includes(value.fullName) + || ['planId', 'jobId', 'caseId', 'launchId'].some(key => !Number.isSafeInteger(value[key]) || value[key] <= 0)) fail(); + return { value, bytes }; +} + +export async function verifySelectionFile(file, env) { + return (await readVerifiedSelection(file, env)).value; +} + +export async function reporter(command, argument, env = process.env, fetchImpl = fetch) { + publicContext(env); + if (command === 'verify-selection') { + if (Object.hasOwn(env, 'ALLURE_TOKEN')) fail(); + await verifySelectionFile(argument, env); + return 0; + } + if (!['bootstrap', 'launch-plan', 'selection', 'upload', 'verify-upload'].includes(command) || !env.RUNNER_TEMP) fail(); + const stage = command === 'selection' ? 'validate-selection' : command === 'upload' ? 'export' : command; + const stateRoot = path.join(env.RUNNER_TEMP, `chat-testops-${command}-state`); + const selectionPath = path.join(env.GITHUB_WORKSPACE ?? process.cwd(), 'chat-testops-selection', 'testops-selection.json'); + let copiedSelectionPath = selectionPath; + if (command === 'upload' || command === 'verify-upload') { + const { bytes } = await readVerifiedSelection(selectionPath, env); + copiedSelectionPath = path.join(env.RUNNER_TEMP, 'chat-testops-authenticated-selection.json'); + if (command === 'upload') await writeFile(copiedSelectionPath, bytes, { flag: 'wx', mode: 0o600 }); + else if (!(await readFile(copiedSelectionPath)).equals(bytes)) fail(); + } + const result = await run({ ...env, CHAT_TESTOPS_OPERATION: stage, CHAT_TESTOPS_STATE_ROOT: stateRoot, + ...(['upload', 'verify-upload'].includes(command) ? { CHAT_TESTOPS_EXPORT_ROOT: path.resolve(argument), CHAT_TESTOPS_SELECTION: copiedSelectionPath, + ALLURE_RESULTS: env.ALLURE_RESULTS ?? path.join(env.RUNNER_TEMP, 'chat-testops-allure-results') } : {}) }, fetchImpl); + if (command === 'selection' && result === 0) { + const bytes = await readFile(path.join(stateRoot, 'selection.json')); + await writeFile(path.join(env.GITHUB_WORKSPACE ?? process.cwd(), 'testops-selection.json'), bytes, { flag: 'wx', mode: 0o600 }); + if (!env.GITHUB_OUTPUT) fail(); + await appendFile(env.GITHUB_OUTPUT, `sha256=${hash(bytes)}\n`); + } + return result; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + reporter(process.argv[2], process.argv[3]).then(code => { process.exitCode = code; }) + .catch(() => { process.stderr.write('Chat TestOps reporter stopped: invalid source, custody or selection\n'); process.exitCode = 1; }); +} diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs new file mode 100644 index 00000000000..3c99400867e --- /dev/null +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -0,0 +1,138 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtemp, mkdir, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { verifySelectionFile, reporter } from './testops-chat-reporter.mjs'; +import { executionId, scenario, fullName } from './chat-feature-testops-api.mjs'; +const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', ALLURE_JOB_RUN_ID: '9', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; +async function fixture(t, change = {}) { + const root = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-selection-'))); + t.after(() => rm(root, { recursive: true })); + const value = { schema: 'paperclip.chat-feature-testops-selection/v1', runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: 'a'.repeat(40), + allureJobRunId: '9', executionId, canonicalScenario: scenario, projectId: 1, planId: 8, jobId: 7, caseId: 91, fullName, launchId: 10, ...change }; + const bytes = Buffer.from(JSON.stringify(value)); + await mkdir(path.join(root, 'chat-testops-selection')); + const file = path.join(root, 'chat-testops-selection', 'testops-selection.json'); await writeFile(file, bytes); + return { file, env: { ...context, GITHUB_WORKSPACE: root, EXPECTED_SELECTION_SHA256: createHash('sha256').update(bytes).digest('hex') } }; +} + +test('actual worker verifier requires no token or network and accepts exact source-bound selection hash', async t => { + const f = await fixture(t); + assert.equal((await verifySelectionFile(f.file, f.env)).caseId, 91); + assert.equal(await reporter('verify-selection', f.file, f.env), 0); +}); + +test('worker verifier rejects token contamination, hash mismatch and correctly-hashed foreign tuples', async t => { + const f = await fixture(t); + await assert.rejects(reporter('verify-selection', f.file, { ...f.env, ALLURE_TOKEN: 'SYNTHETIC_TOKEN' })); + await assert.rejects(verifySelectionFile(f.file, { ...f.env, EXPECTED_SELECTION_SHA256: '0'.repeat(64) })); + for (const change of [{ runId: '456' }, { targetSha: 'b'.repeat(40) }, { allureJobRunId: '10' }, { executionId: 'other' }, { projectId: 2 }]) { + const foreign = await fixture(t, change); + await assert.rejects(verifySelectionFile(foreign.file, foreign.env)); + } +}); + +test('actual upload facade copies authenticated workspace bytes to owned temp before successful synthetic export', async t => { + const f = await fixture(t), workspace = f.env.GITHUB_WORKSPACE; + const temporary = path.join(workspace, 'temp'); await mkdir(temporary); + const root = path.join(workspace, 'safe-export'); await mkdir(root); + const png = Buffer.from([137,80,78,71,13,10,26,10,0,0,0,13,73,72,68,82]); + const hash = bytes => createHash('sha256').update(bytes).digest('hex'); + const shot = { id: 'final-state', file: 'final-state.png', sha256: hash(png) }; + const result = { executionId, suiteId: 'chat-features', caseId: 'create-backlog', profileId: 'runner-opencode-deepseek-1017', provider: 'opencode', + environmentId: 'local', model: 'openrouter/deepseek/deepseek-v4.1-flash-1017', runtimeMode: 'native', attempt: 1, + source: { sha: context.GITHUB_SHA }, status: 'passed', cleanup: 'passed', runIds: ['synthetic'], durationMs: 123, + matcherResults: [{ matcher: { kind: 'json_path', path: 'chatFeature.createdTask', expected: true }, passed: true }], screenshots: [shot] }; + const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2', campaignId: 'gha-123-1', results: [result] }))], + ['junit.xml', Buffer.from('')], [`evidence/${executionId}/attempt-1/final-state.png`, png]]); + for (const [name, bytes] of files) { await mkdir(path.dirname(path.join(root, name)), { recursive: true }); await writeFile(path.join(root, name), bytes); } + await writeFile(path.join(root, 'binding.json'), JSON.stringify({ schema: 'paperclip.chat-feature-testops-export/v1', runId: '123', attempt: '1', + workflowSha: context.GITHUB_SHA, targetSha: context.GITHUB_SHA, executionId, campaignId: 'gha-123-1', resultStatus: 'passed', cleanup: 'passed', + files: [...files].map(([name, bytes]) => ({ path: name, sha256: hash(bytes), bytes: bytes.length })) })); + assert.equal(await reporter('upload', root, { ...f.env, RUNNER_TEMP: temporary }), 0); + assert.deepEqual(await readFile(path.join(temporary, 'chat-testops-authenticated-selection.json')), await readFile(f.file)); + const exported = JSON.parse(await readFile(path.join(temporary, 'chat-testops-upload-state', 'export.json'))); + const jobRun = { id: 9, job: { id: 7, type: 'github' }, launchId: 10, stage: 'finished', url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }; + const fetchImpl = async (url, options) => { + const parsed = new URL(url), route = parsed.pathname; + let value; + if (route === '/api/uaa/oauth/token') { assert.equal(options.method, 'POST'); value = { access_token: 'MOCK_ACCESS_TOKEN' }; } + else { + assert.equal(options.method, 'GET'); + if (route === '/api/jobrun/9') value = jobRun; + else if (route === '/api/launch/10') value = { id: 10, projectId: 1 }; + else if (route === '/api/testresult') value = { content: [{ id: 45, testCaseId: 91 }], totalElements: 1, totalPages: 1 }; + else if (route === '/api/testresult/45') value = { id: 45, projectId: 1, launchId: 10, testCaseId: 91, fullName, jobRun, status: 'passed', + links: [{ url: jobRun.url }, { url: `https://github.com/paperclipai/paperclip/commit/${context.GITHUB_SHA}` }] }; + else if (route === '/api/testresult/attachment') value = { content: exported.attachmentMetadata.map((row, i) => ({ ...row, id: i + 1 })), totalElements: exported.attachmentMetadata.length, totalPages: 1 }; + else assert.fail(route); + } + return { ok: true, status: 200, json: async () => value, text: async () => JSON.stringify(value) }; + }; + assert.equal(await reporter('verify-upload', root, { ...f.env, RUNNER_TEMP: temporary, ALLURE_TOKEN: 'MOCK_API_TOKEN' }, fetchImpl), 0); + const verified = JSON.parse(await readFile(path.join(temporary, 'chat-testops-verify-upload-state', 'verification.json'))); + assert.equal(verified.verification, 'VERIFIED'); assert.equal(verified.caseId, 91); assert.equal(verified.status, 'passed'); + +}); + +test('actual launch-plan facade performs metadata-only plan launch once and exits without waiting for child', async t => { + const f = await fixture(t), workspace = f.env.GITHUB_WORKSPACE; + const temporary = path.join(workspace, 'temp'); await mkdir(temporary); + const eventPath = path.join(workspace, 'event.json'); await writeFile(eventPath, JSON.stringify({ inputs: { operation: 'launch-plan', ALLURE_JOB_RUN_ID: '' } })); + const nonce = '1'.repeat(32), calls = []; + const fetchImpl = async (url, options) => { + const route = new URL(url).pathname, method = options.method; calls.push({ route, method }); + const response = value => ({ ok: true, status: 200, json: async () => value, text: async () => JSON.stringify(value) }); + if (route === '/repos/paperclipai/paperclip') return response({ full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }); + if (route === '/repos/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml') return response({ id: 99, path: '.github/workflows/chat-feature-testops.yml', state: 'active' }); + if (route === '/repos/paperclipai/paperclip/actions/runs/123') return response({ id: 123, workflow_id: 99, run_attempt: 1, run_number: 2, head_sha: context.GITHUB_SHA, head_branch: 'master', event: 'workflow_dispatch', repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, actor: { id: 34892728 }, triggering_actor: { id: 34892728 } }); + if (route === '/api/uaa/oauth/token') return response({ access_token: 'SYNTHETIC_TOKEN' }); + if (route === '/api/testcase/91') return response({ id: 91, projectId: 1, fullName, automated: true }); + if (route === '/api/testplan/8') return response({ id: 8, projectId: 1, name: 'Chat feature evals', testCasesCount: 1, treeSelection: { leafsInclude: [91] } }); + if (route === '/api/job/7') return response({ id: 7, type: 'github', url: 'https://github.com/paperclipai/paperclip/actions/workflows/chat-feature-testops.yml', parameters: [{ id: 17, value: 'evaluate' }] }); + if (route === '/api/integration/2/project/1') return response({ id: 2, projectId: 1, info: { type: 'github' }, disabled: false }); + if (route === '/api/environment/17') return response({ id: 17, name: 'operation' }); + if (route === '/api/testplan/8/job') return response([{ job: { id: 7, parameters: [{ id: 17, value: 'evaluate' }] }, testCasesCount: 1 }]); + if (route === '/api/launch') return response({ content: [], totalPages: 0 }); + if (route === '/api/testplan/8/jobparameter') return { ok: true, status: 204 }; + assert.equal(route, '/api/testplan/8/run'); + return response({ id: 20, projectId: 1, name: `Chat feature evals · ${nonce}` }); + }; + assert.equal(await reporter('launch-plan', undefined, { ...f.env, RUNNER_TEMP: temporary, GITHUB_EVENT_PATH: eventPath, + ALLURE_JOB_RUN_ID: '', ALLURE_TOKEN: 'SYNTHETIC_ONLY', CHAT_TESTOPS_CASE_ID: '91', CHAT_TESTOPS_PLAN_ID: '8', CHAT_TESTOPS_JOB_ID: '7', + CHAT_TESTOPS_OPERATION_PARAMETER_ID: '17', CHAT_TESTOPS_INTEGRATION_ID: '2', CHAT_TESTOPS_LAUNCH_NONCE: nonce, CHAT_TESTOPS_LAUNCH_RUN_NUMBER: '2', GITHUB_RUN_NUMBER: '2', + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: '99', GITHUB_ACTOR_ID: '34892728', TESTOPS_TRIGGERING_ACTOR_ID: '34892728', TESTOPS_ALLOWED_ACTOR_IDS: '[34892728]', GH_TOKEN: 'SYNTHETIC_GH_TOKEN' }, fetchImpl), 0); + const state = path.join(temporary, 'chat-testops-launch-plan-state'); + assert.equal(JSON.parse(await readFile(path.join(state, 'launch.json'))).status, 'REQUESTED'); + assert.equal(calls.filter(row => row.route === '/api/testplan/8/run').length, 1); + assert.equal(calls.at(-1).route, '/api/testplan/8/run'); +}); + +test('actual launch caller rejects five own-run contradictions before TestOps auth or mutation', async t => { + const original = process.stderr.write; process.stderr.write = () => true; + try { + for (const changes of [{ run_number: 3 }, { actor: { id: 1 } }, { head_sha: 'b'.repeat(40) }, { workflow_id: 100 }, { run_attempt: 2 }]) { + const f = await fixture(t), workspace = f.env.GITHUB_WORKSPACE; + const temporary = path.join(workspace, 'temp'); await mkdir(temporary); + const eventPath = path.join(workspace, 'event.json'); await writeFile(eventPath, JSON.stringify({ inputs: { operation: 'launch-plan' } })); + let testopsCalls = 0; + const fetchImpl = async url => { + if (new URL(url).hostname !== 'api.github.com') { testopsCalls++; throw new Error('must not reach TestOps'); } + const value = url.endsWith('/repos/paperclipai/paperclip') ? { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' } : url.includes('/actions/workflows/') ? { id: 99, path: '.github/workflows/chat-feature-testops.yml', state: 'active' } + : { id: 123, workflow_id: 99, run_attempt: 1, run_number: 2, head_sha: context.GITHUB_SHA, head_branch: 'master', event: 'workflow_dispatch', + repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, actor: { id: 34892728 }, triggering_actor: { id: 34892728 }, ...changes }; + return { ok: true, text: async () => JSON.stringify(value) }; + }; + assert.equal(await reporter('launch-plan', undefined, { ...f.env, RUNNER_TEMP: temporary, GITHUB_EVENT_PATH: eventPath, + ALLURE_JOB_RUN_ID: '', ALLURE_TOKEN: 'SYNTHETIC_ONLY', CHAT_TESTOPS_LAUNCH_RUN_NUMBER: '2', GITHUB_RUN_NUMBER: '2', + CHAT_TESTOPS_GITHUB_WORKFLOW_ID: '99', GITHUB_ACTOR_ID: '34892728', TESTOPS_TRIGGERING_ACTOR_ID: '34892728', TESTOPS_ALLOWED_ACTOR_IDS: '[34892728]', GH_TOKEN: 'SYNTHETIC_GH_TOKEN' }, fetchImpl), 1); + assert.equal(testopsCalls, 0); + const error = JSON.parse(await readFile(path.join(temporary, 'chat-testops-launch-plan-state', 'failure.json'))); + assert.equal(error.stage, 'launch_admission'); assert.equal(error.code, 'chat_testops_launch_admission'); + } + } finally { process.stderr.write = original; } +}); diff --git a/tests/runner-e2e/testops-chat-workflow.test.ts b/tests/runner-e2e/testops-chat-workflow.test.ts new file mode 100644 index 00000000000..14484cbd30e --- /dev/null +++ b/tests/runner-e2e/testops-chat-workflow.test.ts @@ -0,0 +1,154 @@ +import { readFileSync } from "node:fs"; +import { parse } from "yaml"; +import { expect, it } from "vitest"; +import { validateChatTestOpsDispatch } from "./testops-chat-dispatch.js"; + +const workflow = parse(readFileSync(new URL("../../.github/workflows/chat-feature-testops.yml", import.meta.url), "utf8")); +function context() { + return { env: { GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", + GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", GITHUB_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", GITHUB_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]" }, + event: { repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, ref: "refs/heads/master", sender: { id: 123 }, inputs: { ALLURE_JOB_RUN_ID: "456" } as Record } }; +} +it("admits the actual TestOps service input and only the closed metadata bootstrap alternative", () => { + const f = context(); expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); + f.event.inputs = { operation: "bootstrap" }; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("bootstrap"); + f.event.inputs.operation = "launch-plan"; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("launch-plan"); + f.event.inputs.ALLURE_JOB_RUN_ID = "456"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); + delete f.event.inputs.ALLURE_JOB_RUN_ID; + f.event.inputs.operation = "other"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); +}); +it.each(["unlisted-trigger", "wrong-source", "replay", "extra-input", "missing-jobrun", "foreign-repo"])("rejects %s before any service credential operation", kind => { + const f = context(); let trigger = 123; + if (kind === "unlisted-trigger") trigger = 456; + if (kind === "wrong-source") f.env.GITHUB_REF = "refs/heads/unreviewed"; + if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; + if (kind === "extra-input") f.event.inputs.target_branch = "other"; + if (kind === "missing-jobrun") f.event.inputs = {}; + if (kind === "foreign-repo") f.event.repository.full_name = "foreign/repo"; + expect(() => validateChatTestOpsDispatch(f.env, f.event, trigger)).toThrow(); +}); +it("keeps provider and TestOps service credentials in separate jobs", () => { + const jobs = workflow.jobs; + for (const [id, job] of Object.entries(jobs) as Array<[string, any]>) { + const text = JSON.stringify(job); + if (text.includes("secrets.OPENROUTER_API_KEY")) expect(id).toBe("chat"); + if (text.includes("secrets.ALLURE_TOKEN")) expect(["select", "upload"]).toContain(id); + if (id !== "chat") expect(text).not.toContain("pnpm test:e2e:runner --"); + expect(job.steps.some((step: any) => step.run === "node tests/runner-e2e/testops-chat-dispatch.ts")).toBe(true); + } + expect(jobs.chat.environment).toBe("runner-e2e-private-paid"); + expect(jobs.select.environment).toBe("runner-e2e-testops-reporting"); + expect(jobs.upload.environment).toBe("runner-e2e-testops-reporting"); +}); +it("requires token-isolated selection before the fixed AWS run and keeps bootstrap outside all model jobs", () => { + expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); + expect(Object.keys(workflow.on.workflow_dispatch.inputs)).toEqual(["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"]); + expect(workflow.on.workflow_dispatch.inputs.operation.default).toBe("evaluate"); + expect(workflow.jobs.chat.needs).toContain("select"); expect(workflow.jobs.build.needs).toContain("select"); + expect(workflow.jobs.chat.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); + expect(workflow.jobs.build.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); + const paid = workflow.jobs.chat.steps.find((s: any) => s.env?.OPENROUTER_API_KEY); + expect(paid.run).toContain("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0"); + expect(workflow.jobs.chat.steps.findIndex((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection")).toBeLessThan(workflow.jobs.chat.steps.indexOf(paid)); + expect(workflow.jobs.authorize.outputs.runner).toBe("runs-on/fleet=paperclip-public-pr-x64/env=public-ci"); + expect(workflow.jobs.chat["runs-on"]).toBe("${{ needs.authorize.outputs.runner }}"); +}); +it("uploads only the trusted safe projection after successful strict private report retention", () => { + const steps = workflow.jobs.report.steps; + expect(steps.find((s: any) => s.id === "export").if).toBe("always() && steps.private_report.outcome == 'success'"); + expect(steps.find((s: any) => s.with?.name?.startsWith("runner-e2e-testops-")).with.path).toBe("runner-e2e-testops/"); + expect(JSON.stringify(workflow)).not.toContain("MANUAL_PRIVATE_APPROVALS_JSON"); + expect(Object.values(workflow.jobs).flatMap((j: any) => j.steps).filter((s: any) => s.uses?.startsWith("actions/upload-artifact")).some((s: any) => s.with.path === "tests/runner-e2e/results/")).toBe(false); +}); + +it("initializes the actual TestOps association and never fabricates setup results", () => { + const steps = workflow.jobs.select.steps; + const installer = steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")); + expect(installer.uses).toBe("allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db"); + expect(installer.with["allurectl-version"]).toBe("2.23.1"); + const bootstrap = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs bootstrap")); + expect(bootstrap.run).toContain("mkdir -m 700"); + expect(bootstrap.run).toContain("allurectl watch -- node"); + expect(bootstrap.env.ALLURE_RESULTS).toBe("${{ runner.temp }}/chat-testops-bootstrap-results"); + const select = steps.find((s: any) => s.id === "selection"); + expect(select.run).toContain("allurectl watch -- node"); + for (const name of ["CHAT_TESTOPS_CASE_ID", "CHAT_TESTOPS_PLAN_ID", "CHAT_TESTOPS_JOB_ID"]) + expect(select.env[name]).toBe(`\${{ vars.${name} }}`); + const metadata = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-setup-")); + expect(metadata.if).toBe("always() && needs.authorize.outputs.operation == 'bootstrap'"); + expect(metadata.with.path).not.toContain("results"); +}); +it("authenticates the same-run selection again before converting and uploading", () => { + const steps = workflow.jobs.upload.steps; + const receipt = steps.find((s: any) => s.with?.path === "chat-testops-selection"); + expect(receipt.with["artifact-ids"]).toBe("${{ needs.select.outputs.artifact_id }}"); + const upload = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs upload")); + expect(upload.run).toContain("allurectl watch -- node"); + expect(upload.env.EXPECTED_SELECTION_SHA256).toBe("${{ needs.select.outputs.sha256 }}"); + expect(steps.indexOf(receipt)).toBeLessThan(steps.indexOf(upload)); + expect(JSON.stringify(workflow.jobs.select)).not.toContain("OPENROUTER_API_KEY"); + expect(JSON.stringify(workflow.jobs.upload)).not.toContain("OPENROUTER_API_KEY"); +}); + +it("keeps the one plan launch outside Allure watch and every AWS/model job", () => { + const steps = workflow.jobs.select.steps; + const launch = steps.find((s: any) => s.run?.endsWith("testops-chat-reporter.mjs launch-plan")); + expect(launch.if).toBe("needs.authorize.outputs.operation == 'launch-plan'"); + expect(launch.run).not.toContain("watch"); + expect(launch.env.CHAT_TESTOPS_OPERATION_PARAMETER_ID).toBe("${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}"); + expect(launch.env.CHAT_TESTOPS_INTEGRATION_ID).toBe("${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}"); + expect(launch.env.CHAT_TESTOPS_LAUNCH_NONCE).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}"); + expect(launch.env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}"); + expect(launch.env.TESTOPS_TRIGGERING_ACTOR_ID).toBe("${{ needs.authorize.outputs.triggering_actor_id }}"); + expect(launch.env.TESTOPS_ALLOWED_ACTOR_IDS).toBe("${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); + expect(launch.env.GH_TOKEN).toBe("${{ github.token }}"); + expect(launch.run).not.toContain("sleep"); + expect(steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")).if).toBe("needs.authorize.outputs.operation != 'launch-plan'"); + for (const name of ["build", "chat"]) expect(workflow.jobs[name].if).toBe("needs.authorize.outputs.operation == 'evaluate'"); + expect(workflow.concurrency["cancel-in-progress"]).toBe(false); +}); + +it("accepts the documented TestOps username metadata without treating it as an actor", () => { + const f = context(); f.event.inputs.ALLURE_USERNAME = "service-metadata"; + expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); + expect(() => validateChatTestOpsDispatch(f.env, f.event, 456)).toThrow(); + expect(JSON.stringify(workflow.jobs)).not.toContain("inputs.ALLURE_USERNAME"); +}); + +it("includes every native Node helper control in the normal unit owner", () => { + const launcher = readFileSync(new URL("./run-unit.mjs", import.meta.url), "utf8"); + const config = readFileSync(new URL("./vitest.config.ts", import.meta.url), "utf8"); + for (const file of ["chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"]) { + expect(launcher).toContain(`"tests/runner-e2e/${file}"`); + expect(config).toContain(`"${file}"`); + } +}); + +it("verifies the actual server outcome after watch without masking an original failed upload", () => { + const steps = workflow.jobs.upload.steps; + const watch = steps.find((s: any) => s.id === "allure_upload"); + const verify = steps.find((s: any) => s.id === "verify_upload"); + expect(watch.run).toContain("allurectl watch -- node"); + expect(watch["continue-on-error"]).toBeUndefined(); + expect(verify.run).toBe("node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops"); + expect(verify.if).toBe("always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')"); + expect(verify.env).toEqual(watch.env); + expect(steps.indexOf(verify)).toBeGreaterThan(steps.indexOf(watch)); + const proof = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-verification-")); + expect(proof.with.path.split("\n").filter(Boolean)).toHaveLength(2); + expect(proof.with.path).not.toContain("allure-results"); +}); + +it("carries only the existing source-reviewed lock through build and worker install", () => { + const build = workflow.jobs.build.steps; + const install = build.find((s: any) => s.run?.includes("pnpm install --frozen-lockfile")); + expect(install.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); + expect(install.run).not.toContain("--lockfile-only"); + const archive = build.find((s: any) => s.run?.includes("tar -czf chat-testops-build")); + expect(archive.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); + expect(archive.run).toContain("chat-testops-lock.json pnpm-lock.yaml"); + const worker = workflow.jobs.chat.steps.find((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection"); + expect(worker.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json"); + expect(worker.run).toContain("cmp chat-testops-lock.json chat-testops-worker-lock.json"); + expect(worker.run.indexOf("cmp chat-testops-lock.json")).toBeLessThan(worker.run.indexOf("pnpm install --frozen-lockfile")); +}); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 448f2df9235..831188aad0a 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -6,7 +6,7 @@ export default defineConfig({ name: "runner-e2e-support", include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs"], + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"], testTimeout: 30_000, }, }); diff --git a/ui/src/api/activity.ts b/ui/src/api/activity.ts index c5dc760a577..31b1058ef4c 100644 --- a/ui/src/api/activity.ts +++ b/ui/src/api/activity.ts @@ -92,5 +92,11 @@ export const activityApi = { forIssue: (issueId: string) => api.get(`/issues/${issueId}/activity`), recentRunsForIssue: (issueId: string) => api.get(`/issues/${issueId}/runs?view=recent`), runsForIssue: (issueId: string) => api.get(`/issues/${issueId}/runs`), + runsByIdsForIssue: (issueId: string, runIds: string[]) => { + const ids = [...new Set(runIds)]; + return Promise.all(Array.from({ length: Math.ceil(ids.length / 100) }, (_, index) => + api.get(`/issues/${issueId}/runs?runIds=${ids.slice(index * 100, (index + 1) * 100).join(",")}`), + )).then(pages => pages.flat()); + }, issuesForRun: (runId: string) => api.get(`/heartbeat-runs/${runId}/issues`), }; diff --git a/ui/src/components/ProjectWorkspaceSummaryCard.test.tsx b/ui/src/components/ProjectWorkspaceSummaryCard.test.tsx index d92fc743ebc..470fe63043b 100644 --- a/ui/src/components/ProjectWorkspaceSummaryCard.test.tsx +++ b/ui/src/components/ProjectWorkspaceSummaryCard.test.tsx @@ -1,6 +1,7 @@ // @vitest-environment jsdom import type { ComponentProps, ReactNode } from "react"; +import { act as reactAct } from "react"; import { flushSync } from "react-dom"; import { createRoot } from "react-dom/client"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; @@ -346,55 +347,49 @@ describe("ProjectWorkspaceSummaryCard", () => { cwd: "/Users/dotta/paperclip/.worktrees/PAP-1552-workspace-polish", }); - await act(async () => { - root.render(withQueryClient( - {}} - onCloseWorkspace={() => {}} - />, - )); - }); + vi.useFakeTimers(); + try { + await reactAct(async () => { + root.render(withQueryClient( + {}} + onCloseWorkspace={() => {}} + />, + )); + }); - const branchTextButton = Array.from(container.querySelectorAll("button")) - .find((button) => button.textContent === summary.branchName); - const pathTextButton = container.querySelector(`button[title="${summary.cwd}"]`); - const branchIconButton = container.querySelector('button[aria-label="Copy branch"]'); - const pathIconButton = container.querySelector('button[aria-label="Copy path"]'); + const branchTextButton = Array.from(container.querySelectorAll("button")) + .find((button) => button.textContent === summary.branchName); + const pathTextButton = container.querySelector(`button[title="${summary.cwd}"]`); + const branchIconButton = container.querySelector('button[aria-label="Copy branch"]'); + const pathIconButton = container.querySelector('button[aria-label="Copy path"]'); - expect(branchTextButton).not.toBeNull(); - expect(pathTextButton).not.toBeNull(); - expect(branchIconButton).not.toBeNull(); - expect(pathIconButton).not.toBeNull(); + const copyWithFeedback = async (button: Element | null | undefined, text: string) => { + expect(button).toBeTruthy(); + await reactAct(async () => { + button!.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + expect(writeClipboard).toHaveBeenLastCalledWith(text); + expect(button!.nextElementSibling?.className).toContain("opacity-100"); + await reactAct(async () => { await vi.advanceTimersByTimeAsync(1499); }); + expect(button!.nextElementSibling?.className).toContain("opacity-100"); + await reactAct(async () => { await vi.advanceTimersByTimeAsync(1); }); + expect(button!.nextElementSibling?.className).toContain("opacity-0"); + }; - await act(async () => { - branchTextButton!.click(); - await new Promise((resolve) => setTimeout(resolve, 0)); - }); - expect(writeClipboard).toHaveBeenLastCalledWith(summary.branchName); - expect(branchTextButton?.nextElementSibling?.className).toContain("opacity-100"); - - await act(async () => { - pathTextButton!.dispatchEvent(new MouseEvent("click", { bubbles: true })); - await new Promise((resolve) => setTimeout(resolve, 0)); - }); - expect(writeClipboard).toHaveBeenLastCalledWith(summary.cwd); - expect(pathTextButton?.nextElementSibling?.className).toContain("opacity-100"); - - await act(async () => { - branchIconButton!.dispatchEvent(new MouseEvent("click", { bubbles: true })); - pathIconButton!.dispatchEvent(new MouseEvent("click", { bubbles: true })); - await new Promise((resolve) => setTimeout(resolve, 0)); - }); - expect(writeClipboard).toHaveBeenCalledWith(summary.branchName); - expect(writeClipboard).toHaveBeenCalledWith(summary.cwd); - - act(() => { - root.unmount(); - }); + await copyWithFeedback(branchTextButton, summary.branchName!); + await copyWithFeedback(pathTextButton, summary.cwd!); + await copyWithFeedback(branchIconButton, summary.branchName!); + await copyWithFeedback(pathIconButton, summary.cwd!); + } finally { + try { + await reactAct(async () => { root.unmount(); }); + } finally { vi.useRealTimers(); } + } }); it("colors live service urls green", () => { const root = createRoot(container); diff --git a/ui/src/components/TaskChatThread.test.tsx b/ui/src/components/TaskChatThread.test.tsx index 4990c447a1d..c834c8df0a1 100644 --- a/ui/src/components/TaskChatThread.test.tsx +++ b/ui/src/components/TaskChatThread.test.tsx @@ -20,7 +20,7 @@ import type { import { heartbeatsApi } from "@/api/heartbeats"; import { nativeRunEventsToTranscript } from "./transcript/native-run-events"; import { pendingConnectionIntentInteraction } from "@/fixtures/issueThreadInteractionFixtures"; -import { TaskChatScrollNavigation } from "./task-chat/scroll-navigation"; +import { TaskChatScrollNavigation, useTaskChatScrollNavigation } from "./task-chat/scroll-navigation"; import type { HeartbeatRunEvent } from "@paperclipai/shared"; const transcriptState = vi.hoisted(() => ({ @@ -390,6 +390,16 @@ it.each([true, false])("coordinates compact cards before first reveal and retain expect(container.querySelector('[data-testid="mock-editor"]')).toBe(editor); }); +it("keeps revealed history scrollable while metadata for another page is pending", async () => { + function Readiness() { return {String(useTaskChatScrollNavigation().ready)}; } + const props = { issueId: "paged-chat", compactHistory: true, comments: createLongThreadComments(), onAdd: async () => {}, threadHeader: }; + render(); + await settleThreadLayout(); + expect(container.querySelector('[data-testid="history-scroll-ready"]')?.textContent).toBe("true"); + render(); + expect(container.querySelector('[data-testid="history-scroll-ready"]')?.textContent).toBe("true"); +}); + it.each([0, 32_000, 100_000])("offers earlier output only when a single legacy run exceeds the compact window (%s bytes)", (logBytes) => { const onExpandHistory = vi.fn(); if (logBytes > 32_000) transcriptState.truncatedRunIds.add("legacy-run"); diff --git a/ui/src/components/TaskChatThread.tsx b/ui/src/components/TaskChatThread.tsx index a2d7bc49cf7..e68b2fbcaa7 100644 --- a/ui/src/components/TaskChatThread.tsx +++ b/ui/src/components/TaskChatThread.tsx @@ -1,3 +1,4 @@ +import type { ScrollingHistory } from "@/components/task-chat/TaskMessageScroller"; import { TaskChatShimmerScope } from "@/components/task-chat/TaskChatShimmer"; import { useRunnerSubmission } from "@/hooks/useRunnerSubmission"; import { @@ -429,6 +430,7 @@ export type TaskChatThreadProps = ComponentProps & { conversationMode?: boolean; creationActivity?: ActivityEvent[]; compactHistory?: boolean; + scrollingHistory?: ScrollingHistory; historyControl?: ReactNode; onExpandHistory?: () => void; initialHistoryPending?: boolean; @@ -619,6 +621,18 @@ export function TaskChatThread(props: TaskChatThreadProps) { const decisionReceipts = useCoordinatorChatDecisionReceipts(issueId, conversationMode); const pendingQuestions = useCoordinatorChatPendingQuestions(issueId, conversationMode); const taskHandoffs = useCoordinatorChatTaskHandoffs(issueId, conversationMode === true); + const scrollingHistory: ScrollingHistory | undefined = props.scrollingHistory ? { + hasMore: props.scrollingHistory.hasMore || !!decisionReceipts.hasNextPage || !!taskHandoffs.hasNextPage, + loading: props.scrollingHistory.loading || decisionReceipts.isFetchingNextPage || taskHandoffs.isFetchingNextPage, + error: props.scrollingHistory.error, + load: () => Promise.all([ + props.scrollingHistory?.hasMore ? props.scrollingHistory.load() : undefined, + !decisionReceipts.isError && decisionReceipts.hasNextPage ? decisionReceipts.fetchNextPage() : undefined, + !taskHandoffs.isError && taskHandoffs.hasNextPage ? taskHandoffs.fetchNextPage() : undefined, + ]), + } : undefined; + const conversationWindowStart = props.scrollingHistory?.hasMore && comments.length + ? Math.min(...comments.map(comment => toMs(comment.createdAt))) : -Infinity; const liveHandoffIds = useLiveChatHandoffIds(`${currentUserId}:${issueId}`, taskHandoffs.handoffs, taskHandoffs.isSuccess); const createdProjectItems = useProjectCreatedItems(props.creationActivity ?? [], companyId); const createdSkillItems = useMemo(() => skillCreatedItems(props.creationActivity ?? []), [props.creationActivity]); @@ -1545,10 +1559,12 @@ export function TaskChatThread(props: TaskChatThreadProps) { entriesWithFailures.push({ id, ms: toMs(question.createdAt), order: 4, item: { id, kind: "pending_question", question } }); } for (const receipt of decisionReceipts.receipts) { + if (toMs(receipt.resolvedAt) < conversationWindowStart) continue; entriesWithFailures.push({ id: `decision-receipt:${receipt.id}`, ms: toMs(receipt.resolvedAt), order: 4, item: { id: `decision-receipt:${receipt.id}`, kind: "decision_receipt", receipt } }); } for (const handoff of standaloneChatHandoffs(taskHandoffs.handoffs, interactions ?? [])) { + if (toMs(handoff.createdAt) < conversationWindowStart) continue; entriesWithFailures.push({ id: handoff.id, ms: toMs(handoff.createdAt), order: 4, item: { id: handoff.id, kind: "task_handoff", handoff, animateEntry: liveHandoffIds.has(handoff.id) } }); } @@ -2237,6 +2253,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { }, [ orderedEntries, coordinatorEnabled, + conversationWindowStart, decisionReceipts.receipts, pendingQuestions.questions, taskHandoffs.handoffs, @@ -3045,7 +3062,12 @@ export function TaskChatThread(props: TaskChatThreadProps) { // latest viewport substantially. Compact opens hydrate only their bounded // recent window, but its saved cards and latest turn must arrive together so // the first readable frame already has its final structure. - const historyPending = initialHistoryPending || planLoading || transcriptHistoryPending; + const chatMetadataPending = Boolean(issueId && conversationMode && ( + (pendingQuestions.enabled && pendingQuestions.isPending) || + (decisionReceipts.enabled && decisionReceipts.isPending) || + (taskHandoffs.enabled && taskHandoffs.isPending) + )); + const historyPending = initialHistoryPending || planLoading || transcriptHistoryPending || chatMetadataPending; const scrollNavigation = useTaskChatScrollNavigation(); useEffect(() => { recordIssueDetailReadiness({ plan: !planLoading, transcripts: !transcriptHistoryPending, history: !historyPending }); @@ -3110,7 +3132,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { return ( - + @@ -3166,7 +3188,7 @@ export function TaskChatThread(props: TaskChatThreadProps) { > {prepareInitialContent ? ( <> - {items.length === 0 && !tailRunId && !optimisticRunnerStartup ? ( + {items.length === 0 && !tailRunId && !optimisticRunnerStartup && !scrollingHistory?.hasMore ? (
@@ -3180,8 +3202,8 @@ export function TaskChatThread(props: TaskChatThreadProps) { data-testid="task-chat-thread-header" > {threadHeaderWithBlockers} - - + +
) : null}
@@ -3201,8 +3223,9 @@ export function TaskChatThread(props: TaskChatThreadProps) { ) : ( {threadHeaderWithBlockers}{!issueBrief ? historyControl : null}} + header={<>{threadHeaderWithBlockers}{!issueBrief ? historyControl : null}} renderInteraction={renderInteraction} renderBrief={renderBrief} renderMessageActions={renderMessageActions} diff --git a/ui/src/components/coordinator/CoordinatorChatDecisionReceipts.tsx b/ui/src/components/coordinator/CoordinatorChatDecisionReceipts.tsx index e9ceb493bc2..49467cf21be 100644 --- a/ui/src/components/coordinator/CoordinatorChatDecisionReceipts.tsx +++ b/ui/src/components/coordinator/CoordinatorChatDecisionReceipts.tsx @@ -48,12 +48,12 @@ export function CoordinatorChatDecisionReceipt({ receipt }: { receipt: DecisionH ; } -export function CoordinatorChatReceiptStatus({ query }: { query: ReturnType }) { +export function CoordinatorChatReceiptStatus({ query, scroll = false }: { scroll?: boolean; query: ReturnType }) { if (!query.enabled) return null; if (query.isError) return
Decision updates couldn’t load.
; - if (query.hasNextPage) return ; return null; diff --git a/ui/src/components/coordinator/CoordinatorChatTaskHandoffs.tsx b/ui/src/components/coordinator/CoordinatorChatTaskHandoffs.tsx index 1ed10c89cc4..6428b0bc53c 100644 --- a/ui/src/components/coordinator/CoordinatorChatTaskHandoffs.tsx +++ b/ui/src/components/coordinator/CoordinatorChatTaskHandoffs.tsx @@ -61,12 +61,12 @@ export function CoordinatorChatTaskHandoff({ handoff, animateEntry = false }: { ; } -export function CoordinatorChatHandoffStatus({ query }: { query: ReturnType }) { +export function CoordinatorChatHandoffStatus({ query, scroll = false }: { scroll?: boolean; query: ReturnType }) { if (!query.enabled) return null; if (query.isError) return
Created tasks couldn’t load.
; - if (query.hasNextPage) return ; return null; diff --git a/ui/src/components/task-chat/TaskChatThreadView.tsx b/ui/src/components/task-chat/TaskChatThreadView.tsx index ea2b0a24d83..7c95a95f071 100644 --- a/ui/src/components/task-chat/TaskChatThreadView.tsx +++ b/ui/src/components/task-chat/TaskChatThreadView.tsx @@ -21,7 +21,7 @@ import { TaskChatUsageReadout } from "./TaskChatUsageReadout"; import { TaskChatRunnerActivityGroup } from "./TaskChatRunnerActivityGroup"; import { TaskChatActivityPhase } from "./TaskChatActivityPhase"; import { TaskChatThinking } from "./TaskChatThinking"; -import { TaskMessageScroller } from "./TaskMessageScroller"; +import { TaskMessageScroller, type ScrollingHistory } from "./TaskMessageScroller"; import { TaskChatProtocolCard } from "./TaskChatProtocolCard"; import { TaskChatProtocolActivityRow } from "./TaskChatProtocolActivityRow"; import { TaskChatPlanPreviewCard } from "./TaskChatPlanPreviewCard"; @@ -33,6 +33,7 @@ import { CoordinatorChatPendingQuestion } from "@/components/coordinator/Coordin const EMPTY_ATTACHMENTS: IssueAttachment[] = []; interface TaskChatThreadViewProps { + scrollingHistory?: ScrollingHistory; items: TaskChatItem[]; /** * Content rendered above the first message INSIDE the scroll viewport (the @@ -303,6 +304,7 @@ export function taskChatItemSpacingClass( * TaskChatItem[] onto the primitives inside the auto-follow scroller. */ export function TaskChatThreadView({ + scrollingHistory, items, header, onApprovalDecision, @@ -433,7 +435,7 @@ export function TaskChatThreadView({ if (!scroll) return body; return ( - + {body} ); diff --git a/ui/src/components/task-chat/TaskMessageScroller.test.tsx b/ui/src/components/task-chat/TaskMessageScroller.test.tsx index 379fba4d6d9..fbc0ce1d9f2 100644 --- a/ui/src/components/task-chat/TaskMessageScroller.test.tsx +++ b/ui/src/components/task-chat/TaskMessageScroller.test.tsx @@ -12,14 +12,14 @@ const PILL_SELECTOR = 'button[aria-label="Scroll to latest"]'; * jsdom has no layout, so scroll geometry is faked per element: scrollHeight / * clientHeight are pinned and scrollTop is backed by a plain variable. */ -function fakeGeometry(el: HTMLElement, { scrollHeight = 1000, clientHeight = 400 } = {}) { +function fakeGeometry(el: HTMLElement, { scrollHeight = 1000, clientHeight = 400, clamp = false } = {}) { let scrollTop = 0; Object.defineProperty(el, "scrollHeight", { value: scrollHeight, configurable: true }); Object.defineProperty(el, "clientHeight", { value: clientHeight, configurable: true }); Object.defineProperty(el, "scrollTop", { get: () => scrollTop, set: (v: number) => { - scrollTop = v; + scrollTop = clamp ? Math.max(0, Math.min(v, el.scrollHeight - el.clientHeight)) : v; }, configurable: true, }); @@ -125,6 +125,118 @@ describe("TaskMessageScroller", () => { vi.unstubAllGlobals(); }); + it("loads one older page on upward scroll, keeps the status slot, and pauses after failure", async () => { + let release!: () => void; + const load = vi.fn(() => new Promise(resolve => { release = resolve; })); + const history = { hasMore: true, loading: false, error: false, load }; + const update = (tick: number) => flushSync(() => root.render( +
saved messages
, + )); + update(0); + fakeGeometry(scroller()); + update(1); + expect(load).not.toHaveBeenCalled(); + await scrollTo(scroller(), 400); + expect(load).not.toHaveBeenCalled(); + await scrollTo(scroller(), 150); + expect(load).toHaveBeenCalledTimes(1); + await scrollTo(scroller(), 100); + scroller().dispatchEvent(new WheelEvent("wheel", { bubbles: true, deltaY: -100 })); + await flushEvents(); + expect(load).toHaveBeenCalledTimes(1); + const status = container.querySelector('[data-testid="chat-history-status"]'); + history.loading = true; + update(2); + expect(container.querySelector('[data-testid="chat-history-status"]')).toBe(status); + expect(status?.textContent).toBe("Loading earlier messages…"); + release(); + await flushEvents(); + history.loading = false; + history.error = true; + update(3); + await scrollTo(scroller(), 50); + expect(load).toHaveBeenCalledTimes(1); + expect(status?.textContent).toContain("Earlier messages couldn’t load."); + (status?.querySelector('button') as HTMLButtonElement).click(); + expect(load).toHaveBeenCalledTimes(2); + release(); + await flushEvents(); + history.error = false; + history.hasMore = false; + update(4); + await scrollTo(scroller(), 0); + expect(load).toHaveBeenCalledTimes(2); + }); + + it("follows a late image resize in the message body after the history status", () => { + const observers: { targets: Set; resize: () => void }[] = []; + vi.stubGlobal("ResizeObserver", class { + targets = new Set(); + constructor(callback: ResizeObserverCallback) { observers.push({ targets: this.targets, resize: () => callback([], this as unknown as ResizeObserver) }); } + observe(target: Element) { this.targets.add(target); } + disconnect() {} + }); + const update = (tick: number) => flushSync(() => root.render( +
image content
, + )); + update(0); fakeGeometry(scroller(), { clamp: true }); update(1); + const body = container.querySelector('[data-testid="image-body"]')!; + Object.defineProperty(scroller(), "scrollHeight", { value: 1400, configurable: true }); + for (const observer of observers) if (observer.targets.has(body)) observer.resize(); + expect(scroller().scrollTop).toBe(1000); + }); + + it("retains upward intent when content reconciles before the scroll event", async () => { + const load = vi.fn(); + const update = (tick: number) => flushSync(() => root.render( +
saved messages
, + )); + update(0); fakeGeometry(scroller()); update(1); + await scrollTo(scroller(), 400); + scroller().dispatchEvent(new WheelEvent("wheel", { bubbles: true, deltaY: -1000 })); + scroller().scrollTop = 0; + update(2); + await dispatch(scroller(), new Event("scroll", { bubbles: true })); + expect(load).toHaveBeenCalledTimes(1); + }); + + it("holds a short conversation when an upward wheel gesture loads history", async () => { + const load = vi.fn().mockResolvedValue(undefined); + const update = (tick: number) => flushSync(() => root.render( +
short conversation
, + )); + update(0); + fakeGeometry(scroller(), { scrollHeight: 200, clientHeight: 400, clamp: true }); + update(1); + await scrollTo(scroller(), 0); + scroller().dispatchEvent(new WheelEvent("wheel", { bubbles: true, deltaY: -100 })); + await flushEvents(); + expect(load).toHaveBeenCalledTimes(1); + Object.defineProperty(scroller(), "scrollHeight", { value: 1200, configurable: true }); + update(2); + expect(scroller().scrollTop).toBe(0); + }); + + it("pages short conversations with touch and keyboard gestures", async () => { + const load = vi.fn().mockResolvedValue(undefined); + const update = (tick: number) => flushSync(() => root.render( +
short conversation
, + )); + update(0); fakeGeometry(scroller(), { scrollHeight: 200, clientHeight: 400, clamp: true }); update(1); + await scrollTo(scroller(), 0); + const touch = (type: string, y: number) => { + const event = new Event(type, { bubbles: true }); + Object.defineProperty(event, "touches", { value: [{ clientY: y }] }); + scroller().dispatchEvent(event); + }; + touch("touchstart", 100); touch("touchmove", 200); touch("touchmove", 250); + await flushEvents(); + expect(load).toHaveBeenCalledTimes(1); + scroller().dispatchEvent(new KeyboardEvent("keydown", { bubbles: true, key: "Home" })); + await flushEvents(); + expect(load).toHaveBeenCalledTimes(2); + }); + it("does not measure historical rows on each streaming update while following the bottom", () => { let initialized = false; let rowMeasurements = 0; diff --git a/ui/src/components/task-chat/TaskMessageScroller.tsx b/ui/src/components/task-chat/TaskMessageScroller.tsx index 5c5501fe29d..a0f7c9e6db2 100644 --- a/ui/src/components/task-chat/TaskMessageScroller.tsx +++ b/ui/src/components/task-chat/TaskMessageScroller.tsx @@ -21,7 +21,15 @@ function motionDisabled(): boolean { return window.matchMedia("(prefers-reduced-motion: reduce)").matches; } +export interface ScrollingHistory { + hasMore: boolean; + loading: boolean; + error?: boolean; + load: () => unknown; +} + interface TaskMessageScrollerProps { + scrollingHistory?: ScrollingHistory; children: ReactNode; /** Value that changes whenever content that could grow the thread updates. */ contentKey: unknown; @@ -44,7 +52,7 @@ interface TaskMessageScrollerProps { * the glide cancels it and treats the user as unpinned. Content-driven follow * while pinned stays instant, so no reflow/jump happens during streaming. */ -export function TaskMessageScroller({ children, contentKey, className }: TaskMessageScrollerProps) { +export function TaskMessageScroller({ children, contentKey, className, scrollingHistory }: TaskMessageScrollerProps) { const streamlined = useStreamlinedTaskChatPresentation(); const navigation = useTaskChatScrollNavigation(); const initialPositionApplied = useRef(false); @@ -57,6 +65,26 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes const scrollbarIdleTimerRef = useRef(null); const scrollbarIdleDelayRef = useRef(null); const [pillPhase, setPillPhase] = useState("hidden"); + const previousScrollTop = useRef(0); + const touchStartY = useRef(null); + const requestingHistory = useRef(false); + const upwardIntent = useRef(false); + const requestEarlierHistory = useCallback(() => { + upwardIntent.current = false; + if (!initialPositionApplied.current || !navigation.ready || !scrollingHistory?.hasMore + || scrollingHistory.loading || scrollingHistory.error || requestingHistory.current) return; + requestingHistory.current = true; + const el = ref.current; + if (el?.clientHeight) { + const rect = el.getBoundingClientRect(); + pinnedRef.current = false; + anchorRef.current = readThreadScrollAnchor(el, rect.top, rect.bottom); + navigation.remember(el.scrollTop, anchorRef.current, false); + } + Promise.resolve().then(() => scrollingHistory.load()).catch(() => {}).finally(() => { + requestingHistory.current = false; + }); + }, [navigation.key, navigation.hash, navigation.ready, scrollingHistory]); const showScrollbarWhileScrolling = useCallback(() => { const el = ref.current; @@ -148,10 +176,15 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes // Compensation preserves the existing reading anchor. Recapture only when // the reader scrolls or navigation selects a different position. rememberAnchor(false); + previousScrollTop.current = el.scrollTop; }, [rememberAnchor, scrollToBottom, hidePill]); const handleScroll = useCallback(() => { if (!ref.current?.clientHeight) return; + const top = ref.current.scrollTop; + const scrollingUp = top < previousScrollTop.current; + previousScrollTop.current = top; + if ((scrollingUp || upwardIntent.current) && top <= 192 && !easingRef.current) requestEarlierHistory(); showScrollbarWhileScrolling(); // A growing composer shrinks this viewport. Some browsers dispatch the // resulting scroll event before ResizeObserver, so preserve the previous @@ -162,7 +195,7 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes hidePill(); return; } - const pinned = isPinned(); + const pinned = requestingHistory.current ? false : isPinned(); if (easingRef.current) { // Smooth re-follow in flight: intermediate scroll events must not // unpin/re-show the pill. Only act once we arrive within the threshold. @@ -185,6 +218,7 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes hidePill, showPill, showScrollbarWhileScrolling, + requestEarlierHistory, ]); const handleJumpToLatest = useCallback(() => { @@ -245,7 +279,7 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes observer.observe(el); // The viewport itself does not resize when an image or historical row // grows. Observe the content box too, before the browser paints it. - if (el.firstElementChild) observer.observe(el.firstElementChild); + for (const child of el.children) observer.observe(child); return () => observer.disconnect(); }, [followViewportResize, hidePill, reconcileContent]); @@ -273,6 +307,26 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes
{ + if (["ArrowUp", "PageUp", "Home"].includes(event.key) && ref.current && ref.current.scrollTop <= 192) requestEarlierHistory(); + }} + onTouchStart={event => { touchStartY.current = event.touches.length === 1 ? event.touches[0]!.clientY : null; }} + onTouchMove={event => { + const start = touchStartY.current; + if (start !== null && event.touches.length === 1 && event.touches[0]!.clientY > start + 16 + && ref.current && ref.current.scrollTop <= 192) { + touchStartY.current = null; + requestEarlierHistory(); + } + }} + onTouchEnd={() => { touchStartY.current = null; }} + onWheel={event => { + upwardIntent.current = event.deltaY < 0; + if (event.deltaY < 0 && ref.current && ref.current.scrollTop <= 192) requestEarlierHistory(); + }} // Keep the viewport tied to the flex-sized wrapper vertically — // percentage heights don't reliably resolve against flex-determined // block heights, which let the thread overflow the page. In the @@ -288,7 +342,17 @@ export function TaskMessageScroller({ children, contentKey, className }: TaskMes )} data-testid="task-chat-scroller" > + {scrollingHistory ? <> + {scrollingHistory && (scrollingHistory.hasMore || scrollingHistory.loading || scrollingHistory.error) ? ( +
+ {scrollingHistory.error ? <> + Earlier messages couldn’t load. + + : scrollingHistory.loading ? "Loading earlier messages…" : "Scroll up for earlier messages"} +
+ ) : null} {children} + : children}
{pillPhase !== "hidden" ? ( @@ -2644,6 +2667,7 @@ const IssueDetailChatTab = memo(function IssueDetailChatTab({ initialMetadataPending || activityPending || linkedRunsPending || + (conversationMode && observedRunIds.length > 0 && commentRuns.isPending) || inputFacts.isLoading || !runtimeSelectionKnown) } @@ -2652,6 +2676,7 @@ const IssueDetailChatTab = memo(function IssueDetailChatTab({ activityError || (!historyExpanded && inputFacts.isError) || linkedRunsError || + (conversationMode && commentRuns.isError) || liveRunsError || (activeRunQueryEnabled && activeRunError) } @@ -2660,6 +2685,7 @@ const IssueDetailChatTab = memo(function IssueDetailChatTab({ void refetchActivity(); if (!historyExpanded && inputCommentIds.length) void inputFacts.refetch(); void refetchLinkedRuns(); + if (conversationMode && observedRunIds.length) void commentRuns.refetch(); void refetchLiveRuns(); if (activeRunQueryEnabled) void refetchActiveRun(); }} @@ -3343,13 +3369,15 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS }, enabled: !!issueId, }); - const { historyExpanded, expandHistory, recent: recentComments } = useIssueHistoryWindow( + const { historyExpanded, expandHistory, recent: recentComments, retainComment, acknowledgedComments } = useIssueHistoryWindow( issueId, // Classic settings can resolve before the personal workspace preference. // Do not persist full-history expansion for a surface we have not chosen. - Boolean(conversation || (taskInterfaceSettingsLoaded && classicTaskInterfaceEnabled)), + Boolean(!conversation && !coreIssueQuery.data?.conversationAgentId && taskInterfaceSettingsLoaded && classicTaskInterfaceEnabled), coreIssueQuery.data, + Boolean(conversation || coreIssueQuery.data?.conversationAgentId), ); + const paginatedConversation = Boolean(conversation || coreIssueQuery.data?.conversationAgentId); const issueQuery = useQuery({ ...getIssueDetailQueryOptions(queryClient, issueId!, { placeholderIssue: issueHeaderSeed @@ -3450,13 +3478,16 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS InfiniteData >(issueId ?? "pending"), }); + const earlierChatComments = useConversationHistoryPages(issueId, paginatedConversation, recentComments.data?.oldestCommentId, recentComments.data?.comments ?? []); const commentAnchor = location.hash.startsWith("#comment-") ? location.hash.slice("#comment-".length) : null; const linkedCommentId = commentAnchor && isUuidLike(commentAnchor) ? commentAnchor.trim() : null; const windowComments = useMemo( - () => mergeWindowComments(recentComments.data?.comments ?? [], flattenIssueCommentPages(commentPages?.pages)), - [commentPages?.pages, recentComments.data?.comments], + () => mergeWindowComments(recentComments.data?.comments ?? [], paginatedConversation + ? earlierChatComments.data?.pages.flatMap(page => page.comments) ?? [] + : flattenIssueCommentPages(commentPages?.pages), acknowledgedComments), + [paginatedConversation, earlierChatComments.data?.pages, commentPages?.pages, recentComments.data?.comments, acknowledgedComments], ); const linkedComment = useQuery({ queryKey: [...queryKeys.issues.comments(issueId!), "linked", linkedCommentId], @@ -4303,6 +4334,14 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS queryKeys.issues.comments(ref), (current) => current && { ...current, pages: removeIssueCommentFromPages(current.pages, commentId) }, ); + queryClient.setQueryData(queryKeys.issues.acknowledgedComments(ref), current => + current?.filter(comment => comment.id !== commentId), + ); + queryClient.setQueryData>( + queryKeys.issues.earlierComments(ref), current => current && { + ...current, pages: current.pages.map(page => ({ ...page, comments: page.comments.filter(comment => comment.id !== commentId) })), + }, + ); queryClient.setQueryData( [...queryKeys.issues.comments(ref), "recent"], (current) => { @@ -4355,6 +4394,14 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS } : current, ); + queryClient.setQueryData(queryKeys.issues.acknowledgedComments(ref), current => + current?.map(existing => existing.id === comment.id ? comment : existing), + ); + queryClient.setQueryData>( + queryKeys.issues.earlierComments(ref), current => current && { + ...current, pages: current.pages.map(page => ({ ...page, comments: page.comments.map(existing => existing.id === comment.id ? comment : existing) })), + }, + ); queryClient.setQueryData( [...queryKeys.issues.comments(ref), "recent"], (current) => current && { @@ -5089,6 +5136,7 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS if (context?.optimisticCommentId) { commentRenderKeys.current.set(comment.id, context.optimisticCommentId); } + if (paginatedConversation) retainComment(comment); queryClient.setQueryData>( queryKeys.issues.comments(issueId ?? comment.issueId), (current) => @@ -5512,6 +5560,7 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS }); } if (comment) { + if (paginatedConversation) retainComment(comment); if (context?.optimisticCommentId) commentRenderKeys.current.set( comment.id, @@ -6696,10 +6745,13 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS const attachmentsInitialLoading = attachmentsLoading && attachments === undefined; - const loadOlderComments = useCallback(() => { - void fetchOlderComments(); - }, [fetchOlderComments]); + const loadOlderComments = useCallback(() => paginatedConversation + ? earlierChatComments.load() : fetchOlderComments(), [paginatedConversation, earlierChatComments.load, fetchOlderComments]); const refetchLatestComments = useCallback(async () => { + if (paginatedConversation) { + await recentComments.refetch(); + return; + } // Refetch page 0 first so comments that arrived after initial load are // visible, then load every remaining older page. The chat thread is // paginated and virtualized, so "latest" must be resolved against the @@ -6729,7 +6781,7 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS } window.requestAnimationFrame(() => resolve()); }); - }, [issueId, queryClient, refetchComments]); + }, [issueId, queryClient, refetchComments, paginatedConversation, recentComments.refetch]); useEffect(() => { if ( !shouldPrefetchOlderComments && @@ -8508,8 +8560,11 @@ export function TaskDetailSurface({ conversation, tasksTab }: { tasksTab?: TaskS documents={issue.documentSummaries ?? []} workProducts={workProducts ?? issue.workProducts ?? []} attachments={attachments ?? []} - hasOlderComments={hasOlderComments} - commentsLoadingOlder={commentsLoadingOlder} + hasOlderComments={paginatedConversation && !historyExpanded + ? (earlierChatComments.data ? earlierChatComments.hasNextPage : recentComments.data?.hasMore) ?? false + : hasOlderComments} + commentsOlderError={paginatedConversation && earlierChatComments.isError} + commentsLoadingOlder={paginatedConversation ? earlierChatComments.isFetching : commentsLoadingOlder} onLoadOlderComments={loadOlderComments} onRefreshLatestComments={refetchLatestComments} composerRef={commentComposerRef}