diff --git a/.github/workflows/chat-completion-testops.yml b/.github/workflows/chat-completion-testops.yml deleted file mode 100644 index 7016f76dfec..00000000000 --- a/.github/workflows/chat-completion-testops.yml +++ /dev/null @@ -1,504 +0,0 @@ -name: Chat completion TestOps -run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Run the exact TestOps selection, register its plan, or launch it once - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps job-run identity; source, model and clocks are fixed - ALLURE_USERNAME: - type: string - required: false - description: Reserved service metadata; never grants authority - -permissions: - contents: read - actions: read - -concurrency: - group: chat-completion-testops - cancel-in-progress: false - -env: - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - operation: ${{ steps.guard.outputs.operation }} - target_sha: ${{ steps.guard.outputs.target_sha }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify actual main-only dispatch and both actors - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard - - select: - name: Register the dedicated plan or validate its real TestOps selection - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-testops-reporting - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} - execution_ids: ${{ steps.selection.outputs.execution_ids }} - matrix: ${{ steps.selection.outputs.matrix }} - policy: ${{ steps.selection.outputs.policy }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register three exact cases with no synthetic test result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap - node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context - - name: Validate case, plan, job, launch and same-run source before provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection - - name: Request the one reviewed plan launch and exit - if: needs.authorize.outputs.operation == 'launch-plan' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }} - CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan - - name: Retain token-free registration and launch receipts - if: always() && needs.authorize.outputs.operation != 'evaluate' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-bootstrap/ - ${{ runner.temp }}/chat-completion-testops-launch-plan/ - retention-days: 30 - if-no-files-found: error - - name: Retain the exact authenticated selection - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-selection-${{ github.run_id }}-1 - path: testops-pilot-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable catalog source without service or model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - name: Resolve the fixed candidate graph with lifecycle scripts disabled - run: | - pnpm install --lockfile-only --ignore-scripts - pnpm install --frozen-lockfile --ignore-scripts - - name: Build existing TypeScript and native Runner outputs - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - - name: Exercise the real workflow and pinned source host contract before model access - run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs - - name: Build native Runner outputs - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package the resolved lock and exact source outputs - env: - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$TARGET_SHA" - tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-build-${{ github.run_id }}-1 - path: | - chat-completion-build.tar.gz - chat-completion-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: ${{ matrix.executionId }} - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - strategy: - fail-fast: false - max-parallel: 1 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - actions: read - id-token: write - env: &retention - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }} - MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }} - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - steps: - - *checkout - - *node - - *guard - - *pnpm - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-completion-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - name: Verify source-bound build and authenticated selection before credentials - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" - node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json - (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) - tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") - - name: Qualify the pinned provider runtime and preinstalled browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Bind exact private evidence under the authenticated TestOps authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only this cell's exact-key storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify private custody before delivering the model key - env: &cell_session - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the selected original case with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain original evidence after success or failure - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and build the exact safe TestOps projection - needs: [authorize, select, build, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure') - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies with lifecycle scripts disabled - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" - - name: Prepare unchanged exact-key report and cell policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact first-cell reader - id: reader_0 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the first exact original - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact second-cell reader - id: reader_1 - if: fromJSON(steps.session.outputs.cell_count) > 1 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_1 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the second exact original - if: steps.reader_1.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '1' - AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact third-cell reader - id: reader_2 - if: fromJSON(steps.session.outputs.cell_count) > 2 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_2 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the third exact original - if: steps.reader_2.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '2' - AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Verify every selected original and its completeness receipt - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing report while preserving failed source grades - id: normalize - continue-on-error: true - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only exact reviewed synthetic fixture results - id: export - if: always() && steps.private_report.outcome == 'success' - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts - - name: Require a complete safe projection even after a failed normalization - if: always() - env: - EXPORT_OUTCOME: ${{ steps.export.outcome }} - run: test "$EXPORT_OUTCOME" = success - - name: Retain the metadata-only TestOps projection - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Record and independently verify each TestOps result - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - *allure - - name: Upload original outcomes, checks and private-evidence receipts - id: allure_upload - env: &upload_env - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results - run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops - - name: Verify actual selected results and complete result-owned attachments - id: verification - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *upload_env - run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops - - name: Retain closed TestOps verification receipts - if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-verify-upload/ - ${{ runner.temp }}/chat-completion-testops-upload/exported.json - retention-days: 30 - if-no-files-found: error diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml deleted file mode 100644 index 826a340ae91..00000000000 --- a/.github/workflows/chat-feature-testops.yml +++ /dev/null @@ -1,451 +0,0 @@ -name: Chat feature TestOps -run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Fixed evaluation, zero-result registration, or one TestOps plan launch - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps supplies its job-run identity; no test/source selector is accepted - ALLURE_USERNAME: - type: string - required: false - description: Reserved TestOps service metadata; never used as authorization - -permissions: - contents: read - -concurrency: - group: chat-feature-testops - cancel-in-progress: false - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - sha: ${{ steps.guard.outputs.sha }} - operation: ${{ steps.guard.outputs.operation }} - triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify fixed default-source dispatch and both actor identities - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - NODE_OPTIONS: '' - NODE_PATH: '' - run: node tests/runner-e2e/testops-chat-dispatch.ts - - select: - name: Validate TestOps selection or register zero-result setup - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - sha256: ${{ steps.selection.outputs.sha256 }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register the public job without an evaluation or a synthetic result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap - node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - - name: Validate the one real TestOps selection before any build or provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection - - name: Start exactly the validated TestOps plan and exit without waiting for its child - if: needs.authorize.outputs.operation == 'launch-plan' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} - CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - GH_TOKEN: ${{ github.token }} - run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan - - name: Retain only closed metadata from zero-result setup - if: always() && needs.authorize.outputs.operation == 'bootstrap' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json - ${{ runner.temp }}/chat-testops-bootstrap-state/once.json - ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json - ${{ runner.temp }}/chat-testops-bootstrap-state/job.json - ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain the one plan-launch receipt and stop state - if: always() && needs.authorize.outputs.operation == 'launch-plan' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-plan-launch-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-launch-plan-state/once.json - ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json - ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain only the token-free exact selection receipt - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-selection-${{ github.run_id }}-1 - path: testops-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable shipped native candidate without model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - name: Install only the source-reviewed graph without lifecycle scripts - run: | - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - pnpm install --frozen-lockfile --ignore-scripts - - name: Build the existing TypeScript and native runner owners - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package source-bound outputs and resolved lock - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 - path: | - chat-testops-build.tar.gz - chat-testops-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: One real native OpenCode backlog request on AWS - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - id-token: write - env: &retention - PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 - TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TARGET_SHA: ${{ needs.authorize.outputs.sha }} - MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Check out the same admitted source for trusted retention - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-testops-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - name: Verify immutable outputs and token-free TestOps selection - env: - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) - tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" - node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json - cmp chat-testops-lock.json chat-testops-worker-lock.json - cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") - - name: Materialize and qualify pinned OpenCode runtime and browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Prepare private evidence with shipped-main authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only the exact private cell storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify strict private custody before the model key - env: &cell_session - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the fixed original request with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain exact original grade and private evidence, including failures - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and export only reviewed fixture results - needs: [authorize, select, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies only - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" - - name: Prepare exact private report and cell-read policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact private cell reader - id: read_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the exact private original and verify all inventories - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: | - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing normalized campaign without exposing original logs - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately even when its grade failed - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only the bound normalized result, JUnit and marked PNGs - id: export - if: always() && steps.private_report.outcome == 'success' - env: - TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts - - name: Upload only the sanitized TestOps projection - if: always() && steps.export.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Report the original outcome to its exact TestOps case - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - *allure - - name: Verify and upload only the bound safe case outcome - id: allure_upload - env: &testops_reporter - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results - run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops - - name: Verify the actual TestOps result after upload processing - id: verify_upload - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *testops_reporter - run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops - - name: Retain only closed actual TestOps verification metadata - if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json - ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json - retention-days: 30 - if-no-files-found: error diff --git a/DESIGN.md b/DESIGN.md index 068a7e4530b..9d53a3c844b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -117,8 +117,8 @@ completes without a provider reply. Preserve the reply through persistence. Task chat shows execution errors and waits only while they remain relevant. Exhausted attempts use a padded, rounded notice with a destructive-tinted background, -an error icon, and foreground text. The surrounding layout supplies separation from -the conversation. Automatic retry countdowns keep their quieter inline styling. +an error icon, and foreground text. A wrapper adds top padding above the tinted +surface. Automatic retry countdowns keep their quieter inline styling. Completing or cancelling a task hides its old execution notices. A newer attempt by the same agent or an explicit successor supersedes earlier run notices; an unresolved execution hold remains visible. Historical turns keep their responses, diff --git a/doc/DATABASE.md b/doc/DATABASE.md index e617f321dcc..04698b00f19 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -522,23 +522,59 @@ leave database construction unchanged. An expired window stays inactive after a restart; remove the setting after the investigation. The diagnostic measures awaited Postgres.js `unsafe` responses before Drizzle -column decoding. It retains the Drizzle builder caller for normal, prepared, -transaction, and dedicated-pool execution. It issues no queries. -It logs the public server caller locations, static schema table, operation, -query/row counts, and response value sizes. It emits at most 2,048 aggregate -groups and a summary. Measurement stops at the deadline. Environment-enabled +column decoding. It issues no queries. Configuration is read when each database +client is created; setting the environment on an already-running process does +not install the capture on existing clients. Prepare the deployment artifact +before choosing the UTC window, then start the window after rollout and warm-up. + +Server entry points use `withDatabaseOperation(owner, action)` to name the work +that executes queries. Owners are fixed semantic labels, never request paths, +company or run IDs, user input, or SQL. The execution owner is captured for each +query and retained through completion, including transactions and dedicated +pools. A prepared query reused by different operations keeps the owner of each +execution. The builder caller is separate from that execution owner. +Owner context is active while a valid future window is armed, up to 30 minutes +before its start, so work that crosses the start boundary keeps its owner. Query +accounting starts only at the window start. With no armed window, or after +expiry, the owner helper calls the action directly without a context scope. + +The capture aggregates owner, static schema table, operation, calls, returned +rows, query errors, and total/maximum completion wall time. `totalWallMs` and +`maxWallMs` include pool, database, network, and event-loop waits; they are not +CPU measurements, and concurrent query durations overlap. Timing excludes the +capture's response sizing. Rank calls and returned rows both per second and per +completed service operation; a busy owner is not necessarily wasteful. + +Response sizing samples successful results 1, 17, 33, and so on per aggregate +group (`sampleEvery: 16`). It examines at most +256 cells, 65,536 string code units, and 65,536 value bytes per sampled result; +an individual string over 16,384 code units is not measured. Parsed objects and +other complex values are not serialized. Oversized, complex, or unexamined +values leave an incomplete sample. `sampledValueBytes` is a lower bound over +sampled primitive values, not a total response-size estimate or network bytes. +Do not extrapolate it as the total bytes returned by the database. Inspect +`sampledResults`, `sampledRows`, `measuredValues`, `unmeasuredValues`, and +`incompleteSamples` alongside it. `sizingWallMs` records the diagnostic's sizing +work separately. Use a separate capture-disabled CPU profile to measure normal +processing cost. + +The diagnostic emits at most 2,048 aggregate groups and a summary. Measurement +stops at the deadline. Environment-enabled captures emit at most 64 records per second and can take another 33 seconds to finish writing the bounded output. Wait for the summary before collecting it. It never logs SQL, parameters, result values, connection strings, or query errors. In-flight queries continue normally. -For a bundled server, keep the linked source maps and enable Node's -`--enable-source-maps` option so the diagnostic can resolve public caller -locations. Restore the prior Node options after capture. Sizes still count when -caller locations are unavailable. +Semantic owners work in a bundled server without source maps. For queries +without an explicit owner, keep the linked source maps and enable Node's +`--enable-source-maps` option to resolve public caller locations. Restore prior +Node options after capture. Known semantic owners avoid repeated stack capture. +Queries without an available owner/caller still contribute to the totals. -Look for `paperclip.db_response_capture.entry.v1` and -`paperclip.db_response_capture.summary.v1` in the process logs. A log provider +Look for `paperclip.db_response_capture.entry.v2` and +`paperclip.db_response_capture.summary.v2` in the process logs. Version 2 uses +sampled, bounded primitive-value sizing; do not interpret it as the version 1 +all-result byte estimate. A log provider can store `event` as a structured field instead of message text. Filter by that field or read the bounded window from the exact deployment. Check the summary and group totals, including the unattributed overflow group. @@ -552,22 +588,26 @@ Fetch a complete capture as follows: from zero through `groupPages - 1`, filtering on the entry event, `captureId`, and `groupPage`. Request at least `groupPageSize` matching records per page. A structured query can use - `@event:paperclip.db_response_capture.entry.v1 AND @captureId: AND @groupPage:0`. + `@event:paperclip.db_response_capture.entry.v2 AND @captureId: AND @groupPage:0`. 3. If the provider's record limit is smaller than a page, fetch each `groupIndex` separately with the same capture ID. Each index matches one entry. Do not split by timestamp: entries can share a timestamp. 4. Deduplicate by `(captureId, groupIndex)`. Require every index from zero through `groups - 1`, with no gaps. Entry count must equal `groups`, summed - `calls` must equal `completedQueries`, and summed `responseValueBytes` must - equal the summary total. A missing page is an incomplete measurement. + `calls` must equal `completedQueries`, and summed `rows`, `sampledResults`, + and `sampledValueBytes` must equal their summary totals. A missing page is an + incomplete measurement. Keep overflow and unattributed groups visible. Tagged raw-driver calls, streams, unawaited raw-driver dispatch, and protocol overhead are not measured. Relational queries use a driver-level caller fallback. -Queries still in flight at the deadline are reported separately. Sizes describe -driver values before column decoding, not billed network bytes. Complex parsed -values use JSON size estimates, counted in `estimatedValues`. Reconcile them with -provider metering and socket measurements. Serialization adds temporary CPU and memory work; -measure the final transfer rate after the window expires. +Queries still in flight at the deadline are reported separately. The census does +not include every PostgreSQL statement or internal transaction command, so its +calls are not interchangeable with database-wide commit counters. It does not +measure SQL compilation, driver parsing CPU, or Drizzle row-mapping CPU. Use the +ranked owners to choose a narrow follow-up profile or synchronous-stage +measurement; do not attribute CPU by subtracting process CPU usage across an +asynchronous query. Remove the capture setting after collecting its final +summary; the absolute deadline already prevents further measurement. ### Push the schema diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index c0827418fe7..d2058a79615 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -2621,6 +2621,16 @@ connections, including provider-native tools. Company permissions, approval gates, and workspace isolation still apply. Explicit restrictive modes remain restrictive; omitted settings use full auto. +Assigned app tools can use `paperclip_search_assigned_tools` and +`paperclip_call_assigned_tool` instead of direct tool definitions. Search with +`{"query":"github"}` and call an exact returned tool name. Omitted, empty or +whitespace-only `schemaTool` values perform normal discovery, including when +`schemaOffset` is zero. To read a large schema, set `schemaTool` to the exact +`inputSchemaRef` from discovery and follow `nextSchemaOffset`. A positive +`schemaOffset` requires a nonblank reference. Unknown or revoked references +return guidance to search again; discovery still checks current grants and work +modes against the session's pinned catalog. + Test questions, saved plan revisions, approval before task handoff, status lookups, and `/new` preserving chat history. Hiring additionally requires the operator-controlled [runner API tools](runner-api-tools.md) rollout; enabling diff --git a/doc/PRODUCT.md b/doc/PRODUCT.md index cba3bec4ae2..62390a783de 100644 --- a/doc/PRODUCT.md +++ b/doc/PRODUCT.md @@ -97,6 +97,11 @@ When an agent creates a subtask or follow-up task, its source task's feed shows a linked card at creation time with the current title, task number, and status. The Tasks tab keeps the full relationship list. +Tasks waiting between automatic checks show **Monitoring** with a blue clock. +Task lists, board columns, filters, search, recent navigation and task properties +use the same read-only display state. The task retains its execution lifecycle +status, and the monitor banner shows when the next check is due. + Automatic monitor checks pause when a task is blocked. A person can use Check now to request one check of a retained monitor through normal execution admission. Dependencies, recovery, pause, budgets and ownership still apply. Provider-usage @@ -141,9 +146,15 @@ choice stays selected while task data refreshes. The task Filters menu also offers **Routine tasks**. It selects tasks with `originKind=routine_execution`, including manually triggered routine runs, even when other automatic tasks are hidden. The routines list's **View all tasks** -link opens `/issues?view=all&originKind=routine_execution` with that checkbox +link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox selected. The URL retains the selection across reloads and task navigation. -Search and pagination apply the routine origin filter on the server. +Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides +saved collection sorting, and the routines link opens newest-created tasks first. +Shared sort links work across Coordinator, Streamlined UI, and Classic UI, including +sort fields without a visible column in the selected interface. Search relevance +and board-column membership keep their existing server order. +Search and pagination apply the routine origin filter on the server. Creation-time +sorting also runs before pagination, with task ID as a stable tie-breaker. Tasks and project lists hide agent-created and system-created tasks by default. The Filters menu's **Show agent-created tasks** checkbox includes routine runs, diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 59874d54e8b..810faf75d9d 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -423,6 +423,7 @@ retain their titles and default to no generation request. - `title` text not null - `description` text null - `status` enum: `backlog | todo | in_progress | in_review | done | blocked | cancelled` +- Task reads additionally expose read-only `displayStatus`: the lifecycle status, or `monitoring` for an agent-owned scheduled ordinary monitor wait with no active execution, locks, review participant or active recovery action. Board collection/search `displayStatus` filters and counts use the same projection before pagination. This field is not persisted and cannot authorize a status transition; existing `status` filters retain their lifecycle semantics. - `priority` enum: `critical | high | medium | low` - `review_policy` nullable enum: `anyone | not_creator | human_only`; null is equivalent to `anyone` - `assignee_agent_id` uuid fk `agents.id` null @@ -2712,9 +2713,16 @@ its upstream authors; operators can pause updates to review changes manually. The task Filters menu also offers **Routine tasks**. It selects tasks with `originKind=routine_execution`, including manually triggered routine runs, even when other automatic tasks are hidden. The routines list's **View all tasks** -link opens `/issues?view=all&originKind=routine_execution` with that checkbox +link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox selected. The URL retains the selection across reloads and task navigation. -Search and pagination apply the routine origin filter on the server. +Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides +saved collection sorting, and the routines link opens newest-created tasks first. +Shared sort links retain their order across task interfaces. Workflow sorting +requests the existing authorized dependency-edge projection. Search and board +queries keep their existing server order. Blocked-attention lists reject +`sortField=created` with `422` because they use their own activity order. +Search and pagination apply the routine origin filter on the server. Creation-time +sorting also runs before pagination, with task ID as a stable tie-breaker. Tasks and project collections default to hiding agent/system-created tasks and provide a saved **Show agent-created tasks** filter. Classify creation using diff --git a/doc/evals.md b/doc/evals.md index 4071cf01866..d0ea2fa5ec3 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -57,8 +57,9 @@ OpenCode Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, using ordinary production instructions, real browser messages and independent state/provenance checks. A separately selected native OpenCode/DeepSeek1017 -backlog cell measures TestOps integration acceptance with the same oracle and -original bounds. Its source registration is not an AWS or TestOps pass. The +backlog cell supplied historical TestOps integration acceptance with the same +oracle and original bounds. Its former smoke plan and the three-case completion +pilot are retired; their cases and measurements remain in history. The [full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops) maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and fixtures use a preserved immutable catalog revision; trusted workflow helpers run diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index dd0378688f5..3548914d2fc 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -112,6 +112,22 @@ The issue is actively owned work. For agent-owned issues, `in_progress` should not be allowed to become a silent dead state. +### Monitoring presentation + +An agent-owned `in_progress` task displays **Monitoring** with a blue clock while +it has a persisted ordinary monitor deadline and no active run, execution or +checkout lock, review participant, or active recovery action. The persisted +status remains `in_progress`. Monitor admission, checkout, finalization, recovery, +and agent lifecycle filters keep their existing contracts. Review, blocked, +terminal, and provider-quota recovery states keep their own presentation. + +Task reads expose optional, read-only `displayStatus`. Board task collections and +search can filter by `displayStatus`; filtering happens before pagination and +counts. Existing `status` filters retain their lifecycle meaning. Monitoring is +not a manually selectable task status or a board drag target. An overdue check +retains its visible deadline warning; the label does not attest that a provider +is running or that a check succeeded. + ### `blocked` The issue cannot proceed until something external changes. @@ -764,6 +780,15 @@ conflict detection. Legacy adapter snapshots remain inline. Snapshot-only worker do not load the adapter process-launch helpers. The directory merge loads those helpers when it resolves its instance-scoped lock root. +Portable remote workspace capture stages only self-contained Git metadata. It +streams working files into the archive without a second working-tree copy and +does not repack an already dissociated mirror. Each streamed file must match its +recorded hash, size and mode and retain its filesystem identity while read. +Capture rechecks the complete source inventory and Git state before returning a +receipt. A changed source or cancelled capture removes its incomplete archive; +an existing archive remains untouched. Temporary space must still hold the +normalized Git metadata and the complete archive together. + ### Remote Codex model compatibility Fresh remote Codex Runner runs check the selected image CLI before saving the @@ -862,6 +887,15 @@ The task thread exposes the guarded Retry action for failed or timed-out convers A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed. +A successful connected run can still have newer admitted input waiting behind it. +Task Retry resumes that input's existing durable chat receipt through the inbound +owner, instead of replaying the successful run as a failed attempt. The original +actor, source, result, and publication history remain intact. Duplicate clicks +follow one canonical receipt. Current board permission, task ownership, source +authorization, and prior execution cleanup are rechecked before normal admission; +pause, budget, approval, and dependency gates remain effective. Completion or +newer execution supersedes an unadmitted retry. + A valid recovery action must name: - the source issue and company diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 09d4c670601..ef45a370ed9 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -133,6 +133,17 @@ fingerprint and RPC company scope for later read-only verification. Retained allocations, unfinished runs/accounting/retries, export intents and live setup credentials continue to block sleep. +For a reused remote allocation, a completed predecessor can resolve termination +through the final owner's destruction receipt only when the full host-recorded +lease handoff chain matches. Company, agent, task, workspace, environment, provider +and allocation identities must agree. A stopped successor, broken handoff or +unrelated owner grants no cleanup authority. This resolution does not rewrite +historical leases, execute remotely or claim that uncollected files were saved. +The normal agent-file recovery worker retains the no-save outcome and completes +cleanup after confirmed destruction. A completed cleanup receipt with a stale +retry deadline is reconciled once through the existing deadline worker. Clearing +that deadline compares the receipt so a concurrent cleanup obligation is kept. + Pending human-only connection setup can wait asleep when no active enabled connection of the requested purpose exists. Available access reblocks sleep so the existing reconciliation path can run; answering a setup card continues to diff --git a/doc/observability.md b/doc/observability.md index a96ae96058b..abadb0e6538 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1900,6 +1900,49 @@ name to `sandbox.daytona.`. The host maps any other value to daytona provider emits these spans today, so the segment is the literal `daytona`. +### Daytona lease acquisition and resume + +The worker invocation carries nested, monotonic timing receipts for lease work. +The host also admits the following fixed `lease.*` names and prefixes them with +`sandbox.daytona.`. These use the existing opt-in endpoint and host-owned parent +context; they do not create an exporter or first-party Telemetry events. + +```text +execution.plugin_dispatch +└── plugin.worker + ├── sandbox.daytona.lease.wait_for_teardown (only when teardown blocks resume) + ├── sandbox.daytona.lease.lookup + ├── sandbox.daytona.lease.refresh (missing-container confirmation) + ├── sandbox.daytona.lease.channel_cleanup (before restart) + ├── sandbox.daytona.lease.start (or lease.recover) + └── sandbox.daytona.lease.verify + ├── sandbox.daytona.lease.workspace + ├── sandbox.daytona.lease.sentinel + └── sandbox.daytona.lease.shell +``` + +`lease.lookup` includes cached-handle lookup and any required provider refresh. +`lease.start` and `lease.recover` measure the actual SDK operation, including its +internal readiness wait. They also cover starts triggered by execution or sync. +They do not separate SDK-internal requests or provider-side queue time. +Their `paperclip.sandbox.startup.state` attribute records the state before the +call: `started`, `stopped`, `archived`, `error`, or `other`. Both the plugin and +host restrict this value; no provider error text or resource identity is sent. + +Fresh acquisition emits `lease.create`, `lease.workspace`, `lease.shell`, +`lease.expiry`, and `lease.sentinel`. `lease.create` covers the existing creation +helper, including qualification-only snapshot resolution and creation-failure +cleanup. Setup-failure deletion is a separate `lease.cleanup` sibling, so it +does not inherit a failed workspace or verification phase. Existing deadlines, +ownership, cleanup and retry behavior are unchanged. Failed awaited steps close +with error status; the receipt contains no exception message. + +Children nest beneath the current `plugin.worker` receipt and verification +children nest beneath `lease.verify`. Never add parent and child durations. +Receipt budgets and clock validation still apply. Older workers without +`tracer.withSpan` keep the original behavior without these detail spans. Tracing +failures preserve the operation's result or error and never repeat it. + The `sandbox.daytona.*` spans use this closed attribute allowlist. The host drops every other key, so a command, an argument, a path, an id, a standard output, or a standard error never rides a provider span. The host records only @@ -1908,6 +1951,7 @@ the attributes that the producer sends for one span. | Attribute | Type | Optional | Meaning | | ------------------------------------------------ | ------ | -------- | --------------------------------------------------------------------------------------------------------- | | `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. | +| `paperclip.sandbox.startup.state` | string | yes | State before start/recovery: `started`, `stopped`, `archived`, `error`, or `other`. | | `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). | | `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.daytona.pack` span. | | `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.daytona.transfer` span. | diff --git a/doc/performance/native-backup-publication.md b/doc/performance/native-backup-publication.md new file mode 100644 index 00000000000..b461ea49d76 --- /dev/null +++ b/doc/performance/native-backup-publication.md @@ -0,0 +1,53 @@ +# Native backup publication benchmark + +Retiring a native harness backup can involve many immutable directories and +small files. Cleanup repairs directory permissions before removing the retired +tree. Publication waits for cleanup, preserves the previous backup when lease +stamping fails, and emits its completion notification after cleanup succeeds. +Cleanup uses asynchronous filesystem operations so unrelated requests can +continue while it runs. Directory preparation is serial and does not follow +symlinks into live trees. + +Run the focused correctness tests from an installed checkout: + +```sh +pnpm --filter @paperclipai/server exec vitest run src/services/native-runtime/native-harness-backup-cleanup.test.ts src/services/native-runtime/native-harness-backup-publication.test.ts +``` + +Run the performance comparison from the repository root with the Node version +used by the target server. Use a quiet machine and an explicit baseline commit: + +```sh +node server/scripts/benchmark-native-backup-publication.mjs \ + --baseline-ref 774ddc72713d6aa7cb516691c71f1fc23bcfa5ff \ + --output /tmp/native-backup-publication.json +``` + +The script compiles the real publication service and its dependencies for both +the baseline commit and the working tree. Each run uses a separate server +process. Fixture preparation occurs before measurement. The default fixture has +100 immutable directories with 100 small files in each of the `current` and +`previous` backups: 20,000 files to remove. Both backups also have a symlink to a +read-only live directory outside the retired trees. The new pending backup has +a distinct manifest. Every completed run checks the new manifest, complete +removal of retired and pending trees, and unchanged live contents and mode. + +A separate process sends requests to a small loopback HTTP endpoint every five +milliseconds. Arrivals use a fixed clock and include overdue requests. The +report includes request completion and error counts, generator lateness, p95 +and maximum foreground latency, requests served during publication, publication +wall time, process CPU time, and maximum event-loop delay. Three pairs alternate +baseline and treatment order. Change the fixture with `--pairs`, `--directories`, +`--files-per-directory`, or `--interval-ms`. + +The result records Node/platform, baseline SHA, treatment HEAD and uncommitted +service diff. Compiled modules and source maps stay in the printed evidence +directory. Successful fixtures are removed; failed fixtures remain for diagnosis. + +This is a service benchmark. The HTTP endpoint is a responsiveness probe, not +the assembled Paperclip API. The lease-stamp callback resolves without database +work. It does not prove a hosted latency or total-CPU improvement. Process CPU +during publication includes foreground requests that the synchronous baseline +cannot serve until cleanup completes. Async cleanup can consume more CPU while +keeping the event loop responsive. Qualify the affected run/checkpoint/recovery +journey and compare matched hosted windows before claiming deployed acceptance. diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index 71e4bf658b2..fac22b56194 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,10 +1,201 @@ # Zero-downtime web deployments -Date: 2026-10-31. Status (2026-11-01 15:39 UTC): existing-volume staging enrollment -and manual sleep/wake preserve saved files and read/save behavior. The full local -active-agent journey now passes on `cc23019691`; hosted browser continuity and -the ten-deploy staging campaign remain unqualified. Natural database autosuspension -passes after abandoned test-resource cleanup. No production canary yet. +Current state (2026-11-02 18:31 UTC): cleanup recovery PR +[#18633](https://github.com/paperclipai/paperclip/pull/18633) merged as +`60a99307b3bb1062078d35af6677e3f377d8bab8`. Its final head passed all PR CI, +full local typecheck and build, and seven focused cleanup regressions. Greptile +reviewed that head at 5/5; both review threads are resolved. The duplicate full +local test run was stopped after six chat shards passed, while general suites +were still running. Its incomplete local result is preserved separately from +successful full PR CI. + +Canonical Linux release verification failed on the unchanged two-second busy-row +startup test on several consecutive revisions, including the cleanup merge. +The held-row diagnostic completed in 420 ms on macOS without a database lock +wait. Inspection found that `currentNativeControllerIdentity` always called the +server-info snapshot, which synchronously probes Git three times even when the +OS process birth time is available. Controlled one-second delays in real Git +commands reproduced the exact startup assertion failure with real PostgreSQL. +The follow-up separates the constant fallback process birth time from diagnostic +Git metadata. New regressions cover both available and unavailable OS birth time. +The two-second startup limit and release gates remain unchanged. Linux release +verification must pass before the new source can enter hosted qualification. + +The follow-up is [#18649](https://github.com/paperclipai/paperclip/pull/18649). +Its first head received two 5/5 reviews and passed every CI test shard except +an existing voice-retention precision test. That test advances its fake clock +automatically while asserting behavior across a one-millisecond boundary. A +controlled 60 ms database-entry delay reproduced the exact failure. Holding the +clock fixed makes the same assertion pass with that delay; the final correction +only disables automatic clock advancement in that one precision test. The +original CI failure and both controlled results are retained. No production +voice code or assertion was weakened. + +The earlier ten-roll result remains evidence for its recorded images. Neither +the cleanup fix nor the controller-identity follow-up has reached the staging +fixture. Final acceptance still requires a qualified compatible image pair, +fresh staging replacements, natural idle and browser sleep/wake, followed by the +isolated production canary. Cloud source is `b44304603d`; its private packages +require comparison against the new Core SDK and shared archive. Earlier candidate +Core `42c64d4ec4` differs in SDK; `b9e94b85a6` differs in shared types. Neither is +a valid rollback pair for the cleanup merge. No release gate was bypassed. + +The production browser was signed in as Dotta B on the ordinary new-organization +form. No organization was submitted and no production config changed. The only +hosted target changed so far is the pinned staging qualification fixture. +Disposable local PostgreSQL and Moto S3 fixtures passed an independent object +read/write preflight. Docker subsequently stopped; local container qualification +requires restoring that dependency. Hosted acceptance uses real object storage. + +Earlier retained evidence and failure history follows. + +Date: 2026-10-31. Status (2026-11-02 17:23 UTC): Core #18554 is merged as +`1a6017d6460267ad89d7e1784e2b521c28d8e9ac`. The ten-roll and native-agent +proof used Cloud `461c4aea28`. Separately reviewed Cloud #1091/#1092 subsequently +reached staging as `89dd0cfec2`; their changes concern workspace cleanup and +Agent Computer resource fencing. Railway, fleet and deployment workflow paths +are unchanged. Production remains on its original controller revision; the +next manual controller deployment must use current reviewed master. +Both immutable Cloud distributions built from that fixed source passed packaged +Linux rehearsals through the Railway wrapper: 260 reads, 53 saves, zero failures, +legacy-volume migration, an unready replacement, and removal of both predecessor +homes. Each fixture cleaned its containers. These rehearsals do not exercise an +active agent or an actual browser. The two distributions have distinct image +digests but identical application source; their compatible rollback qualifies +artifact replacement, not compatibility between different source revisions. + +The final Core PR has two 5/5 reviews and all required checks passed. One legacy +browser CI shard failed because its deferred-wake fixture observed no queued wake; +the unchanged case passed locally on isolated PostgreSQL and its single CI retry +passed. The original failure remains in the PR evidence. The full local live-agent +journey passed on `cc23019691` before the final non-suspended dispatch refinement; +that refinement passed its three focused Linux regressions, server typecheck, +build and final CI. Do not describe the earlier live run as a rerun of the merge. + +The fixed canonical package baseline passed on the pinned migrated staging fixture +using the allowed one-time maintenance path. An actual Chromium tab running the +older UI reconnected, retained an unsaved title edit, and saved it through the new +server; an independent API read confirmed the title. That old tab opened the +Agents page, and all four previously captured asset hashes still match after the +old deployment was removed. Baseline maintenance interruptions are preserved and +are not counted as zero-downtime qualification. + +The first ten-roll attempt was cancelled at step one, before Railway replacement. +The controller's pre-existing hot-upgrade allowlist named an older test stack, +so it drained rather than negotiated preservation of the new fixture's active run. +Continuous read/save and browser probes remained healthy, but this is NOT active +execution continuity evidence. The original command completed normally once; its +downloaded output bytes and native terminal were independently verified. The +owned fixture has been added to the existing allowlist without changing fleet +defaults. The normal staging controller workflow is applying that correction. +The rollout harness now checks both canary policy and positive tenant readiness +before submitting a campaign. The cancelled attempt remains failed evidence, +with no automatic campaign retry. +The configuration deployment then failed before Railway submission when Sentry +release registration failed or timed out. Its source checks passed; the original +logs are retained. One unchanged whole-workflow retry preserves attempt-bound +preflight requirements. That retry failed at the same pre-Railway Sentry request. +A read-only Sentry lookup confirms the release exists (HTTP 200). The canary-only +configuration was then applied by redeploying the existing successful controller +deployment: the source and replacement image digests are identical, and health +attests unchanged Cloud `461c4aea28`. Both failed workflow attempts are retained; +no Sentry gate was disabled and no new application release was built in this +configuration recovery. The new active-command attempt and gated sequence are +starting with the corrected controller. The first preflight stopped before task +admission on a missing optional plan's 404. `useIssuePlanDocument` explicitly +maps that response to null; an independent read confirmed no plan on the owned +completed task. The raw response remains recorded and the browser gate permits +only that verified path. The configuration redeploy's actual browser reconnect +was 1.22 seconds, with no probe read/save failure. + +All ten Railway replacements are independently qualified. Final continuous proof +passed 12,521 reads and 1,566 saves, with every acknowledged save present exactly +once and all 34 file hashes matching. There were zero HTTP failures. Actual +browser reconnects across the ten deployments ranged from 1.16 to 1.29 seconds. +The old UI assets and original agent artifact remain readable after deployment +ten and provider cleanup. Both traffic observers have stopped cleanly. The first +replaced the webserver while the original eight-minute remote command was +running. Its one execution ID started before replacement and completed afterward +with exit zero. The second replaced the server after completion; a later follow-up +retained the same native/provider session and runner, with accepted native +terminals and finalized accounting for both runs. The 91-byte artifact was +downloaded and independently checked for exactly one start and completion. +Actual browser reconnects were 1.28, 1.16 and 1.25 seconds; continuous reads and +saves have no failures so far. Previously unopened Projects and Skills pages rendered in the +retained old tab. A title draft survived the third cutover and was saved through +the UI, then independently read back from the replacement. + +The harness stopped after deployment three on another owned task's expected +missing optional-plan 404. The UI explicitly treats this as null. Both exact task +paths were independently checked, and the provider deployment, package eligibility, +serving identity, removal of the predecessor, browser timing and continuous probe +were revalidated. The failed original receipt remains unchanged alongside an +explicit classification review. The remaining seven replacements continue without +redeploying or retrying deployment three. An announcement also obscured one +sidebar click; its ordinary Dismiss control resolved that retained interaction +timeout. A separate large run-log diagnostic timed out; a narrower public-events +read succeeded and proves the original command's single execution identity. +A later bounded 64-KiB log read succeeded from shared storage after both the +originating webserver and remote sandbox were removed. + +The native recovery event records SIGTERM without a retry run, then adoption of +the same runner at controller generation 2. Each of the two runs has exactly one +accepted result and terminal event. The independent accounting inspector has no +findings or pending work; the task totals 52.941 cents across two runs. Including +the earlier no-replacement attempt, staging has three estimated cost events totaling +113.84284 cents. The temporary agent is paused, its secret removed and the prior +company budget restored; all three provider resources are independently confirmed absent. The first delete +was refused while the sandbox stopped; that failure remains alongside the +successful cleanup after a fresh stopped-state observation. Both owned local +PostgreSQL/object-storage containers are removed; test evidence is retained. + +Deployment nine also completed, but the local test reader observed the browser +snapshot while its writer had truncated the file. The campaign response was +already parsed and successful; the only non-atomic JSON read in that failing +checkpoint is the local browser report. A concurrent-write regression reproduced +the problem. The reader now retries only local JSON syntax errors for at most +100 ms; persistent corruption still fails, and HTTP/acceptance failures are never +retried. Deployment nine was independently revalidated without redeployment. +Both raw failed sequence receipts and their explicit classification reviews remain; +the ten provider deployments themselves have not been retried. The final staging +driver stopped at the harness failure and is being resumed from the retained +sequence. The normal UI's 256,000-byte run-log read also passes after replacement. + +The first repair check stopped before mutation because an unrelated staging +campaign was active. That campaign has finished, and the pinned canary remains +active and unlocked on the qualified final image. Both one-target configuration repair campaigns passed: backup policy was +restored, the temporary marker removed, and durable files/shared assets remained +enabled on the same volume-free image. The current-image eight-minute natural-idle check failed with all browser pages and +traffic probes closed. Bounded SQL diagnosis found three completed native runs +still in agent-file collection recovery. Normal archive of both completed test +workspaces independently passed close-readiness and preserved saved workspace +files. It produced provider-confirmed destruction receipts for the latest leases. +Two copies then completed no-save cleanup, but retained stale retry deadlines; +the earlier run of a reused sandbox still lacks direct termination evidence even +though its validated successor allocation is destroyed. These are source recovery +gaps, not an idle-proof pass. No historical rows were manually rewritten. The +failed idle and cancelled browser lifecycle receipts remain unchanged. Fix +ownership belongs to remote termination proof resolution and agent-directory +cleanup scheduling; no polling cadence or sleep-safety check should be relaxed. + +The isolated source branch is now based on current master `97c4dc969e`. The +recovery fix resolves the destroyed allocation through the existing validated +handoff-chain contract at read time; it does not stamp historical leases or infer +stop from task status. Agent-file cleanup also reconciles stale retry deadlines +with a receipt comparison. Both owning service suites passed 169 tests, and the +server typecheck passed. The first new deadline test caught a public guard that +skipped settlement; that failed attempt is retained and the corrected five-case +selection passed. The final seven-case selection passed, including a concurrent receipt change and +cross-company rejection. Repository-wide checks and PR review are pending. +This source has not been deployed and is not covered by the prior ten-roll proof. + +Next: review and ship the cleanup fix, complete current-image idle/sleep/wake, then the +isolated production forward/rollback canary. No production deployment has occurred. +Dotta completed the authorized GitHub sign-in; the production browser is back on +the ordinary organization list. The production canary still waits for staging. Its new organization form is +prepared but unsubmitted. Production remains on its original controller revision, +with no campaigns, the single-volume default and continuous deployment disabled. Earlier +staging sleep/wake and natural database-idle proofs remain valid for their image. Core #18543 merged as `7751fc3107` after 55 successful CI checks, Greptile 5/5 and no unresolved threads. It supplies runtime admission, immutable-cache @@ -202,7 +393,7 @@ Measure one-time setup separately from repeated attempts. Keep failed evidence. | Item | State | Next action | | --- | --- | --- | | Prior prerequisites | #18403 and all five storage workstreams merged | Retain the ephemeral filesystem contract | -| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 reviewed at 5/5 with CI pending; local packaged migration, browser overlap, cold restoration and durable command proofs recorded below | Publish and qualify the exact fixed release after CI/review merge | +| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 merged with all required checks and 5/5 reviews; both exact fixed images passed packaged migration and deletion; active hosted output verified | Finish hosted continuation and accounting reconciliation | | Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged; #1102 serving on staging | Verify repeated Railway overlap behavior after Core recovery qualification | | Staging | Original enrollment complete; migrated data and signing key preserved; volume detached and retained | Sleep/wake and natural database idle pass; verify repair and ten app-only replacements with browser continuity | | Production | No canary mutation performed | Proceed only after the staging and active-agent acceptance gates pass | diff --git a/maps/agent-access-map/chat-channels.md b/maps/agent-access-map/chat-channels.md index 16413dd1eff..da6e5478a37 100644 --- a/maps/agent-access-map/chat-channels.md +++ b/maps/agent-access-map/chat-channels.md @@ -1028,6 +1028,13 @@ receipt preserves the original skipped receipt and older failed native result. Current access and normal native stop/cleanup guards still govern execution; board wake metadata alone cannot manufacture chat authority or replay old work. +After a successful connected run, board Retry can resume a newer deferred inbound +receipt through the original chat owner. It rechecks the board's current task +permission and the saved source's current authority, and waits for prior cleanup. +It grants no permission to replay the successful result or manufacture a chat +request. Concurrent clicks share the canonical retry receipt; completed tasks +and superseding execution retire unadmitted retry intent. + Deferred chat Retry excludes independent board queues from its pinned batch. An explicit board Retry can restore a historical mixed batch only with exact never-started evidence, current source authority, unchanged board comments and diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index a16aef4da4d..53d9400f4de 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -214,7 +214,9 @@ use the server-owned authenticated relay. The provider sees bounded then uses pinned aliases. Search describes its 1–20 page limit and default of 5 in the tool and parameter descriptions. It intersects fresh grants and work modes; calls retain the run's original gateway policy, validation, approvals, revocation and -audit path. Prompt arguments cannot select gateway authority. Assigned gateway +audit path. Blank schema references perform normal discovery; positive schema +offsets require an exact reference, and unavailable references explain how to +search again. Prompt arguments cannot select gateway authority. Assigned gateway launch credentials stay on the controller. Former direct-catalog OpenCode checkpoints refresh; other provider fingerprints stay unchanged. The [projection controls](../../server/src/services/native-runtime/assigned-mcp-tools.test.ts) diff --git a/maps/agent-access-map/tasks.md b/maps/agent-access-map/tasks.md index 7f80a1314c0..331a2694955 100644 --- a/maps/agent-access-map/tasks.md +++ b/maps/agent-access-map/tasks.md @@ -74,7 +74,7 @@ API: `present` — GET /api/companies/{companyId}/issues; optional view=compact. Agent access: `partial` — REST/call_api full query; CLI issue list exposes status/agent/project and local --match on one fetched page; native search_tasks query up to 500 characters, statuses, limit 1..200 (default 50), and a nextCursor in ascending task-ID order. Responses expose effective limit, returnedCount and hasMore. Exhaust nextCursor with the same filters before treating an inventory as complete. Each page checks current access; concurrent task changes require a fresh scan. Project/label/workspace knobs remain REST-only. -Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|id, sortDir asc|desc, afterId. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results. +Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|created|id, sortDir asc|desc, afterId. Blocked attention lists reject created and id ordering with 422. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results. Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [issues.ts](../../server/src/routes/issues.ts), [issues.ts](../../server/src/services/issues.ts), [paperclip-runner-tool-authority.ts](../../server/src/services/native-runtime/paperclip-runner-tool-authority.ts), [issue.ts](../../cli/src/commands/client/issue.ts), [issues-list-query-parsing.test.ts](../../server/src/__tests__/issues-list-query-parsing.test.ts), [issues-creation-visibility.test.ts](../../server/src/__tests__/issues-creation-visibility.test.ts), [issue-list-assignee-filter-routes.test.ts](../../server/src/__tests__/issue-list-assignee-filter-routes.test.ts), [private-json-etag.test.ts](../../server/src/__tests__/private-json-etag.test.ts). No live execution claimed. @@ -120,7 +120,10 @@ Agent access: `not-applicable` — no agent layout API; permitted underlying tas Policy: IssuesList loads/saves browser preferences by company/collection, with separate agent-created visibility. The Tasks page also owns a routine-only checkbox and `originKind=routine_execution` URL state; it uses the existing -company-scoped task-list origin filter before pagination. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred. +company-scoped task-list origin filter before pagination. Sort controls write +`sortField` and `sortDir` to the URL; explicit URL sorting overrides browser +preferences. `created` sorting runs on the server before pagination, with ID as +a stable tie-breaker. The routines task link requests `created` descending. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred. Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [IssuesList.tsx](../../ui/src/components/IssuesList.tsx), [Issues.tsx](../../ui/src/pages/Issues.tsx), [Issues.test.tsx](../../ui/src/pages/Issues.test.tsx). No live execution claimed. @@ -522,3 +525,9 @@ Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7 Verification: `not-run` — Compare eligible owner/admin ordinary denied with explicit read/audit+notice; ordinary operator/agent/foreign denied, grants unchanged. Gaps: Local implicit/instance admin bypass cannot establish ordinary two-user private isolation. + +Task reads expose optional read-only `displayStatus`. `monitoring` describes a +scheduled ordinary monitor wait; it is not accepted as a writable task `status`. +Existing lifecycle `status` filters retain their meaning. Board collection and +search readers can use the separate `displayStatus` filter before pagination. +Agents keep using the existing monitor and semantic completion tools. diff --git a/maps/agent-access-map/workspaces.md b/maps/agent-access-map/workspaces.md index c2b261a7f4f..cc71214ed52 100644 --- a/maps/agent-access-map/workspaces.md +++ b/maps/agent-access-map/workspaces.md @@ -74,6 +74,9 @@ without a controller checkout fallback. Current authorization and responsible-us checks still apply; read-only cross-project referenced folders and explicitly concurrent shared writers are rejected. Managed mutable same-project repositories are supported. +Portable capture streams validated working files while staging self-contained +Git metadata. The smaller temporary-space requirement grants no new agent access +or execution authority. Changed sources still fail before a save receipt. An inherited isolated workspace or operator branch can be reserved through the task's current persisted binding. The original source task remains provenance. Reservation checks task/run, binding, trust, and retained source visibility before diff --git a/maps/feature-map/chat-channels.md b/maps/feature-map/chat-channels.md index 45802a29aeb..7eb26bbc117 100644 --- a/maps/feature-map/chat-channels.md +++ b/maps/feature-map/chat-channels.md @@ -278,6 +278,14 @@ After successful admission, reload and verify that the prior skipped receipt doe not strand a later real provider reply or tree resume. Provider fixtures support this behavior; live fixed-source runtime and destination proof remain separate. +Also complete an older review successfully while a newer authorized PR event +remains deferred. Use Retry and verify that the existing saved-input receipt +starts exactly one new turn, repeated clicks share that turn, and the older +accepted result remains unchanged. Revoke the connection or change the source +binding before dispatch and verify no run starts. Keep prior cleanup pending, +then settle it and verify the same receipt resumes once. These database-backed +provider fixtures do not establish live GitHub publication at the fixed revision. + Also defer Retry behind cleanup, queue an independent board message, then complete cleanup. Verify one run receives the exact GitHub batch and the board message enters its own turn. For an older mixed batch rejected before provider diff --git a/maps/feature-map/inbox-search.md b/maps/feature-map/inbox-search.md index e6904ca8d2e..eb11c0d5f85 100644 --- a/maps/feature-map/inbox-search.md +++ b/maps/feature-map/inbox-search.md @@ -91,7 +91,11 @@ Cold-open verification: load Mine with the task-index response delayed while approvals/runs resolve. Only the stable list placeholder should appear until the initial sources settle; a failed-run row must not jump when tasks arrive. Release visible-row details separately and compare row/title bounds. Mine reads its own -100-row index, with parent/run context by ID; sidebar badges use the count endpoint. +100-row index. Missing parents use the same index projection for their captions; +parents already in the index are reused. Failed-run tasks still load rich row +context by ID, including when they are also parents or already indexed, so resolved +interruptions stay hidden. The initial reveal waits for both context sources. +Sidebar badges use the count endpoint. Coordinator desktop and mobile navigation share `/attention/summary` for the outstanding count. Needs You owns its complete feed; Tasks → Needs attention owns a compact complete task-ID index. Navigation reuses either page response. Coordinator keeps the optimized Inbox even when diff --git a/maps/feature-map/routines.md b/maps/feature-map/routines.md index 75f8e21a926..bb830f726a8 100644 --- a/maps/feature-map/routines.md +++ b/maps/feature-map/routines.md @@ -30,7 +30,7 @@ Open `/routines`, create one, and inspect `/routines/:routineId` and its edit se Use the routine's trigger controls and Runs/Recent Runs/Audit links; lists also offer Run now. **View all tasks** opens the Tasks list with **Routine tasks** -selected, using `/issues?view=all&originKind=routine_execution`. Uncheck the +selected, using `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc`. Uncheck the filter to return to other tasks. **History** records configuration versions, not executions. ## Driving it with Playwright and the Paperclip CLI diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index 11aa505a521..b728b8bac04 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -273,6 +273,10 @@ For native OpenCode with assigned connections, inspect the provider tool surface on both local and sandbox execution. It should contain bounded assigned-tool search/call schemas. Confirm the search description states the 1–20 page limit and default of 5. Follow `nextOffset` to inspect later pages. +Repeat discovery with empty and whitespace-only `schemaTool` values, including +`schemaOffset: 0`, and confirm it still returns searchable tools. An unknown +schema reference or a positive schema offset without a reference must explain +how to correct the call. Discover an allowed tool, call its returned alias, revoke its grant, and confirm discovery and execution enforce that revocation. Repeat in planning/ask mode with a write tool and confirm denial. The controller must diff --git a/maps/feature-map/tasks.md b/maps/feature-map/tasks.md index a9afa296295..934b1d50ce9 100644 --- a/maps/feature-map/tasks.md +++ b/maps/feature-map/tasks.md @@ -56,9 +56,17 @@ Use New task from the global launcher, task list, or project task view. ### `task-list` +Scheduled ordinary monitor waits display **Monitoring** with a blue clock in +current task indicators, groups, board columns, and filters. Compare Monitoring +with In Progress, then start the check and verify the status returns to its +execution presentation. Retained monitors must not override blocked/review or +terminal states. Inspect due/overdue copy separately; no runner is implied. + Open `/issues` or a project’s Issues view; legacy `/tasks` links redirect. Filters → **Routine tasks** selects tasks created by routine runs. The -`/issues?view=all&originKind=routine_execution` link opens that selection. +`/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` link opens that selection with newest-created tasks first. Sort controls update +`sortField` and `sortDir` in the URL. Reloads and task return links keep that order. +Shared sort links also retain their order when opened in another task interface. ### `task-detail` diff --git a/maps/feature-map/workspaces.md b/maps/feature-map/workspaces.md index 83d5504ff63..bd4f98cfede 100644 --- a/maps/feature-map/workspaces.md +++ b/maps/feature-map/workspaces.md @@ -83,6 +83,14 @@ or original Git remote. Read-only cross-project referenced folders must fail admission. Separate workspaces may run concurrently; shared concurrent writers must be rejected. +Repeat remote capture with enough free space for the archive and normalized Git +metadata, but not another working-tree copy. Verify saved files and Git state +after restore. A concurrent file change must fail capture without publishing a +receipt or retaining a partial archive. The +[portable archive suite](../../server/src/services/remote-workspace-snapshot-archive.test.ts) +checks the space allowance and concurrent-change boundary; its filesystem quota +is controlled test input, not a live sandbox qualification. + Create a child that inherits a saved isolated workspace or operator branch. Verify it continues the same files after the prior writer saves, including on a new allocation. Two bound children must not acquire concurrent writer authority. diff --git a/packages/adapter-utils/src/acpx-engine/startup-timing.ts b/packages/adapter-utils/src/acpx-engine/startup-timing.ts index 74f1977c5c3..5ce26855ef2 100644 --- a/packages/adapter-utils/src/acpx-engine/startup-timing.ts +++ b/packages/adapter-utils/src/acpx-engine/startup-timing.ts @@ -67,6 +67,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = { * command, a path, an argument, or an environment value can never ride a span. */ export const SANDBOX_STARTUP_SPAN_ATTRS = { + /** Provider state before a lease start or recovery; the host clamps values. */ + state: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}state`, /** The low-cardinality provider family (through `normalizeProviderFamily`). */ provider: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}provider`, /** The step or execution outcome: `ok`, `skipped`, or `failed`. */ diff --git a/packages/adapter-utils/src/command-redaction.test.ts b/packages/adapter-utils/src/command-redaction.test.ts index 321866bf7c7..2afd383a6d9 100644 --- a/packages/adapter-utils/src/command-redaction.test.ts +++ b/packages/adapter-utils/src/command-redaction.test.ts @@ -145,3 +145,42 @@ second-line\" status=401`; expect(output).toContain(REDACTED_COMMAND_TEXT_VALUE); }); }); + +describe("redactCommandText on long runs", () => { + const jwt = + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"; + + function redactWithinTimeBudget(input: string) { + const startedAt = performance.now(); + const output = redactCommandText(input); + expect(performance.now() - startedAt).toBeLessThan(1_000); + return output; + } + + it("keeps redacting options, assignments, and tokens inside dash-separated runs", () => { + const cases = [ + ["x--token flag-secret", `x--token ${REDACTED_COMMAND_TEXT_VALUE}`], + ["--x--token flag-secret", `--x--token ${REDACTED_COMMAND_TEXT_VALUE}`], + [ + "a-b--api-key=assigned-secret --password 'quoted-secret'", + `a-b--api-key=${REDACTED_COMMAND_TEXT_VALUE} --password '${REDACTED_COMMAND_TEXT_VALUE}'`, + ], + ["prefix-MY_API_KEY=env-secret", `prefix-MY_API_KEY=${REDACTED_COMMAND_TEXT_VALUE}`], + [`--${jwt}`, `--${REDACTED_COMMAND_TEXT_VALUE}`], + [`session=${jwt}`, `session=${REDACTED_COMMAND_TEXT_VALUE}`], + ["my-token is here", "my-token is here"], + ]; + + for (const [input, expected] of cases) { + expect(redactCommandText(input)).toBe(expected); + } + }); + + it("scans long letter/digit and dash-separated runs in linear time", () => { + const base64Url = Buffer.from(Array.from({ length: 90_000 }, (_, index) => (index * 7919) % 256)).toString("base64url"); + for (const run of ["a".repeat(120_000), "a-".repeat(60_000), "a--".repeat(40_000), base64Url]) { + expect(redactWithinTimeBudget(`${run} --token run-secret`)).toBe(`${run} --token ${REDACTED_COMMAND_TEXT_VALUE}`); + expect(redactWithinTimeBudget(`${run}--token run-secret`)).toBe(`${run}--token ${REDACTED_COMMAND_TEXT_VALUE}`); + } + }); +}); diff --git a/packages/adapter-utils/src/command-redaction.ts b/packages/adapter-utils/src/command-redaction.ts index 46eeb6f5040..451f0ccf58e 100644 --- a/packages/adapter-utils/src/command-redaction.ts +++ b/packages/adapter-utils/src/command-redaction.ts @@ -11,16 +11,19 @@ export function isPublicExecutorToolSelector(value: string): boolean { return PUBLIC_EXECUTOR_TOOL_SELECTORS.has(value); } -const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_NAME_WORDS = String.raw`(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*${SECRET_NAME_WORDS}`; +const SECRET_NAME_START = String.raw`(? { expect(parseSetupTokenPrompt(completeOutput(withKey))).toBeNull(); }); + it("skips the 2.1.291 copy hint between the URL and the prompt", () => { + const hint = chaSpaced("Hold Shift while selecting to use your terminal's native copy".split(" ")); + const text = [PREAMBLE_LINE, hardWrap(CLAUDE_COM_URL, 130), "", hint, "", "", PROMPT_LINE].join("\n"); + const result = parseSetupTokenPrompt(text); + expect(result?.url).toBe(CLAUDE_COM_URL); + expect(result?.prompt).toBe(SETUP_TOKEN_PROMPT); + }); + it("returns null when a code-like value sits on the line after the URL", () => { // The line right after the URL must hold the browser-code prompt. A code-like // value on that line cannot bind, so the parser returns null. diff --git a/packages/adapters/claude-local/src/server/setup-token-parse.ts b/packages/adapters/claude-local/src/server/setup-token-parse.ts index d845f5ab348..6028695d16a 100644 --- a/packages/adapters/claude-local/src/server/setup-token-parse.ts +++ b/packages/adapters/claude-local/src/server/setup-token-parse.ts @@ -181,6 +181,10 @@ const URL_PREAMBLE_RE = /the url below to sign in/i; // the line. const PROMPT_LINE_RE = /^Paste code here if prompted\b[\s>*]*$/; +// The copy hint that `claude` 2.1.291 prints between the URL block and the +// prompt. The parser skips this exact line and binds no value from it. +const COPY_HINT_LINE_RE = /^Hold\s+Shift\s+while\s+selecting\s+to\s+use\s+your\s+terminal['’]s\s+native\s+copy\.?$/i; + // The maximum number of characters between the end of the URL preamble line and // the start of the URL. The Claude UI prints the URL right after the preamble. // The parser accepts a URL start only inside this window, so a URL far from the @@ -363,9 +367,11 @@ function reassembleUrl(lines: string[], startLine: number): SetupTokenUrlMatch | * line after the URL block. The parser reads the non-blank lines after the URL * end line. The Claude UI can emit one full URL line per wrapped display row, so * it repeats the same full authorization URL on a few consecutive lines. The - * parser skips a line that repeats `url`, then binds the prompt on the first - * non-blank, non-repeat line. That line must match the exact prompt shape. - * Returns null when the first non-blank, non-repeat line is not the prompt. + * parser skips a line that repeats `url`. The `claude` 2.1.291 UI also prints a + * copy hint before the prompt, so the parser skips a line that matches {@link + * COPY_HINT_LINE_RE}. The parser then binds the prompt on the first non-blank + * line that is not a repeat and not the copy hint. That line must match the + * exact prompt shape. Returns null when that line is not the prompt. */ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): string | null { for (let i = endLine + 1; i < lines.length; i += 1) { @@ -375,6 +381,7 @@ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): s // punctuation first, the same way the repeated-URL comparison needs, so a // repeated URL with a trailing punctuation mark still matches. if (trimmed.replace(TRAILING_PUNCTUATION_RE, "") === url) continue; + if (COPY_HINT_LINE_RE.test(trimmed)) continue; return PROMPT_LINE_RE.test(trimmed) ? SETUP_TOKEN_PROMPT : null; } return null; diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 53abe16e078..6cab39406a0 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -50,3 +50,4 @@ export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; +export { withDatabaseOperation } from "./response-capture.js"; diff --git a/packages/db/src/response-capture-postgres.test.ts b/packages/db/src/response-capture-postgres.test.ts index 42d5d805a0c..c737e455c9b 100644 --- a/packages/db/src/response-capture-postgres.test.ts +++ b/packages/db/src/response-capture-postgres.test.ts @@ -4,9 +4,9 @@ import { eq, sql } from "drizzle-orm"; import { pgTable, text } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; -import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture } from "./response-capture.js"; +import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, withDatabaseOperation } from "./response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { encodeRunEventPayload, runEventPayload } from "./run-event-payload.js"; +import { runEventPayload } from "./run-event-payload.js"; import { databaseWorkPending, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const table = pgTable("response_capture_fixture", { id: text("id").primaryKey(), payload: text("payload").notNull() }); @@ -31,17 +31,17 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi vi.restoreAllMocks(); vi.unstubAllEnvs(); }); afterAll(async () => { await temporary?.cleanup(); }); - function fixture(prepare?: boolean) { + function fixture(prepare?: boolean, maxConnections?: number, clock?: () => number) { const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); - const db = createDb(temporary.connectionString, { prepare }); + const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), clock); + const db = createDb(temporary.connectionString, { prepare, maxConnections }); clients.push(db); captures.push(capture); expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db); expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db); return { db, capture, events }; } - function summary(events: Array>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; } + function summary(events: Array>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v2")!; } it("retains capture counts when ORM preparation is enabled", async () => { const f = fixture(true), id = randomUUID(), payload = "prepared-capture-value"; @@ -68,12 +68,12 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await f.db.delete(table).where(eq(table.id, id)).returning()).toEqual([{ id, payload: "changed" }]); f.capture.flush(); expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4 }); - expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1").map(event => event.operation).sort()) + expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2").map(event => event.operation).sort()) .toEqual(["delete", "insert", "select", "update"]); expect(JSON.stringify(f.events)).not.toContain(payload); expect(JSON.stringify(f.events)).not.toContain(id); }); - it("counts the stored encoding before a custom column decoder changes the value", async () => { + it("skips complex stored values before a custom column decoder changes them", async () => { const setup = createDb(temporary.connectionString); clients.push(setup); const id = randomUUID(), payload = { text: "codec-payload-sentinel\u0000value".repeat(512) }; await setup.insert(encodedTable).values({ id, payload }); @@ -81,8 +81,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await f.db.select().from(encodedTable).where(eq(encodedTable.id, id))).toEqual([{ id, payload }]); f.capture.flush(); const entry = f.events.find(event => event.table === "response_capture_encoded_fixture")!; - expect(entry.responseValueBytes).toBe(Buffer.byteLength(id) + Buffer.byteLength(encodeRunEventPayload(payload))); - expect(entry.responseValueBytes).toBeGreaterThan(Buffer.byteLength(JSON.stringify({ id, payload }))); + expect(entry).toMatchObject({ sampledValueBytes: Buffer.byteLength(id), measuredValues: 1, unmeasuredValues: 1, incompleteSamples: 1 }); expect(JSON.stringify(f.events)).not.toContain("codec-payload-sentinel"); }); it("keeps a raw driver query lazy and counts repeated awaits without replay", async () => { @@ -109,6 +108,69 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(summary(f.events)).toMatchObject({ startedQueries: 3, completedQueries: 3 }); expect(JSON.stringify(f.events)).not.toContain("prepared-value"); }); + it("attributes concurrent prepared reuse, pooled queries, and transaction errors to their execution owners", async () => { + const setup = createDb(temporary.connectionString); clients.push(setup); + const id = randomUUID(); + await setup.insert(table).values({ id, payload: "owner-private-value" }); + const f = fixture(true, 1); + const prepared = withDatabaseOperation("builder.only", () => + f.db.select().from(table).where(eq(table.id, sql.placeholder("id"))).prepare("owner_reused")); + await Promise.all([ + withDatabaseOperation("response.presentation", async () => { + expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]); + await f.db.transaction(async tx => { + expect(await tx.select().from(table).where(eq(table.id, id))).toEqual([{ id, payload: "owner-private-value" }]); + }); + }), + withDatabaseOperation("budget.admission", async () => { + expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]); + await expect(f.db.execute(sql`select 1/0`)).rejects.toThrow(); + }), + ]); + f.capture.flush(); + const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); + const counts = (owner: string) => entries.filter(event => event.owner === owner).reduce((n, event) => n + Number(event.calls), 0); + expect(counts("response.presentation")).toBe(2); + expect(counts("budget.admission")).toBe(2); + expect(counts("builder.only")).toBe(0); + expect(entries.every(event => (event.callers as string[]).length === 0)).toBe(true); + expect(entries.find(event => event.queryErrors === 1)).toMatchObject({ owner: "budget.admission", rows: 0, sampledResults: 0 }); + expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4, rows: 3 }); + expect(JSON.stringify(f.events)).not.toContain("owner-private-value"); + }); + it("attributes a lazy raw query to its first consumer rather than its builder or repeated await", async () => { + const f = fixture(); + const query = withDatabaseOperation("builder.only", () => f.db.$client.unsafe("select 42 as answer")); + const result = await withDatabaseOperation("consumer.first", async () => await query); + expect(result).toEqual([{ answer: 42 }]); + expect(await withDatabaseOperation("consumer.again", async () => await query)).toBe(result); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "consumer.first", calls: 1, rows: 1 }); + expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + }); + it("samples bounded driver values while retaining every query and row", async () => { + const f = fixture(); + await withDatabaseOperation("sampling.owner", async () => { + for (let index = 0; index < 33; index += 1) { + expect(await f.db.execute(sql`select 42 as answer`)).toEqual([{ answer: 42 }]); + } + }); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "sampling.owner", calls: 33, rows: 33, sampledResults: 3, + sampledRows: 3, sampledValueBytes: 6, measuredValues: 3, unmeasuredValues: 0 }); + expect(summary(f.events)).toMatchObject({ completedQueries: 33, rows: 33, sampledResults: 3, sampledValueBytes: 6 }); + }); + it("leaves a real in-flight query running when the capture deadline passes", async () => { + let clock = Date.now(); + const f = fixture(undefined, 1, () => clock); + const pending = withDatabaseOperation("ending.owner", async () => await f.db.execute(sql`select pg_sleep(0.05), 42 as answer`)); + await new Promise(resolve => setImmediate(resolve)); + clock += 60_000; + f.capture.flush(); + expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1 }); + expect((await pending)[0].answer).toBe(42); + expect(f.events).toHaveLength(1); + }); it("retains distinct builder callers through normal, prepared, and transaction execution", async () => { const f = fixture(), id = randomUUID(); const setup = createDb(temporary.connectionString); clients.push(setup); @@ -128,7 +190,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await fromB(tx, id)).toEqual([{ id, payload: "caller-sentinel" }]); }); f.capture.flush(); - const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); expect(entries).toHaveLength(2); for (const name of ["a", "b"]) { expect(entries.find(event => (event.callers as string[]).some(location => location.startsWith(`services/capture-caller-${name}.ts:`)))) @@ -191,13 +253,14 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi vi.stubEnv("PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW", `${new Date(now - 100).toISOString()}/${new Date(now + 59_000).toISOString()}`); const db = createDb(temporary.connectionString, { prepare }); clients.push(db); const capture = databaseResponseCaptureFromEnv()!; captures.push(capture); - expect(await withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`))) + expect(await withDatabaseOperation("dedicated.owner", () => withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`)))) .toEqual([{ value: "dedicated-value" }]); capture.flush(); - await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v1")).toBe(true), + await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v2")).toBe(true), { timeout: 1_000, interval: 5 }); const events = logs.map(line => JSON.parse(line) as Record); expect(summary(events)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + expect(events.find(event => event.event === "paperclip.db_response_capture.entry.v2")).toMatchObject({ owner: "dedicated.owner" }); expect(logs.join("\n")).not.toContain("dedicated-value"); expect(logs.join("\n")).not.toContain(temporary.connectionString); }); diff --git a/packages/db/src/response-capture.test.ts b/packages/db/src/response-capture.test.ts index 73d25d8fc3c..f22a5021c17 100644 --- a/packages/db/src/response-capture.test.ts +++ b/packages/db/src/response-capture.test.ts @@ -1,6 +1,7 @@ +import { AsyncLocalStorage } from "node:async_hooks"; import { integer, pgTable } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow } from "./response-capture.js"; +import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow, withDatabaseOperation } from "./response-capture.js"; const epoch = Date.parse("2020-01-01T00:00:00Z"); const metadata = { operation: "select", table: "capture_fixture", callers: ["services/fixture.ts:4:2"] }; @@ -14,6 +15,9 @@ function fixture(emitOverride?: (event: Record) => void) { captures.push(capture); return { capture, events, now: (value: number) => { clock = value; } }; } +function driverRows(rows: unknown[][]) { + return Object.assign(rows, { columns: Array.from({ length: rows[0]?.length ?? 0 }, (_, index) => ({ name: `column_${index}` })) }); +} describe("bounded database response capture", () => { it("paces a complete environment capture without extending the measurement window or leaving timers", () => { @@ -36,13 +40,13 @@ describe("bounded database response capture", () => { capture.flush(); vi.advanceTimersByTime(32_000); expect(emitted).toHaveLength(2049); - const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v1"); + const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v2"); const summary = emitted.at(-1)!.event; expect(groups).toHaveLength(2048); - expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v1", completedQueries: 2200, + expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2200, groups: 2048, closedAt: "2020-01-01T00:01:00.000Z" }); expect(groups.reduce((calls, item) => calls + Number(item.event.calls), 0)).toBe(summary.completedQueries); - expect(groups.reduce((bytes, item) => bytes + Number(item.event.responseValueBytes), 0)).toBe(summary.responseValueBytes); + expect(groups.reduce((bytes, item) => bytes + Number(item.event.sampledValueBytes), 0)).toBe(summary.sampledValueBytes); expect(groups.map(item => item.event.groupIndex)).toEqual(Array.from({ length: 2048 }, (_, index) => index)); for (const { at } of emitted) expect(emitted.filter(item => item.at >= at && item.at < at + 1000).length).toBeLessThanOrEqual(64); expect(JSON.stringify(emitted)).not.toContain("capture-response-sentinel"); @@ -89,13 +93,13 @@ describe("bounded database response capture", () => { expect(installDatabaseResponseCapture(db, { table })).toBe(db); expect(db.select).toBe(select); expect(db.transaction).toBe(transaction); }); - it("preserves exact values and only emits counts, sizes, and caller metadata", async () => { - const f = fixture(), rows = [{ text: "sentinel-result-value🙂\n\"", n: 4n }]; + it("preserves exact values and only emits counts, sampled sizes, and caller metadata", async () => { + const f = fixture(), text = "sentinel-result-value🙂\n\"", rows = driverRows([[text, 4n]]); expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows); f.capture.flush(); - const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v1")!; - expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, serializationErrors: 0, - responseValueBytes: Buffer.byteLength(JSON.stringify(rows, (_, value) => typeof value === "bigint" ? value.toString() : value)) }); + const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v2")!; + expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, owner: "unattributed", sampledResults: 1, + sampledValueBytes: Buffer.byteLength(text) + 1, measuredValues: 2, unmeasuredValues: 0, incompleteSamples: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-result-value"); expect(f.events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1, inFlightAtEnd: 0, rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false }); @@ -106,10 +110,91 @@ describe("bounded database response capture", () => { { name: "sentinel-column-name" }, { name: "b" }, { name: "n" }, { name: "empty" } ] }); expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows); f.capture.flush(); - expect(f.events[0]).toMatchObject({ responseValueBytes: Buffer.byteLength(text) + 1 + 2, estimatedValues: 0 }); + expect(f.events[0]).toMatchObject({ sampledValueBytes: Buffer.byteLength(text) + 1 + 2, measuredValues: 4, unmeasuredValues: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-driver-text"); expect(JSON.stringify(f.events)).not.toContain("sentinel-column-name"); }); + it("counts every result but sizes only the first and every sixteenth successful result per group", () => { + const f = fixture(); + for (let index = 0; index < 33; index += 1) { + const rows = driverRows([["abc", 12]]); + if (index % 16 !== 0) Object.defineProperty(rows[0], 0, { get() { throw new Error("unsampled cell read"); } }); + expect(withDatabaseOperation("fixture.owner", () => f.capture.execute(() => rows, metadata))).toBe(rows); + } + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "fixture.owner", calls: 33, rows: 33, sampledResults: 3, + sampledRows: 3, sampledValueBytes: 15, measuredValues: 6, unmeasuredValues: 0, incompleteSamples: 0 }); + expect(f.events[0].totalWallMs).toBeGreaterThanOrEqual(0); + expect(f.events[0].maxWallMs).toBeLessThanOrEqual(Number(f.events[0].totalWallMs)); + }); + it("bounds sampled cells, string scanning, and bytes without traversing large or complex values", () => { + const f = fixture(); + const rows = driverRows([["x".repeat(1_000_000), { large: "private-json" }, ...Array.from({ length: 300 }, () => "y".repeat(16_384))]]); + Object.defineProperty(rows[0], 256, { get() { throw new Error("cell budget exceeded"); } }); + expect(f.capture.execute(() => rows, metadata)).toBe(rows); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ sampledResults: 1, sampledValueBytes: 65_536, measuredValues: 4, + unmeasuredValues: 298, incompleteSamples: 1 }); + expect(JSON.stringify(f.events)).not.toContain("private-json"); + const cellBound = fixture(), cells = driverRows([Array.from({ length: 257 }, () => null)]); + Object.defineProperty(cells[0], 256, { get() { throw new Error("cell budget exceeded"); } }); + cellBound.capture.execute(() => cells, metadata); + cellBound.capture.flush(); + expect(cellBound.events[0]).toMatchObject({ sampledValueBytes: 0, measuredValues: 256, unmeasuredValues: 1, incompleteSamples: 1 }); + }); + it("restores nested owners and uses the direct path without an active capture", () => { + const returned = Promise.resolve("unchanged"); + expect(withDatabaseOperation("disabled", () => returned)).toBe(returned); + const f = fixture(); + withDatabaseOperation("outer", () => { + withDatabaseOperation("inner", () => f.capture.execute(() => driverRows([[1]]), metadata)); + f.capture.execute(() => driverRows([[2]]), metadata); + }); + f.now(epoch + 60_001); + withDatabaseOperation("expired", () => { + const next = fixture(); + next.capture.execute(() => driverRows([[3]]), metadata); + next.capture.flush(); + expect(next.events[0]).toMatchObject({ owner: "unattributed" }); + }); + f.capture.flush(); + expect(f.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["inner", "outer"]); + }); + it("retains an armed operation across the capture start without counting earlier queries", async () => { + let clock = epoch; + const events: Array> = []; + const capture = createDatabaseResponseCapture({ start: epoch + 1_000, end: epoch + 60_000 }, event => events.push(event), () => clock); + captures.push(capture); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const pending = withDatabaseOperation("started.before.window", async () => { + capture.execute(() => driverRows([[1]]), metadata); + await gate; + return capture.execute(() => driverRows([[2]]), metadata); + }); + clock = epoch + 2_000; + release(); + await pending; + capture.flush(); + expect(events[0]).toMatchObject({ owner: "started.before.window", calls: 1, rows: 1 }); + expect(events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + }); + it("keeps overlapping capture scopes but clears retained owners when the last capture closes", () => { + const first = fixture(), second = fixture(); + let destination = second.capture; + const retained = withDatabaseOperation("previous.capture", () => AsyncLocalStorage.bind(() => + destination.execute(() => driverRows([[1]]), metadata))); + first.capture.flush(); + retained(); + second.capture.flush(); + expect(second.events[0]).toMatchObject({ owner: "previous.capture", calls: 1 }); + const later = fixture(); + destination = later.capture; + withDatabaseOperation("new.capture", () => destination.execute(() => driverRows([[2]]), metadata)); + retained(); + later.capture.flush(); + expect(later.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["new.capture", "unattributed"]); + }); it("does not alter synchronous or asynchronous errors or retry them", async () => { const f = fixture(), error = new Error("sentinel-private-error"); const sync = vi.fn(() => { throw error; }), asyncFailure = vi.fn(() => Promise.reject(error)); @@ -117,14 +202,17 @@ describe("bounded database response capture", () => { await expect(f.capture.execute(asyncFailure, metadata)).rejects.toBe(error); expect(sync).toHaveBeenCalledTimes(1); expect(asyncFailure).toHaveBeenCalledTimes(1); f.capture.flush(); - expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, responseValueBytes: 0 }); + expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, sampledValueBytes: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-private-error"); }); - it("keeps unserializable results and logging failures out of the query outcome", () => { + it("does not serialize complex values or let logging failures change query outcomes", () => { const f = fixture(), circular: Record = {}; circular.self = circular; - expect(f.capture.execute(() => circular, metadata)).toBe(circular); + const toJSON = vi.fn(() => { throw new Error("must not serialize"); }); + const rows = driverRows([[circular, { toJSON }]]); + expect(f.capture.execute(() => rows, metadata)).toBe(rows); f.capture.flush(); - expect(f.events[0]).toMatchObject({ calls: 1, serializationErrors: 1 }); + expect(f.events[0]).toMatchObject({ calls: 1, sampledValueBytes: 0, unmeasuredValues: 2, incompleteSamples: 1 }); + expect(toJSON).not.toHaveBeenCalled(); const brokenLogger = fixture(() => { throw new Error("logger fixture"); }); expect(brokenLogger.capture.execute(() => circular, metadata)).toBe(circular); expect(() => brokenLogger.capture.flush()).not.toThrow(); @@ -152,12 +240,12 @@ describe("bounded database response capture", () => { expect(await result).toBe(rows); f.capture.flush(); expect(f.events).toHaveLength(1); - expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0 }); + expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, sampledValueBytes: 0 }); expect(JSON.stringify(f.events)).not.toContain("late-before-flush-sentinel"); }); - it("bounds cardinality without dropping response-byte totals", () => { + it("bounds cardinality while distinguishing sampled bytes from complete call and row counts", () => { const f = fixture(); - for (let i = 0; i < 2200; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] }); + for (let i = 0; i < 2200; i += 1) f.capture.execute(() => driverRows([[i]]), { ...metadata, callers: [`services/f${i}.ts:1:1`] }); f.capture.flush(); const summary = f.events.at(-1)!; expect(summary).toMatchObject({ startedQueries: 2200, completedQueries: 2200, groups: 2048, overflowCalls: 153, @@ -169,7 +257,8 @@ describe("bounded database response capture", () => { expect(entries.filter(entry => entry.groupPage === page)).toHaveLength(128); } expect(entries.reduce((sum, entry) => sum + Number(entry.calls), 0)).toBe(2200); - expect(summary.responseValueBytes).toBe(Array.from({ length: 2200 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0)); + expect(summary).toMatchObject({ rows: 2200, sampledResults: 2057, sampleEvery: 16 }); + expect(summary.sampledValueBytes).toBe(entries.reduce((sum, entry) => sum + Number(entry.sampledValueBytes), 0)); }); it.each([0, 129])("numbers every page including an empty or partial final page (%s)", count => { const f = fixture(); diff --git a/packages/db/src/response-capture.ts b/packages/db/src/response-capture.ts index d53b5fe0c07..64e1bcf2448 100644 --- a/packages/db/src/response-capture.ts +++ b/packages/db/src/response-capture.ts @@ -3,11 +3,27 @@ import { randomUUID } from "node:crypto"; import { getTableName, is, Table } from "drizzle-orm"; type Metadata = { operation: string; table: string; callers: string[] }; -type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; queryErrors: number; serializationErrors: number; estimatedValues: number }; +type Entry = Metadata & { owner: string; calls: number; rows: number; queryErrors: number; totalWallMs: number; maxWallMs: number; + sampledResults: number; sampledRows: number; sampledValueBytes: number; maxSampledValueBytes: number; + measuredValues: number; unmeasuredValues: number; incompleteSamples: number; sizingWallMs: number }; type Event = Record; type Method = (this: unknown, ...args: unknown[]) => unknown; export type ResponseCaptureWindow = { start: number; end: number }; const GROUP_PAGE_SIZE = 128; +const SAMPLE_EVERY = 16; +const MAX_SAMPLE_CELLS = 256; +const MAX_SAMPLE_STRING_LENGTH = 16_384; +const MAX_SAMPLE_STRING_UNITS = 65_536; +const MAX_SAMPLE_BYTES = 65_536; +let operationScope = new AsyncLocalStorage(); +const captureWindows = new Set<() => boolean>(); + +export function withDatabaseOperation(owner: string, action: () => T): T { + for (const armed of captureWindows) { + if (armed()) return operationScope.run(owner, action); + } + return action(); +} export function responseCaptureWindow(value: string | undefined, now = Date.now()): ResponseCaptureWindow | undefined { if (!value) return; @@ -32,28 +48,37 @@ export function publicDatabaseCallers(stack: string | undefined): string[] { return callers; } -function responseValueSize(value: unknown): { bytes: number; estimatedValues: number } { +function responseValueSize(value: unknown): { bytes: number; measuredValues: number; unmeasuredValues: number } { const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined; if (!Array.isArray(value) || !Array.isArray(columns)) { - const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); - return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 }; + return { bytes: 0, measuredValues: 0, unmeasuredValues: 1 }; } - let bytes = 0, estimatedValues = 0; + let bytes = 0, measuredValues = 0, visitedValues = 0, stringUnits = 0; + const totalValues = value.length * columns.length; + if (columns.length === 0) return { bytes: 0, measuredValues: 0, unmeasuredValues: 0 }; for (const row of value) { - const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]); - for (const cell of cells) { - if (cell === null || cell === undefined) continue; - if (typeof cell === "string") bytes += Buffer.byteLength(cell); - else if (typeof cell === "boolean") bytes += 1; - else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell)); - else { - estimatedValues += 1; - const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); - if (serialized !== undefined) bytes += Buffer.byteLength(serialized); + for (let index = 0; index < columns.length; index += 1) { + if (visitedValues >= MAX_SAMPLE_CELLS || bytes >= MAX_SAMPLE_BYTES) { + return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues }; } + visitedValues += 1; + const cell = Array.isArray(row) ? row[index] : row[columns[index].name]; + let size: number; + if (cell === null || cell === undefined) size = 0; + else if (typeof cell === "string") { + if (cell.length > MAX_SAMPLE_STRING_LENGTH || stringUnits + cell.length > MAX_SAMPLE_STRING_UNITS) continue; + stringUnits += cell.length; + size = Buffer.byteLength(cell); + } else if (typeof cell === "boolean") size = 1; + else if (typeof cell === "number") size = Buffer.byteLength(String(cell)); + else if (typeof cell === "bigint" && cell >= -9_223_372_036_854_775_808n && cell <= 9_223_372_036_854_775_807n) size = Buffer.byteLength(String(cell)); + else continue; + if (bytes + size > MAX_SAMPLE_BYTES) continue; + measuredValues += 1; + bytes += size; } } - return { bytes, estimatedValues }; + return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues }; } export function createDatabaseResponseCapture( @@ -71,32 +96,45 @@ export function createDatabaseResponseCapture( const send = (event: Event) => { try { emit(event); } catch {} }; const expired = () => closed || now() >= window.end; const active = () => !expired() && now() >= window.start; + const armed = () => !expired(); + captureWindows.add(armed); const flush = () => { if (closed) return; closed = true; clearTimeout(timer); + captureWindows.delete(armed); + if (captureWindows.size === 0) { + operationScope.disable(); + operationScope = new AsyncLocalStorage(); + } scope.disable(); - const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes); - for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v1", captureId: id, + const ordered = [...entries.values()].sort((a, b) => b.calls - a.calls); + for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v2", captureId: id, groupIndex, groupPage: Math.floor(groupIndex / GROUP_PAGE_SIZE), - operation: entry.operation, table: entry.table, callers: entry.callers, calls: entry.calls, rows: entry.rows, - responseValueBytes: entry.bytes, maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues, - queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors }); - send({ event: "paperclip.db_response_capture.summary.v1", captureId: id, + ...entry }); + send({ event: "paperclip.db_response_capture.summary.v2", captureId: id, windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(), closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed, inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: GROUP_PAGE_SIZE, groupPages: Math.ceil(ordered.length / GROUP_PAGE_SIZE), - overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0), + overflowCalls, rows: ordered.reduce((sum, entry) => sum + entry.rows, 0), + sampledResults: ordered.reduce((sum, entry) => sum + entry.sampledResults, 0), + sampledValueBytes: ordered.reduce((sum, entry) => sum + entry.sampledValueBytes, 0), + sizingWallMs: ordered.reduce((sum, entry) => sum + entry.sizingWallMs, 0), + sampleEvery: SAMPLE_EVERY, sampling: "first-and-every-16th-successful-result-per-group", + maxSampleCells: MAX_SAMPLE_CELLS, maxSampleStringLength: MAX_SAMPLE_STRING_LENGTH, + maxSampleStringUnits: MAX_SAMPLE_STRING_UNITS, maxSampleBytes: MAX_SAMPLE_BYTES, rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false, measurementBoundary: "postgres-js results before Drizzle column decoding", + wallTimeBoundary: "first query consumption to driver result; includes queue, database, network and event-loop waits; not CPU", + byteMeasurement: "sampled primitive values only; complex/oversized/budget-exceeding values unmeasured; no extrapolation", relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true, taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false }); entries.clear(); }; const timer = setTimeout(flush, Math.max(0, window.end - now())); timer.unref(); - function finish(metadata: Metadata, value: unknown, failed: boolean) { + function finish(metadata: Metadata & { owner: string }, value: unknown, failed: boolean, startedAt: number) { if (!active()) return; inFlight -= 1; completed += 1; @@ -104,38 +142,52 @@ export function createDatabaseResponseCapture( let key = JSON.stringify(metadata); if (!entries.has(key) && entries.size >= 2047) { overflowCalls += 1; - metadata = { operation: "other", table: "other", callers: [] }; + metadata = { owner: "other", operation: "other", table: "other", callers: [] }; key = "overflow"; } let entry = entries.get(key); if (!entry) { - entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, queryErrors: 0, serializationErrors: 0, estimatedValues: 0 }; + entry = { ...metadata, calls: 0, rows: 0, queryErrors: 0, totalWallMs: 0, maxWallMs: 0, + sampledResults: 0, sampledRows: 0, sampledValueBytes: 0, maxSampledValueBytes: 0, + measuredValues: 0, unmeasuredValues: 0, incompleteSamples: 0, sizingWallMs: 0 }; entries.set(key, entry); } entry.calls += 1; + const wallMs = performance.now() - startedAt; + entry.totalWallMs += wallMs; + entry.maxWallMs = Math.max(entry.maxWallMs, wallMs); if (failed) { entry.queryErrors += 1; return; } entry.rows += Array.isArray(value) ? value.length : 0; + if ((entry.calls - entry.queryErrors - 1) % SAMPLE_EVERY !== 0) return; + entry.sampledResults += 1; + entry.sampledRows += Array.isArray(value) ? value.length : 0; + const sizingStartedAt = performance.now(); try { const size = responseValueSize(value); - entry.bytes += size.bytes; - entry.maxBytes = Math.max(entry.maxBytes, size.bytes); - entry.estimatedValues += size.estimatedValues; - } catch { entry.serializationErrors += 1; } + entry.sampledValueBytes += size.bytes; + entry.maxSampledValueBytes = Math.max(entry.maxSampledValueBytes, size.bytes); + entry.measuredValues += size.measuredValues; + entry.unmeasuredValues += size.unmeasuredValues; + if (size.unmeasuredValues) entry.incompleteSamples += 1; + } catch { entry.incompleteSamples += 1; } + finally { entry.sizingWallMs += performance.now() - sizingStartedAt; } } catch {} } function execute(action: () => unknown, metadata: Metadata) { if (!active()) return action(); inFlight += 1; started += 1; + const owned = { ...metadata, owner: operationScope.getStore() ?? "unattributed" }; + const startedAt = performance.now(); let result: unknown; try { result = action(); } - catch (error) { finish(metadata, undefined, true); throw error; } + catch (error) { finish(owned, undefined, true, startedAt); throw error; } let asynchronous = false; try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {} if (asynchronous) { - return Promise.resolve(result).then(value => { finish(metadata, value, false); return value; }, - error => { finish(metadata, undefined, true); throw error; }); + return Promise.resolve(result).then(value => { finish(owned, value, false, startedAt); return value; }, + error => { finish(owned, undefined, true, startedAt); throw error; }); } - finish(metadata, result, false); + finish(owned, result, false, startedAt); return result; } return { window, roots, queries, factories, clients, driverQueries, scope, active, expired, execute, flush }; @@ -240,7 +292,7 @@ export function installDatabaseResponseCapture(db: T, schema: } return scoped ? { ...scoped, table: scoped.table === "other" ? table : scoped.table } : { operation: text.match(/^\s*(select|insert|update|delete)\b/i)?.[1]?.toLowerCase() ?? "execute", - table, callers: publicDatabaseCallers(new Error().stack) }; + table, callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] }; } function client(value: unknown) { if ((!object(value) && typeof value !== "function") || capture!.clients.has(value)) return; @@ -277,7 +329,8 @@ export function installDatabaseResponseCapture(db: T, schema: try { if (method === "with") { root(result); return result; } const operation = method.startsWith("select") ? "select" : method; - const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), callers: publicDatabaseCallers(new Error().stack) }; + const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), + callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] }; if (operation === "select") factory(result, "from", metadata, true); else if (method === "insert") factory(result, "values", metadata); else if (method === "update") factory(result, "set", metadata); diff --git a/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts new file mode 100644 index 00000000000..dbb8b9eac18 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts @@ -0,0 +1,273 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NOOP_PLUGIN_TRACER, startWorkerRpcHost, type PluginInvocationContext, type PluginTracer } from "@paperclipai/plugin-sdk"; +import plugin, { __resetDaytonaSandboxHandleCacheForTest, __setDaytonaPluginContextForTest } from "./plugin.js"; +import manifest from "./manifest.js"; +import { leaseStateAttributes, withLeaseSpan } from "./lease-tracing.js"; + +const { get, create } = vi.hoisted(() => ({ get: vi.fn(), create: vi.fn() })); +vi.mock("@daytonaio/sdk", () => ({ + Daytona: class { get = get; create = create; }, + DaytonaNotFoundError: class extends Error {}, + DaytonaTimeoutError: class extends Error {}, +})); + +type Receipt = { + name: string; + startTimeMs: number; + endTimeMs: number; + attributes?: Record; + status?: { code: number }; + children: Receipt[]; +}; + +function sandbox(id = "fixture-sandbox", state = "archived") { + return { + id, state, target: "us", recoverable: true, errorReason: null as string | null, + start: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + refreshData: vi.fn().mockResolvedValue(undefined), + recover: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + getWorkDir: vi.fn().mockResolvedValue("/home/daytona"), + fs: { createFolder: vi.fn().mockResolvedValue(undefined), uploadFile: vi.fn().mockResolvedValue(undefined) }, + process: { executeCommand: vi.fn(async (command: string) => ({ + exitCode: 0, result: command.startsWith("cat ") ? '{"token":"private-token"}' : "bash", + })) }, + }; +} + +const params = { + driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment", + providerLeaseId: "fixture-sandbox", config: { apiKey: "private-api-key", timeoutMs: 300000, reuseLease: true }, + leaseMetadata: { workspaceSentinel: { token: "private-token", result: "written" } }, +}; + +function workerHarness() { + const stdin = new PassThrough(), stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const pending = new Map(); + const receipts: Array<{ invocationId: string; span: Receipt }> = []; + let sequence = 0; + lines.on("line", line => { + const message = JSON.parse(line); + if (message.method) { + if (message.method === "span.record") receipts.push({ invocationId: message.paperclipInvocationId, span: message.params }); + if (message.id !== undefined) stdin.write(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: null }) + "\n"); + } else { + const waiter = pending.get(message.id); + pending.delete(message.id); + if (message.error) waiter?.reject(new Error(message.error.message)); + else waiter?.resolve(message.result); + } + }); + const worker = startWorkerRpcHost({ plugin, stdin, stdout }); + function call(method: string, input: unknown, invocation?: PluginInvocationContext): Promise { + const id = String(++sequence); + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: input, + ...(invocation ? { paperclipInvocation: invocation } : {}) }) + "\n"); + }); + } + return { + call, receipts, + initialize: () => call("initialize", { manifest, config: {}, apiVersion: 1, + instanceInfo: { instanceId: "fixture", hostVersion: "0.0.0" } }), + close() { worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); }, + }; +} + +function invocation(id: string, enabled = true): PluginInvocationContext { + return { id, scope: { companyId: "fixture-company" }, + ...(enabled ? { traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } : {}) }; +} + +describe("Daytona lease worker tracing", () => { + beforeEach(() => { get.mockReset(); create.mockReset(); __resetDaytonaSandboxHandleCacheForTest(); }); + afterEach(() => { __setDaytonaPluginContextForTest(null); __resetDaytonaSandboxHandleCacheForTest(); }); + + it.each([true, false])("records the archived start and nested verification only with trace context: %s", async enabled => { + const resource = sandbox(); get.mockResolvedValue(resource); + const worker = workerHarness(); + try { + await worker.initialize(); + expect(await worker.call("environmentResumeLease", params, invocation("resume", enabled))) + .toMatchObject({ providerLeaseId: resource.id, metadata: { resumedFromState: "archived" } }); + expect(resource.start).toHaveBeenCalledExactlyOnceWith(300); + expect(get).toHaveBeenCalledExactlyOnceWith(resource.id); + if (!enabled) { expect(worker.receipts).toEqual([]); return; } + expect(worker.receipts).toHaveLength(1); + const root = worker.receipts[0].span; + expect(root.name).toBe("plugin.worker"); + expect(root.children.map(span => span.name)).toEqual(["lease.lookup", "lease.channel_cleanup", "lease.start", "lease.verify"]); + expect(root.children[2].attributes).toMatchObject({ "paperclip.sandbox.startup.state": "archived" }); + expect(root.children[3].children.map(span => span.name)).toEqual(["lease.workspace", "lease.sentinel", "lease.shell"]); + function checkBounds(parent: Receipt) { + for (const child of parent.children) { + expect(child.startTimeMs).toBeGreaterThanOrEqual(parent.startTimeMs); + expect(child.endTimeMs).toBeGreaterThanOrEqual(child.startTimeMs); + expect(child.endTimeMs).toBeLessThanOrEqual(parent.endTimeMs); + checkBounds(child); + } + } + checkBounds(root); + expect(JSON.stringify(root)).not.toMatch(/private-|fixture-sandbox|fixture-company|\/home\//); + } finally { worker.close(); } + }); + + it("keeps simultaneous resume receipts separate and closes failed starts", async () => { + const slow = sandbox(), failed = sandbox("failed-sandbox", "stopped"); + let release!: () => void, started!: () => void; + const waiting = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { started = resolve; }); + slow.start.mockImplementation(async () => { started(); await waiting; }); + failed.start.mockRejectedValue(new Error("private-provider-error")); + get.mockImplementation(async id => id === slow.id ? slow : failed); + const worker = workerHarness(); + try { + await worker.initialize(); + const first = worker.call("environmentResumeLease", params, invocation("slow")); + await entered; + await expect(worker.call("environmentResumeLease", { ...params, providerLeaseId: failed.id }, invocation("failed"))) + .rejects.toThrow("private-provider-error"); + release(); await first; + expect(worker.receipts.map(record => record.invocationId)).toEqual(["failed", "slow"]); + const failure = worker.receipts[0].span; + expect(failure.status).toEqual({ code: 2 }); + expect(failure.children.find(span => span.name === "lease.start")?.status).toEqual({ code: 2 }); + expect(failure.children.some(span => span.name === "lease.verify")).toBe(false); + expect(JSON.stringify(worker.receipts)).not.toContain("private-provider-error"); + expect(slow.start).toHaveBeenCalledTimes(1); + expect(failed.start).toHaveBeenCalledTimes(1); + } finally { release(); worker.close(); } + }); + + it("waits for a pending teardown and records the wait under the resume worker", async () => { + const resource = sandbox("fixture-sandbox", "started"); get.mockResolvedValue(resource); + let release!: () => void, stopping!: () => void; + const waiting = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { stopping = resolve; }); + resource.stop.mockImplementation(async () => { stopping(); await waiting; resource.state = "stopped"; }); + const worker = workerHarness(); + try { + await worker.initialize(); + const teardown = worker.call("environmentReleaseLease", params, invocation("release")); + await entered; + let settled = false; + const resume = worker.call("environmentResumeLease", params, invocation("resume")) + .then(result => { settled = true; return result; }); + // Process the resume request while the actual provider stop is held open. + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toBe(false); + expect(get).toHaveBeenCalledTimes(1); + expect(resource.start).not.toHaveBeenCalled(); + release(); + await teardown; + expect(await resume).toMatchObject({ providerLeaseId: resource.id }); + const root = worker.receipts.find(receipt => receipt.invocationId === "resume")!.span; + expect(root.name).toBe("plugin.worker"); + expect(root.children.filter(span => span.name === "lease.wait_for_teardown")).toHaveLength(1); + expect(root.children[0].name).toBe("lease.wait_for_teardown"); + expect(root.children[0].endTimeMs).toBeLessThanOrEqual(root.children[1].startTimeMs); + expect(get).toHaveBeenCalledTimes(2); + expect(resource.start).toHaveBeenCalledTimes(1); + } finally { release(); worker.close(); } + }); + + it.each([false, true])("records missing-container confirmation and preserves refresh failures: %s", async fails => { + const resource = sandbox("fixture-sandbox", "error"); + resource.recoverable = false; + resource.errorReason = `not found: failed to inspect sandbox container ${resource.id}: Error response from daemon: No such container: ${resource.id}`; + get.mockResolvedValue(resource); + resource.refreshData.mockImplementation(async () => { + if (fails) throw new Error("private-refresh-failure"); + resource.state = "started"; + }); + const worker = workerHarness(); + try { + await worker.initialize(); + const result = worker.call("environmentResumeLease", params, invocation("resume")); + if (fails) await expect(result).rejects.toThrow("private-refresh-failure"); + else expect(await result).toMatchObject({ providerLeaseId: resource.id }); + expect(resource.refreshData).toHaveBeenCalledTimes(1); + const root = worker.receipts[0].span; + expect(root.children.map(span => span.name)).toEqual(fails + ? ["lease.lookup", "lease.refresh"] : ["lease.lookup", "lease.refresh", "lease.verify"]); + expect(root.children[1].status).toEqual(fails ? { code: 2 } : undefined); + expect(resource.start).not.toHaveBeenCalled(); + expect(resource.delete).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(JSON.stringify(worker.receipts)).not.toContain("private-refresh-failure"); + } finally { worker.close(); } + }); + + it.each(["started", "error"])("traces a %s sandbox without adding a provider start", async state => { + const resource = sandbox("fixture-sandbox", state); get.mockResolvedValue(resource); + const worker = workerHarness(); + try { + await worker.initialize(); + await worker.call("environmentResumeLease", params, invocation("resume")); + expect(resource.start).not.toHaveBeenCalled(); + expect(resource.recover).toHaveBeenCalledTimes(state === "error" ? 1 : 0); + const phases = worker.receipts[0].span.children; + expect(phases.some(span => span.name === "lease.start")).toBe(false); + expect(phases.some(span => span.name === "lease.recover")).toBe(state === "error"); + } finally { worker.close(); } + }); + + it("closes the verification group and failing workspace child", async () => { + const resource = sandbox(); get.mockResolvedValue(resource); + resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error")); + const worker = workerHarness(); + try { + await worker.initialize(); + await expect(worker.call("environmentResumeLease", params, invocation("resume"))) + .rejects.toThrow("private-workspace-error"); + const verify = worker.receipts[0].span.children.find(span => span.name === "lease.verify")!; + expect(verify.status).toEqual({ code: 2 }); + expect(verify.children.map(span => span.name)).toEqual(["lease.workspace"]); + expect(verify.children[0].status).toEqual({ code: 2 }); + expect(resource.delete).not.toHaveBeenCalled(); + } finally { worker.close(); } + }); + + it("records setup failure and compensating cleanup as siblings", async () => { + const resource = sandbox(); create.mockResolvedValue(resource); + resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error")); + const worker = workerHarness(); + try { + await worker.initialize(); + await expect(worker.call("environmentAcquireLease", { ...params, runId: "fixture-run" }, invocation("acquire"))) + .rejects.toThrow("private-workspace-error"); + expect(resource.delete).toHaveBeenCalledTimes(1); + const root = worker.receipts[0].span; + expect(root.children.map(span => span.name)).toEqual(["lease.create", "lease.workspace", "lease.cleanup"]); + expect(root.children[1].status).toEqual({ code: 2 }); + expect(root.children[2].status).toBeUndefined(); + } finally { worker.close(); } + }); +}); + +describe("lease tracing cannot change provider work", () => { + it.each(["missing", "before", "after", "during"])("preserves the result and error with a %s tracer failure", async mode => { + const tracer: PluginTracer = { ...NOOP_PLUGIN_TRACER, + ...(mode === "missing" ? {} : { async withSpan(_name: string, work: () => T | Promise): Promise { + if (mode === "before") throw new Error("trace failure"); + const result = work(); + if (mode === "during") throw new Error("trace failure"); + await result; + throw new Error("trace failure"); + } }), + }; + const success = vi.fn(async () => 42), error = new Error("provider failure"), failure = vi.fn(async () => { throw error; }); + expect(await withLeaseSpan(tracer, "lease.lookup", success)).toBe(42); + await expect(withLeaseSpan(tracer, "lease.lookup", failure)).rejects.toBe(error); + expect(success).toHaveBeenCalledTimes(1); expect(failure).toHaveBeenCalledTimes(1); + }); + + it("normalizes unknown provider states before recording", () => { + expect(leaseStateAttributes("provider-private-state")).toEqual({ "paperclip.sandbox.startup.state": "other" }); + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts new file mode 100644 index 00000000000..8ad09499f64 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts @@ -0,0 +1,49 @@ +import type { PluginTracer } from "@paperclipai/plugin-sdk"; + +export type LeaseSpanName = + | "lease.wait_for_teardown" + | "lease.lookup" + | "lease.refresh" + | "lease.channel_cleanup" + | "lease.start" + | "lease.recover" + | "lease.verify" + | "lease.workspace" + | "lease.sentinel" + | "lease.shell" + | "lease.create" + | "lease.expiry" + | "lease.cleanup"; + +/** Keep provider state useful without exporting a new SDK value as free text. */ +export function leaseStateAttributes(state: unknown): Record { + return { + "paperclip.sandbox.startup.state": + state === "started" || state === "stopped" || state === "archived" || state === "error" + ? state : "other", + }; +} + +/** Use the invocation's async parent, while keeping tracer faults out of work. */ +export async function withLeaseSpan( + tracer: PluginTracer, + name: LeaseSpanName, + work: () => Promise, + attributes?: Record, +): Promise { + let result: Promise | undefined; + // A tracer can fail before or after invoking work. Preserve its exact result + // or error and never repeat provider operations in either case. + const runOnce = () => result ??= Promise.resolve().then(work); + try { + if (tracer.withSpan) { + await tracer.withSpan(name, runOnce, { + "paperclip.sandbox.startup.provider": "daytona", + ...attributes, + }); + } + } catch { + // The operation's error is still held by result and is rethrown below. + } + return runOnce(); +} diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 74dd8029c15..55f1aa8eaf0 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -50,6 +50,7 @@ import type { } from "@paperclipai/plugin-sdk"; import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js"; import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js"; +import { withLeaseSpan, leaseStateAttributes, type LeaseSpanName } from "./lease-tracing.js"; import { observeSessionStreamExit } from "./session-stream-exit.js"; import { DaytonaIdleActivity, trackDaytonaHooks, type DaytonaIdleProof } from "./idle-activity.js"; @@ -126,6 +127,10 @@ export function getPluginTracer(): PluginTracer { return pluginContext?.tracer ?? NOOP_PLUGIN_TRACER; } +function traceLeaseStep(name: LeaseSpanName, work: () => Promise, attributes?: Record): Promise { + return withLeaseSpan(getPluginTracer(), name, work, attributes); +} + /** * Test seam: set the module-level plugin context, and return a restore function. * `plugin.test.ts` uses it to inject a recording tracer without running `setup`. @@ -528,12 +533,16 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P const startBoundMs = timeoutSeconds * 1_000 + LIVENESS_START_TIMEOUT_MARGIN_MS; if (sandbox.state === "error") { if (sandbox.recoverable) { - await withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)); + await traceLeaseStep("lease.recover", () => + withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)), + leaseStateAttributes(sandbox.state)); return; } throw new Error(`Daytona sandbox ${sandbox.id} is in an unrecoverable error state: ${sandbox.errorReason ?? "unknown error"}`); } - await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)); + await traceLeaseStep("lease.start", () => + withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)), + leaseStateAttributes(sandbox.state)); } function hasMissingSandboxContainer(sandbox: Sandbox): boolean { @@ -1362,6 +1371,7 @@ const sandboxHandleActivityGates = (() => { type SandboxLeaseAdmissionOptions = { allowClosed?: boolean; + traceLeaseWait?: boolean; }; const sandboxHandleLeaseAdmissionStates = (() => { @@ -1402,7 +1412,8 @@ async function withSandboxActivityGate( const teardownGate = sandboxHandleTeardownGates.current(scope); if (teardownGate) { - await teardownGate.promise; + if (options.traceLeaseWait) await traceLeaseStep("lease.wait_for_teardown", () => teardownGate.promise); + else await teardownGate.promise; if (!options.allowClosed && sandboxHandleLeaseAdmissionStates.isClosed(scope)) { throw new Error(`Daytona sandbox lease ${scope.providerLeaseId} is no longer active.`); } @@ -2682,39 +2693,39 @@ const plugin = definePlugin(trackDaytonaHooks({ return timeoutFailure(); }; const acquire = async (): Promise => { - const sandbox = await createSandbox(params, config, { + const sandbox = await traceLeaseStep("lease.create", () => createSandbox(params, config, { onCreateAttempt: (attempt) => { assertActive(); cleanup = attempt; }, onCreateFailure: observeFailure, onSnapshotIdentity: identity => { snapshotIdentity = identity; }, - }); + })); try { assertActive(); if (cleanup) cleanup.observedProviderLeaseId = sandbox.id; phase = "workspace"; - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox)); assertActive(); phase = "shell"; - const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now()))); + const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now())))); assertActive(); // Configure a provider-side destroy time at or before a caller deadline, so // an abandoned sandbox self-destroys even if Paperclip is down. The lease // carries the real provider expiry (or none) as evidence of the bound. phase = "expiry"; - const expiresAt = await configureSandboxExpiry({ + const expiresAt = await traceLeaseStep("lease.expiry", () => configureSandboxExpiry({ sandbox, requestedExpiresAt: params.requestedExpiresAt, nowMs: Date.now(), preserveCreationExpiry: config.ttlMinutes != null, - }); + })); assertActive(); phase = "sentinel"; - const workspaceSentinel = await writeWorkspaceSentinel({ + const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => writeWorkspaceSentinel({ sandbox, remoteCwd, params, config, timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())), - }); + })); assertActive(); sandboxHandleLeaseAdmissionStates.open({ driverKey: params.driverKey, @@ -2757,7 +2768,7 @@ const plugin = definePlugin(trackDaytonaHooks({ cleanupInProgress = true; try { const proof = idleActivity.proof(providerActivityKey(params, config, sandbox.id)); - await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true); + await traceLeaseStep("lease.cleanup", () => sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true)); idleActivity.confirm(proof); } catch (cleanupError) { if (cleanup) { @@ -2821,7 +2832,7 @@ const plugin = definePlugin(trackDaytonaHooks({ throw new Error("The stopped Daytona sandbox is still settling cancelled work. Retry shortly."); } return await withSandboxActivityGate(scope, async () => { - const sandbox = await getSandboxOrNull(scope, { bypassTeardownGate: true }); + const sandbox = await traceLeaseStep("lease.lookup", () => getSandboxOrNull(scope, { bypassTeardownGate: true })); if (!sandbox) { return { providerLeaseId: null, metadata: { expired: true } }; } @@ -2831,7 +2842,7 @@ const plugin = definePlugin(trackDaytonaHooks({ // backup-guarded replacement path. Unknown errors preserve the lease. if (hasMissingSandboxContainer(sandbox)) { try { - await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()); + await traceLeaseStep("lease.refresh", () => withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData())); assertHandleMatchesLease(sandbox, params.providerLeaseId); } catch (error) { evictSandboxHandle(scope); @@ -2859,48 +2870,50 @@ const plugin = definePlugin(trackDaytonaHooks({ // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel // is dead after a real restart. Close and drop every channel on this lease // before the restart, so no stale channel id survives the resume. - await closeDaytonaDuplexChannelsForLease(params.providerLeaseId); + await traceLeaseStep("lease.channel_cleanup", () => closeDaytonaDuplexChannelsForLease(params.providerLeaseId)); } const resumedFromState = sandbox.state ?? null; await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); try { - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox, false); - // C3: a resumed lease must clear the workspace sentinel before it is - // trusted, even when the handle came from the cache. On any non-match we - // evict the cached handle and expire the lease so a stale/foreign sandbox - // is never reused on the subsequent (sentinel-skipping) exec path. - const workspaceSentinel = await verifyWorkspaceSentinel({ - sandbox, - remoteCwd, - leaseMetadata: params.leaseMetadata, - timeoutSeconds: toTimeoutSeconds(config.timeoutMs), - }); - if (workspaceSentinel.result !== "matched") { - evictSandboxHandle(scope); - return { - providerLeaseId: null, - metadata: { expired: true, workspaceSentinel }, - }; - } - const shellCommand = await detectSandboxShellCommand( - sandbox, - toTimeoutSeconds(config.timeoutMs), - ); - sandboxHandleCache.markFresh(scope); - sandboxHandleLeaseAdmissionStates.open(scope); - return { - providerLeaseId: sandbox.id, - metadata: leaseMetadata({ - config, + return await traceLeaseStep("lease.verify", async () => { + const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox, false)); + // C3: a resumed lease must clear the workspace sentinel before it is + // trusted, even when the handle came from the cache. On any non-match we + // evict the cached handle and expire the lease so a stale/foreign sandbox + // is never reused on the subsequent (sentinel-skipping) exec path. + const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => verifyWorkspaceSentinel({ sandbox, - shellCommand, - remoteCwd: requestedCwd ?? remoteCwd, - resumedLease: true, - resumedFromState, - sandboxState: "started", - workspaceSentinel, - }), - }; + remoteCwd, + leaseMetadata: params.leaseMetadata, + timeoutSeconds: toTimeoutSeconds(config.timeoutMs), + })); + if (workspaceSentinel.result !== "matched") { + evictSandboxHandle(scope); + return { + providerLeaseId: null, + metadata: { expired: true, workspaceSentinel }, + }; + } + const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand( + sandbox, + toTimeoutSeconds(config.timeoutMs), + )); + sandboxHandleCache.markFresh(scope); + sandboxHandleLeaseAdmissionStates.open(scope); + return { + providerLeaseId: sandbox.id, + metadata: leaseMetadata({ + config, + sandbox, + shellCommand, + remoteCwd: requestedCwd ?? remoteCwd, + resumedLease: true, + resumedFromState, + sandboxState: "started", + workspaceSentinel, + }), + }; + }); } catch (error) { evictSandboxHandle(scope); // A timeout, rate limit, or provider 5xx does not prove this sandbox is @@ -2910,7 +2923,7 @@ const plugin = definePlugin(trackDaytonaHooks({ throw error; } }, - { allowClosed: true }, + { allowClosed: true, traceLeaseWait: true }, ); }, diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 2d6d93331d5..28f9fe42585 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -198,6 +198,12 @@ export const ISSUE_STATUSES = [ ] as const; export type IssueStatus = (typeof ISSUE_STATUSES)[number]; +/** Read-only task presentation. Monitoring never grants execution authority. */ +export const ISSUE_DISPLAY_STATUSES = [ + "backlog", "todo", "in_progress", "monitoring", "in_review", "done", "blocked", "cancelled", +] as const; +export type IssueDisplayStatus = (typeof ISSUE_DISPLAY_STATUSES)[number]; + export const INBOX_MINE_ISSUE_STATUSES = [ "backlog", "todo", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index d6a9e5cbd02..da43d193d30 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -364,6 +364,7 @@ export { AGENT_ICON_NAMES, PROJECT_ICON_NAMES, ISSUE_STATUSES, + ISSUE_DISPLAY_STATUSES, ISSUE_SUMMARIES_MAX_IDS, INBOX_MINE_ISSUE_STATUSES, INBOX_MINE_ISSUE_STATUS_FILTER, @@ -562,6 +563,7 @@ export { type AgentIconName, type ProjectIconName, type IssueStatus, + type IssueDisplayStatus, type IssuePriority, type IssueReviewPolicy, type IssueWorkMode, diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 29a8008ec80..451f745db97 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -20,6 +20,7 @@ import type { IssueHarnessKind, IssueOriginKind, IssuePriority, + IssueDisplayStatus, IssueReviewPolicy, IssueRecoveryActionKind, IssueRecoveryActionOutcome, @@ -270,6 +271,7 @@ export interface AcceptedPlanDecompositionSummary extends AcceptedPlanDecomposit } export interface IssueRelationIssueSummary { + displayStatus?: IssueDisplayStatus; id: string; identifier: string | null; title: string; @@ -850,12 +852,14 @@ export type IssueChanges = Record; export type IssueSummary = Pick< Issue, - "id" | "companyId" | "title" | "identifier" | "status" | "updatedAt" + "id" | "companyId" | "title" | "identifier" | "status" | "displayStatus" | "updatedAt" > & { externalConversationState: "active" | "waiting" | null; }; export interface Issue { + /** Read-only status for task presentation; never use it for execution admission. */ + displayStatus?: IssueDisplayStatus; /** Collection projections omit task bodies and cannot seed the detail cache. */ listProjection?: "index" | "rows"; statusVersion?: number; @@ -985,6 +989,7 @@ export type CompactIssue = Pick< | "privacyRootIssueId" | "title" | "description" + | "displayStatus" | "status" | "workMode" | "priority" diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts index 825d4d4d1c4..bc9ea562d53 100644 --- a/packages/shared/src/types/search.ts +++ b/packages/shared/src/types/search.ts @@ -1,4 +1,4 @@ -import type { IssuePriority, IssueStatus } from "../constants.js"; +import type { IssueDisplayStatus, IssuePriority, IssueStatus } from "../constants.js"; export const COMPANY_SEARCH_SCOPES = ["all", "issues", "comments", "documents", "artifacts", "agents", "projects"] as const; export type CompanySearchScope = (typeof COMPANY_SEARCH_SCOPES)[number]; @@ -38,6 +38,7 @@ export interface CompanySearchIssueSummary { identifier: string | null; title: string; status: IssueStatus; + displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null; priority: IssuePriority; assigneeAgentId: string | null; @@ -76,6 +77,7 @@ export interface CompanySearchResult { export interface CompanySearchFilterOptionCounts { status: Partial>; + displayStatus?: Partial>; priority: Partial>; assigneeAgentId: Record; assigneeUserId: Record; diff --git a/packages/shared/src/types/user-profile.ts b/packages/shared/src/types/user-profile.ts index d961a407485..1ac26e033a7 100644 --- a/packages/shared/src/types/user-profile.ts +++ b/packages/shared/src/types/user-profile.ts @@ -1,4 +1,4 @@ -import type { IssuePriority, IssueStatus } from "../constants.js"; +import type { IssuePriority, IssueStatus, IssueDisplayStatus } from "../constants.js"; export interface UserProfileIdentity { id: string; @@ -42,6 +42,7 @@ export interface UserProfileIssueSummary { identifier: string | null; title: string; status: IssueStatus; + displayStatus?: IssueDisplayStatus; priority: IssuePriority; assigneeAgentId: string | null; assigneeUserId: string | null; diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts index 839f4cb5296..3b81dae3836 100644 --- a/packages/shared/src/validators/search.ts +++ b/packages/shared/src/validators/search.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js"; +import { ISSUE_DISPLAY_STATUSES, ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js"; import { isUuidLike } from "../agent-url-key.js"; import { COMPANY_SEARCH_EXTRACT_KINDS, @@ -156,6 +156,9 @@ export const companySearchQuerySchema = z.object({ status: z.unknown() .optional() .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)), + displayStatus: z.unknown() + .optional() + .transform((value, ctx) => parseEnumList(value, ctx, "displayStatus", ISSUE_DISPLAY_STATUSES)), priority: z.unknown() .optional() .transform((value, ctx) => parseEnumList(value, ctx, "priority", ISSUE_PRIORITIES)), diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 97290f343e1..12dbe906011 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -579,8 +579,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), - readWorkflow("chat-feature-testops.yml"), - readWorkflow("chat-completion-testops.yml"), readWorkflow("chat-catalog-testops.yml"), ]; @@ -621,8 +619,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", "chat-catalog-testops.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); @@ -705,38 +701,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { } continue; } - if (name === "chat-feature-testops.yml") { - assert.equal(providerJobs.length, 1); - const assertBoundary = (block) => { - assert.match(block, /^ chat:\n/); - assert.match(block, /\n needs: \[authorize, select, build\]\n/); - assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); - assert.match(block, /\n environment: runner-e2e-private-paid\n/); - assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); - assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/); - assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); - assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/); - assert.doesNotMatch(block, /ALLURE_TOKEN/); - assert.equal([...block.matchAll(providerSecretReference)].length, 1); - assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); - assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference)); - }; - const block = providerJobs[0]; - assertBoundary(block); - assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); - assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/); - assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/); - assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/); - for (const mutant of [ - block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), - block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), - block.replace(" - *guard\n", ""), - block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"), - block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"), - block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"), - ]) assert.throws(() => assertBoundary(mutant)); - continue; - } + for (const block of providerJobs) { if (name === "runner-protocol-live-evals.yml") { assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}")); diff --git a/scripts/benchmark-github-webhook-ingress.ts b/scripts/benchmark-github-webhook-ingress.ts index 3b24116a650..25cbf5891e0 100644 --- a/scripts/benchmark-github-webhook-ingress.ts +++ b/scripts/benchmark-github-webhook-ingress.ts @@ -5,7 +5,7 @@ import { promises as fs, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { applyPendingMigrations, chatActions, createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../packages/db/src/response-capture.js"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../server/src/__tests__/helpers/full-database-response-capture.js"; import { createGitHubWebhookIngress } from "../server/src/services/chat-channels/github/webhook-ingress.js"; import { captureIngressQueries, cleanupIngressFixture, githubIngressFixture, type CapturedQuery } from "../server/src/__tests__/helpers/github-webhook-ingress.js"; import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js"; @@ -60,7 +60,7 @@ try { queries.length = 0; f.calls.length = 0; const events: Array> = [], timestamp = Date.now(); - const capture = iteration === 11 ? createDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined; + const capture = iteration === 11 ? createFullDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, { chatActions }, capture); const start = performance.now(); const response = await service.processGitHubWebhookIngress(action.id); @@ -80,7 +80,7 @@ try { const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${candidate.sql}`, candidate.params as never[]); const positionalRows = await db.$client.unsafe(candidate.sql, candidate.params as never[]).values(); measurements[name] = { queries: executed, initialQueryPlan: plan["QUERY PLAN"], initialReadJsonBytes: Buffer.byteLength(JSON.stringify(positionalRows)), - responseValueBytes: events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes, + responseValueBytes: events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes, captureEvents: events }; } } diff --git a/server/scripts/benchmark-native-backup-publication.mjs b/server/scripts/benchmark-native-backup-publication.mjs new file mode 100644 index 00000000000..78d607ed3a8 --- /dev/null +++ b/server/scripts/benchmark-native-backup-publication.mjs @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { fork, execFileSync } from "node:child_process"; +import { createServer, get, Agent } from "node:http"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { setTimeout as delay } from "node:timers/promises"; + +const filename = fileURLToPath(import.meta.url); +const checkout = path.resolve(path.dirname(filename), "../.."); +const arg = (name, fallback) => { + const position = process.argv.indexOf(`--${name}`); + return position < 0 ? fallback : process.argv[position + 1]; +}; +const percentile = (values, fraction) => { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? null; +}; + +if (process.argv.includes("--worker")) { + const { publishNativeHarnessBackup } = await import(pathToFileURL(arg("module"))); + let publishing = false; + const server = createServer((_request, response) => { + response.setHeader("x-publication-active", publishing ? "1" : "0"); + response.end("ok"); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + process.send({ kind: "ready", port: server.address().port }); + process.on("message", async message => { + if (message.kind === "close") { + server.closeAllConnections(); + server.close(() => process.disconnect()); + return; + } + if (message.kind !== "publish") return; + const histogram = monitorEventLoopDelay({ resolution: 10 }); + histogram.enable(); + await delay(20); + const cpu = process.cpuUsage(); + const started = performance.now(); + publishing = true; + try { + await publishNativeHarnessBackup({ + db: {}, companyId: "benchmark", scopeDigest: "fixture", + backupRoot: message.backupRoot, pendingRoot: message.pendingRoot, + stamp: async () => {}, + }); + const durationMs = performance.now() - started; + const cpuUsed = process.cpuUsage(cpu); + publishing = false; + await delay(20); + histogram.disable(); + assert.equal(await fs.readFile(path.join(message.backupRoot, "current/manifest.json"), "utf8"), "replacement"); + assert.deepEqual((await fs.readdir(message.backupRoot)).sort(), ["current"]); + assert.equal(await fs.readFile(path.join(message.liveRoot, "untouched.txt"), "utf8"), "live"); + assert.equal((await fs.stat(message.liveRoot)).mode & 0o777, 0o555); + process.send({ kind: "result", durationMs, cpuMs: (cpuUsed.user + cpuUsed.system) / 1000, + eventLoopMaxMs: histogram.max / 1e6, workCompleted: true }); + } catch (error) { + publishing = false; + process.send({ kind: "error", message: error.stack }); + } + }); +} else { + const baselineRef = arg("baseline-ref"); + assert(baselineRef, "--baseline-ref is required"); + const pairs = Number(arg("pairs", "3")); + const directories = Number(arg("directories", "100")); + const filesPerDirectory = Number(arg("files-per-directory", "100")); + const intervalMs = Number(arg("interval-ms", "5")); + for (const value of [pairs, directories, filesPerDirectory, intervalMs]) assert(Number.isSafeInteger(value) && value > 0); + const evidence = await fs.mkdtemp(path.join(tmpdir(), "native-backup-publication-proof-")); + const { build } = await import("esbuild"); + const buildVariant = async variant => { + const outfile = path.join(evidence, `${variant}.mjs`); + await build({ absWorkingDir: checkout, entryPoints: ["server/src/services/native-runtime/native-harness-backup-publication.ts"], + outfile, bundle: true, platform: "node", format: "esm", target: "node24", sourcemap: true, + alias: { "@paperclipai/db/work-signals": path.join(checkout, "packages/db/src/work-signals.ts") }, + plugins: variant === "baseline" ? [{ name: "pinned-source", setup(builder) { + builder.onLoad({ filter: /\.ts$/ }, args => { + const relative = path.relative(checkout, args.path); + if (relative.startsWith("..") || relative.includes("node_modules")) return; + return { contents: execFileSync("git", ["show", `${baselineRef}:${relative}`], { cwd: checkout, encoding: "utf8" }), + loader: "ts", resolveDir: path.dirname(args.path) }; + }); + } }] : [], + }); + return outfile; + }; + const modules = { baseline: await buildVariant("baseline"), treatment: await buildVariant("treatment") }; + const makeFixture = async label => { + const root = await fs.mkdtemp(path.join(tmpdir(), `backup-publication-${label}-`)); + const backupRoot = path.join(root, "backups"); + const liveRoot = path.join(root, "live"); + const pendingRoot = path.join(backupRoot, "pending"); + await fs.mkdir(liveRoot); + await fs.writeFile(path.join(liveRoot, "untouched.txt"), "live"); + await fs.chmod(liveRoot, 0o555); + for (const version of ["current", "previous"]) { + const versionRoot = path.join(backupRoot, version); + for (let directory = 0; directory < directories; directory++) { + const target = path.join(versionRoot, `skill-${directory}`); + await fs.mkdir(target, { recursive: true }); + for (let file = 0; file < filesPerDirectory; file++) { + await fs.writeFile(path.join(target, `file-${file}.txt`), "retired instructions\n".repeat(4), { mode: 0o444 }); + } + await fs.chmod(target, 0o555); + } + await fs.symlink(liveRoot, path.join(versionRoot, "outside")); + await fs.chmod(versionRoot, 0o555); + } + await fs.mkdir(pendingRoot); + await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement"); + return { root, backupRoot, liveRoot, pendingRoot }; + }; + const run = async variant => { + const fixture = await makeFixture(variant); + const worker = fork(filename, ["--worker", "--module", modules[variant]], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); + const exited = new Promise(resolve => worker.once("exit", resolve)); + const nextMessage = kind => new Promise((resolve, reject) => { + const onMessage = message => { + if (message.kind !== kind && message.kind !== "error") return; + worker.off("message", onMessage); worker.off("exit", onExit); + if (message.kind === "error") reject(new Error(message.message)); else resolve(message); + }; + const onExit = code => reject(new Error(`worker exited before ${kind}: ${code}`)); + worker.on("message", onMessage); worker.once("exit", onExit); + }); + const { port } = await nextMessage("ready"); + const agent = new Agent({ keepAlive: true, maxSockets: 512 }); + const requests = []; + let sent = 0; + const epoch = performance.now(); + const dispatch = intended => { + const dispatched = performance.now(); + requests.push(new Promise(resolve => { + const request = get({ hostname: "127.0.0.1", port, agent }, response => { + response.resume(); + response.on("end", () => resolve({ intended, dispatchLateMs: dispatched - intended, + latencyMs: performance.now() - intended, active: response.headers["x-publication-active"] === "1", error: false })); + }); + request.setTimeout(10_000, () => request.destroy(new Error("request timeout"))); + request.on("error", () => resolve({ intended, dispatchLateMs: dispatched - intended, + latencyMs: performance.now() - intended, active: false, error: true })); + })); + }; + let ticker; + const pump = () => { + const now = performance.now(); + while (epoch + (sent + 1) * intervalMs <= now) dispatch(epoch + (++sent * intervalMs)); + ticker = setTimeout(pump, Math.max(0, epoch + (sent + 1) * intervalMs - performance.now())); + }; + ticker = setTimeout(pump, intervalMs); + try { + await delay(250); + const pending = nextMessage("result"); + const start = performance.now(); + worker.send({ kind: "publish", ...fixture }); + const publication = await pending; + const end = performance.now(); + await delay(250); + clearTimeout(ticker); + const completed = await Promise.all(requests); + const during = completed.filter(request => request.intended >= start && request.intended <= end); + assert(during.length > 0); + const result = { variant, ...publication, sent, completed: completed.length, + errors: completed.filter(request => request.error).length, duringRequests: during.length, + requestsServedDuringPublication: completed.filter(request => request.active).length, + foregroundP95Ms: percentile(during.map(request => request.latencyMs), 0.95), + foregroundMaxMs: Math.max(...during.map(request => request.latencyMs)), + generatorMaxLateMs: Math.max(...completed.map(request => request.dispatchLateMs)) }; + console.log(JSON.stringify(result)); + return result; + } finally { + clearTimeout(ticker); agent.destroy(); + if (worker.connected) worker.send({ kind: "close" }); + await exited; + await fs.chmod(fixture.liveRoot, 0o700); + if ((await fs.readdir(fixture.backupRoot)).join() === "current") await fs.rm(fixture.root, { recursive: true }); + } + }; + const measurements = []; + for (let pair = 0; pair < pairs; pair++) { + for (const variant of pair % 2 ? ["treatment", "baseline"] : ["baseline", "treatment"]) measurements.push(await run(variant)); + } + const report = { baselineRef: execFileSync("git", ["rev-parse", baselineRef], { cwd: checkout, encoding: "utf8" }).trim(), + treatmentHead: execFileSync("git", ["rev-parse", "HEAD"], { cwd: checkout, encoding: "utf8" }).trim(), + treatmentDiff: execFileSync("git", ["diff", "HEAD", "--", "server/src/services/native-runtime/native-harness-backup-cleanup.ts", "server/src/services/native-runtime/native-harness-backup-publication.ts"], { cwd: checkout, encoding: "utf8" }), + node: process.version, platform: `${process.platform}/${process.arch}`, pairs, directories, filesPerDirectory, + filesRemovedPerPublication: 2 * directories * filesPerDirectory, intervalMs, evidence, measurements }; + const output = arg("output", path.join(evidence, "result.json")); + await fs.writeFile(output, JSON.stringify(report, null, 2)); + console.log(`Evidence: ${output}`); +} diff --git a/server/scripts/benchmark-skill-list.ts b/server/scripts/benchmark-skill-list.ts index b535fb2bed1..be1be55b982 100644 --- a/server/scripts/benchmark-skill-list.ts +++ b/server/scripts/benchmark-skill-list.ts @@ -16,7 +16,7 @@ import { authUsers, companies, companyMemberships, companySkills, companySkillSo projectSkillSources, projects, applyPendingMigrations, closeRegisteredClients, createDb, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js"; import * as schema from "../../packages/db/src/schema/index.js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../../packages/db/src/response-capture.js"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../src/__tests__/helpers/full-database-response-capture.js"; import { withRunExecutionTiming } from "../src/services/execution-preparation-timing.js"; const argument = (name: string) => process.argv.find(value => value.startsWith(`${name}=`))?.slice(name.length + 1); @@ -195,7 +195,7 @@ try { }; const first = await request(); assert.ok(first.etag); const events: Array> = [], now = Date.now(); - const capture = captureValues ? createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined; + const capture = captureValues ? createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, schema, capture); const samples = []; try { @@ -213,7 +213,7 @@ try { } finally { capture?.flush(); } assert.equal(await sourceHash(), measuredSourceHash, "Source changed during measurement"); const timings = samples.map(sample => sample.elapsedMs).sort((a, b) => a - b); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1"); + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1"); if (capture) assert.ok(summary && summary.inFlightAtEnd === 0 && summary.overflowCalls === 0); const rowCount = (await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).length; console.log(JSON.stringify({ schema: "paperclip.skill-list-benchmark.v1", sourceRevision, measuredSourceHash, diff --git a/server/scripts/bundle-runtime.test.mjs b/server/scripts/bundle-runtime.test.mjs index a634d11b2ac..adacbca2871 100644 --- a/server/scripts/bundle-runtime.test.mjs +++ b/server/scripts/bundle-runtime.test.mjs @@ -130,6 +130,42 @@ describe("bundleRuntime", () => { } }); + it("attributes asynchronous database operations in a plain runtime bundle", async () => { + const captureModule = fileURLToPath(new URL("../../packages/db/src/response-capture.ts", import.meta.url)); + write(path.join(root, "server", "dist", "services", "database-owner.js"), [ + `import { withDatabaseOperation } from ${JSON.stringify(captureModule)};`, + 'export const run = (owner, db) => withDatabaseOperation(owner, async () => {', + ' await Promise.resolve();', + ' return await db.$client.unsafe("select $1", ["private-parameter"]);', + '});', + ].join("\n")); + write(path.join(root, "server", "dist", "index.js"), [ + `import { createDatabaseResponseCapture, installDatabaseResponseCapture } from ${JSON.stringify(captureModule)};`, + 'import { run } from "./services/database-owner.js";', + 'const events = [];', + 'const now = Date.now();', + 'const capture = createDatabaseResponseCapture({ start: now, end: now + 60000 }, event => events.push(event));', + 'const db = { $client: { unsafe: () => ({ then(resolve, reject) { return Promise.resolve(Object.assign([["private-result"]], { columns: [{ name: "value" }] })).then(resolve, reject); } }) } };', + 'installDatabaseResponseCapture(db, {}, capture);', + 'await Promise.all([run("fixture.first", db), run("fixture.second", db)]);', + 'capture.flush();', + 'console.log(JSON.stringify(events));', + ].join("\n")); + const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir: path.join(root, "server", "dist-runtime") }); + const env = { ...process.env }; + delete env.NODE_OPTIONS; + delete env.PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW; + const { stdout } = await execFileAsync(process.execPath, [summary.outfile], { cwd: path.join(root, "server"), env, timeout: 30_000 }); + const events = JSON.parse(stdout); + const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); + expect(entries.map(entry => entry.owner).sort()).toEqual(["fixture.first", "fixture.second"]); + for (const entry of entries) expect(entry).toMatchObject({ calls: 1, rows: 1, callers: [] }); + expect(events.at(-1)).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2 }); + expect(stdout).not.toContain("private-parameter"); + expect(stdout).not.toContain("private-result"); + expect(stdout).not.toContain(root); + }); + it("emits a server module that loads without starting the server", async () => { const outdir = path.join(root, "server", "dist-runtime"); const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir }); diff --git a/server/src/__tests__/agent-auth-identity-transfer.test.ts b/server/src/__tests__/agent-auth-identity-transfer.test.ts index 819b23dc64a..995464f7933 100644 --- a/server/src/__tests__/agent-auth-identity-transfer.test.ts +++ b/server/src/__tests__/agent-auth-identity-transfer.test.ts @@ -4,7 +4,7 @@ import request from "supertest"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { actorMiddleware } from "../middleware/auth.js"; @@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport(); it("bounds actual middleware responses and preserves the complete stored run", async () => { const fixture = await seed(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`); @@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport(); expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId, identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" }); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384); diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index fcf3c5ebd6a..415781328d3 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js"; -import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; +import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js"; import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js"; @@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js"; import type { EnvironmentRuntimeService } from "../services/environment-runtime.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js"; import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js"; import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { heartbeatRunEvents } from "@paperclipai/db"; @@ -123,6 +124,44 @@ describe("persistent agent directories", () => { for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true); } finally { await worker.stop(); } }); + it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => { + const copy = await run(); + await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(), + nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm"); + try { + await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready; + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull(); + }); + await expect.poll(() => idleWorkSnapshot().active).toBe(0); + const reads = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled(); + expect(worker.nextWakeAt()).toBeNull(); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + } finally { await worker.stop(); } + }); + it("honors a cleanup obligation published before stale-deadline settlement", async () => { + const copy = await run(), dueAt = new Date(Date.now() + 30_000); + await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(), + nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + const transact = db.transaction.bind(db); + const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => { + await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + return transact(callback, config); + }); + try { + await copies.release(companyId, copy.runId); + expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null }); + await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + } finally { race.mockRestore(); } + }); it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => { const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory"); clock(); const worker = coordinator(); @@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => { return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId }; } + it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)( + "recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => { + const { copy, lease } = await unavailableRemote(); + const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID(); + const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000); + await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" }); + await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" }); + const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment", + issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } }) + .where(eq(environmentLeases.id, lease.id)).returning(); + const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt, + releasedAt: new Date(), metadata: { agentId } }; + if (scenario === "foreign_company") { + const foreignCompanyId = randomUUID(); + await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) }); + successor.companyId = foreignCompanyId; + } + const handoff = createReusableLeaseHandoff(previous!, successor); + const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId, + state: scenario === "stopped" ? "stopped" : "destroyed" }); + await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } }) + .where(eq(environmentLeases.id, lease.id)); + await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination, + reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } }); + if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" }); + await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute")); + try { + await copies.recoverStopped(); + const recovered = (await copies.get(companyId, copy.runId))!; + if (scenario === "destroyed") { + expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", + receipt: { cleanupPending: false }, nextAttemptAt: null }); + expect(recovered.processStoppedAt).toBeInstanceOf(Date); + await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } else { + expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null }); + expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial); + } + expect(shell).not.toHaveBeenCalled(); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)); + expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination"); + } finally { shell.mockRestore(); } + }); + it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])( "defers unavailable cleanup without exact destruction proof (%s)", async proof => { const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed", diff --git a/server/src/__tests__/agent-record-read-projections.test.ts b/server/src/__tests__/agent-record-read-projections.test.ts index bf0a5183202..7786b017d3e 100644 --- a/server/src/__tests__/agent-record-read-projections.test.ts +++ b/server/src/__tests__/agent-record-read-projections.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { agents, companies, costEvents, createDb } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => { async function observe(run: (observed: ReturnType) => Promise) { const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agents, costEvents }, capture); try { const result = await run(observed); diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index 5af78c2cb8b..ca18a4a2d3c 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { agents, @@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => { }))); } const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const feed = await attentionService(db).list(companyId, { userId: "board-user" }); expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64); @@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => { }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const feed = await attentionService(db).list(companyId, { @@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => { expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort()) .toEqual([...keptIds].sort()); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4); @@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => { const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction"); const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item])); const summaryEvents: Array> = [], captureNow = Date.now(); - const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event)); + const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event)); installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture); try { const summary = await attentionService(db).summary(companyId, { userId: "board-user" }); @@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => { expect(index.totalCount).toBe(feed.totalCount); } finally { summaryCapture.flush(); } - const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && ["documents", "assets", "issue_attachments"].includes(String(event.table))); expect(cardReads).toEqual([]); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 08c61039a80..cabb3d84093 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -33,11 +33,13 @@ import { budgetService } from "../services/budgets.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js"; import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js"; import { createSystemRecovery } from "../services/system-recovery-state.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js"; import type { Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js"; import * as activityLogModule from "../services/activity-log.js"; import * as connectionCredentialBindings from "../services/connection-credential-bindings.js"; @@ -112,6 +114,7 @@ import { isNotNull, isNull, like, + ne, notInArray, or, sql, @@ -2887,7 +2890,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk }; await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id)); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations, chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks, companyMemberships, chatActions, heartbeatRuns }, capture); @@ -2916,7 +2919,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } }; await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id)); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatDeliveries }, capture); let projected: { event: Record }; try { @@ -7152,6 +7155,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0); expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0); }); + it("offers Retry for a saved automatic PR event after a successful older review", async () => { + const f = await reviewBotFixture(); + const payload = { + action: "opened", installation: { id: 2468 }, + repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } }, + sender: { id: 77, login: "maintainer" }, + pull_request: { number: 81, title: "Review successive heads", body: "", draft: false, + base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) }, + user: { id: 42, login: "octocat", type: "User" }, labels: [] }, + }; + const send = async (event: typeof payload) => { + await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({ + event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret, + })); + await f.service.processPendingDeliveries(); + }; + await send(payload); + await expect.poll(async () => { + await f.service.processPendingDeliveries(); + return (await readChatConversationsByEndpointId(f.endpoint.id)).length; + }, { timeout: 10_000 }).toBe(1); + const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id); + const [task] = await readIssuesById(conversation.issueId!); + const [original] = await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"))); + await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, + issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id, + contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] }, + }).returning(); + await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id)); + await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } }); + await expect.poll(async () => { + await f.service.processPendingDeliveries(); + return (await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length; + }).toBe(2); + const [pending] = await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id))); + await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1); + const [receipt] = await readAgentWakeupRequestsById(pending.id); + await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, + payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]), + requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id)); + expect(receipt.requestedByActorType).toBe("user"); + await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id)); + expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" })) + .toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id }); + await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }) + .where(eq(agents.id, f.assignedAgentId)); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() }); + f.wakeup.mockImplementation(heartbeatService(db).wakeup); + const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user", + actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); + await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId }); + const [resumed] = await readAgentWakeupRequestsById(pending.id); + expect(resumed.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null, + contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } }); + expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId }); + }); + it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => { const f = await reviewBotFixture(); const { buildSkillMentionHref } = await import("@paperclipai/shared"); @@ -62866,7 +62931,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const commentsBeforeDisabledLifecycle = await db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)); + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(issueComments.id); const priorLifecycleIds = new Set( ( await db @@ -62946,7 +63012,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)), + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(issueComments.id), ).resolves.toEqual(commentsBeforeDisabledLifecycle); await service.shutdown(); }); @@ -64140,6 +64207,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { return { ...f, pending, originalDelivery }; } + async function successfulGitHubQueuedInputFixture() { + const f = await currentGitHubTaskRetryFixture(true); + const companyId = f.fixture.companyId, issueId = f.issue.id; + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload, + [String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id)); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {}, + canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed", + completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId)); + const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId, + completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId, + resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning(); + await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id)); + const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }; + return { ...f, input, result }; + } + + it("retries saved GitHub input after a successful run without replaying that run", async () => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const { input } = f; + const companyId = f.fixture.companyId; + const before = (await readHeartbeatRunsById(f.runId))[0]; + const [request, duplicate] = await Promise.all([requestTaskRetry(input), + requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]); + expect(request.accepted).toBe(true); + expect(duplicate.requestId).toBe(request.requestId); + await reconcileTaskChatRetryReceipts(db, { companyId }); + const [source] = await readAgentWakeupRequestsById(f.pending!.id); + expect(source.runId).toBeTruthy(); + const [successor] = await readHeartbeatRunsById(source.runId!); + expect(successor).toMatchObject({ status: "queued", retryOfRunId: null, + contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } }); + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(request.requestId!))[0]) + .toMatchObject({ status: "coalesced", runId: successor.id }); + expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + } finally { await f.service.shutdown(); } + }); + + it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)( + "keeps saved-input Retry behind current admission gates: %s", async gate => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const companyId = f.fixture.companyId; + const request = await requestTaskRetry(f.input); + const before = (await readHeartbeatRunsById(f.runId))[0]; + if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id)); + if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } }) + .where(eq(chatActions.id, f.pending!.id)); + if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id)); + if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId)); + const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId, + provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" }; + if (gate === "cleanup_pending") { + await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" }); + await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id)); + } + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull(); + expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + if (gate === "cleanup_pending") { + expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution"); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) }, + }).where(eq(environmentLeases.id, lease.id)); + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy(); + } + } finally { await f.service.shutdown(); } + }); + + it.each(["issue.comment.reopen", "boardMessageRetry"] as const)( + "resumes saved GitHub input after a successful board turn: %s", async origin => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const [previous] = await readHeartbeatRunsById(f.runId); + await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: { + ...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }), + } }).where(eq(heartbeatRuns.id, f.runId)); + const before = (await readHeartbeatRunsById(f.runId))[0]; + const retry = await requestTaskRetry(f.input); + await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! }); + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + expect(saved.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null, + contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } }); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + } finally { await f.service.shutdown(); } + }); + + it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const companyId = f.fixture.companyId; + const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId, + provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" }; + await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" }); + await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id)); + const retry = await requestTaskRetry(f.input); + await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull(); + const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id, + authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning(); + const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId, + source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: f.input.actorId, + payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id, + _paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" }, + }, [comment.id]), + }).returning(); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) }, + }).where(eq(environmentLeases.id, lease.id)); + await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! }); + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + expect(saved.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued", + contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } }); + expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board); + } finally { await f.service.shutdown(); } + }); + + it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => { + const f = await successfulGitHubQueuedInputFixture(); + const { issueRoutes } = await import("../routes/issues.js"); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); }); + app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service })); + app.use(errorHandler); + try { + const body = { observedRunId: f.runId, idempotencyKey: randomUUID() }; + const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body); + expect(first.status, JSON.stringify(first.body)).toBe(202); + await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy(); + const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body); + expect(second.status).toBe(202); + expect(second.body.requestId).toBe(first.body.requestId); + expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1); + const detail = await request(app).get(`/api/issues/${f.issue.id}`); + expect(detail.status).toBe(200); + expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false }); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + } finally { await f.service.shutdown(); } + }); + it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)( "admits current GitHub task Retry without replaying obsolete input: %s", async kind => { const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input")); @@ -64269,7 +64493,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { status: "active", membershipRole: "operator" }); } const queryEvents: Record[] = []; - const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1, + const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1, end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments, agentWakeupRequests, chatEndpoints, chatConversations }, capture); @@ -64289,7 +64513,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]); if (capture) { capture.flush(); - expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries)) + expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries)) .toBeLessThan(180); } expect(two.requestId).toBe(one.requestId); @@ -67489,7 +67713,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }))); const scope = { companyId: context.fixture.companyId, issueId: context.issue.id }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture); let scopes: Array<{ companyId: string; issueId: string }>; try { @@ -67506,7 +67730,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1, completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 }); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1") + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1") .reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length); } finally { await context.service.shutdown(); await other.service.shutdown(); @@ -77214,7 +77438,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }))); const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture); let scopes: Array<{ companyId: string; issueId: string }>; try { @@ -77231,7 +77455,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1, completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 }); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1") + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1") .reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length); } finally { await context.service.shutdown(); await other.service.shutdown(); diff --git a/server/src/__tests__/company-skills-service.test.ts b/server/src/__tests__/company-skills-service.test.ts index 8da12bed840..6ee86511ad7 100644 --- a/server/src/__tests__/company-skills-service.test.ts +++ b/server/src/__tests__/company-skills-service.test.ts @@ -17,7 +17,7 @@ import { projectWorkspaces, } from "@paperclipai/db"; import { parseFrontmatterMarkdown } from "@paperclipai/shared"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => { ]); await svc.list(companyId); const events: Record[] = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { agents, costEvents }, capture); let listed: Awaited>; try { listed = await svc.list(companyId); } finally { capture.flush(); } expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2); expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1); expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0); - const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents"); + const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents"); expect(agentReads).toHaveLength(1); expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0); + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0); await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId)); const refreshed = await svc.list(companyId, { q: "First" }); expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1); @@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => { const before = await svc.list(companyId); expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core"); const events: Record[] = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkills }, capture); let after: Awaited>; try { after = await svc.list(companyId); } finally { capture.flush(); } expect(after).toEqual(before); - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills"); + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills"); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000); const stored = await svc.getById(companyId, inserted[0]!.id); expect(stored?.markdown).toBe(savedMarkdown); @@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => { .orderBy(companySkillVersions.id); expect(before.length).toBeGreaterThan(0); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkillVersions }, capture); try { expect(await companySkillService(db).list(companyId)).toEqual(expected); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skill_versions" && event.operation === "select"); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096); @@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => { ? { type, companyId: otherCompanyId, agentId: randomUUID() } : { type, companyIds: [otherCompanyId] }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkillVersions }, capture); try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); } finally { capture.flush(); } - expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0); + expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0); }); it("restores a deleted release after warm validation", async () => { diff --git a/server/src/__tests__/conversation-settlement-transfer.test.ts b/server/src/__tests__/conversation-settlement-transfer.test.ts index 6a128b00e95..82b84ed75f3 100644 --- a/server/src/__tests__/conversation-settlement-transfer.test.ts +++ b/server/src/__tests__/conversation-settlement-transfer.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { settleConversationTurnForRun } from "../services/agent-conversations.js"; @@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport(); it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => { const input = await seed(reply), before = await sourceDigests(input); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture); try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table)) && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4); diff --git a/server/src/__tests__/database-operation-attribution.test.ts b/server/src/__tests__/database-operation-attribution.test.ts new file mode 100644 index 00000000000..41144ae6b27 --- /dev/null +++ b/server/src/__tests__/database-operation-attribution.test.ts @@ -0,0 +1,137 @@ +import express from "express"; +import request from "supertest"; +import { QueryPromise } from "drizzle-orm/query-promise"; +import { afterEach, expect, it, vi } from "vitest"; +import { withDatabaseOperation } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals"; +import { databaseOperationMiddleware } from "../middleware/database-operation.js"; +import { withBudgetSpan } from "../services/budget-tracing.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js"; +import { + createRunPreparationTiming, + measureRunPhase, + withRunExecutionTiming, + withRunResponsePresentationTiming, + withRunWorkspaceFinalizationTiming, +} from "../services/execution-preparation-timing.js"; + +const { owners } = await vi.hoisted(async () => { + const { AsyncLocalStorage } = await import("node:async_hooks"); + return { owners: new AsyncLocalStorage() }; +}); +vi.mock("@paperclipai/db", async importOriginal => ({ + ...await importOriginal(), + withDatabaseOperation: (owner: string, action: () => T): T => owners.run(owner, action), +})); + +class OwnerQuery extends QueryPromise { + async execute() { return owners.getStore(); } +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.useRealTimers(); +}); + +it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); + await withDatabaseOperation("http.request", async () => { + const query = new OwnerQuery(); + await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt"); + await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization"); + await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation"); + await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read"); + await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup"); + await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory"); + await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction"); + await expect(query).resolves.toBe("http.request"); + }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); + const failure = new Error("private accounting failure"); + await withRunExecutionTiming("private-run", async () => { + for (let i = 0; i < 18; i++) { + await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory"); + } + await expect(measureRunPhase("execution.budget_reservation", async () => { + await expect(withBudgetSpan("budget.admission.reserve", async () => { + await Promise.resolve(); + expect(owners.getStore()).toBe("budget.admission.reserve"); + throw failure; + })).rejects.toBe(failure); + return new OwnerQuery(); + })).resolves.toBe("execution.budget_reservation"); + await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt"); + }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", ""); + const app = express(); + app.use(databaseOperationMiddleware); + app.use(express.json()); + let arrived = 0; + let release!: () => void; + const both = new Promise(resolve => { release = resolve; }); + app.all("/:privateId", async (req, res) => { + const before = await new OwnerQuery(); + const work = async () => { + if (++arrived === 2) release(); + await both; + return new OwnerQuery(); + }; + const during = req.params.privateId === "first" + ? await withBudgetSpan("budget.admission.reserve", work) + : await measureRunPhase("skills.inventory", work); + res.json({ before, during, after: await new OwnerQuery() }); + }); + const [first, second] = await Promise.all([ + request(app).get("/first?secret=private-first").expect(200), + request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200), + ]); + expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" }); + expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => { + vi.useFakeTimers(); + const execute = vi.fn(async () => [{ xid: "42" }]); + const database = installDatabaseWorkSignals({ execute, async transaction(action: (tx: object) => Promise): Promise { + return action({ execute }); + } }); + const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = []; + const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true, + onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); }, + }); + let fail = true; + try { + const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue => + coordinator.register(queue, { retryMs: 5000, + run: async () => { + await Promise.resolve(); + seen.push({ queue, phase: "run", owner: await new OwnerQuery() }); + if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); } + }, + hasPending: async () => { + seen.push({ queue, phase: "pending", owner: await new OwnerQuery() }); + return false; + }, + }))); + await Promise.all(workers.map(worker => worker.ready)); + await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx => + notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback))); + await vi.advanceTimersByTimeAsync(1); + expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]); + for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`); + expect(owners.getStore()).toBeUndefined(); + expect(coordinator.nextWakeAt()).toBeNull(); + } finally { + await coordinator.stop(); + } +}); diff --git a/server/src/__tests__/github-operation-credentials.test.ts b/server/src/__tests__/github-operation-credentials.test.ts index 499134e2b55..75eeeacfcb5 100644 --- a/server/src/__tests__/github-operation-credentials.test.ts +++ b/server/src/__tests__/github-operation-credentials.test.ts @@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js"; import { errorHandler } from "../middleware/index.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, @@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); await db.insert(toolConnections).values(irrelevant); const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { toolConnections }, capture); try { expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" })) @@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }).where(eq(heartbeatRuns.id, input.runId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({ diff --git a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts index 98a002f8557..71328d1b64b 100644 --- a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts +++ b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { describe, expect, it } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { agents, companies, @@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => { protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } }, }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture); try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16); diff --git a/server/src/__tests__/heartbeat-run-log.test.ts b/server/src/__tests__/heartbeat-run-log.test.ts index 250db00acd7..18eb05b7354 100644 --- a/server/src/__tests__/heartbeat-run-log.test.ts +++ b/server/src/__tests__/heartbeat-run-log.test.ts @@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => { expect(compacted).not.toContain("paperclip-flag-secret"); }); }); + +describe("compactRunLogChunk with long base64 commands", () => { + it("redacts a tool call that writes base64 of source text without stalling", () => { + const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n"; + const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024); + expect(blob).toMatch(/^[A-Za-z0-9]+$/); + const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`; + const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`; + + const startedAt = performance.now(); + const compacted = compactRunLogChunk(chunk); + + expect(performance.now() - startedAt).toBeLessThan(1_000); + expect(compacted).toContain(blob); + expect(compacted).not.toContain("tool-call-secret"); + expect(compacted).toContain("***REDACTED***"); + }); +}); diff --git a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts index c610d17ded9..a610b5b25bd 100644 --- a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts +++ b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import express from "express"; import request from "supertest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, }).where(eq(heartbeatRuns.id, f.privateRunId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId)); const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const app = createApp(f.companyId, f.grantedAgentId, observed); diff --git a/server/src/__tests__/helpers/full-database-response-capture.test.ts b/server/src/__tests__/helpers/full-database-response-capture.test.ts new file mode 100644 index 00000000000..14a46c2ee8e --- /dev/null +++ b/server/src/__tests__/helpers/full-database-response-capture.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vitest"; +import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js"; + +const metadata = { operation: "select", table: "heartbeat_runs", callers: [] }; + +it("measures every oversized string and complex value without sampling or truncation", () => { + const events: Array> = []; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0); + const text = "private-payload🙂".repeat(30_000); + const nested = { instructions: text, additional: [{ prompt: text }] }; + const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] }); + const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested)); + try { + for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows); + capture.flush(); + expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1", + calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes, + estimatedValues: 17, serializationErrors: 0, + }); + expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1", + startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes, + }); + expect(JSON.stringify(events)).not.toContain("private-payload"); + expect(nested.instructions).toBe(text); + } finally { capture.flush(); } +}); + +it("preserves asynchronous results and original query errors", async () => { + const events: Array> = []; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0); + const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] }); + const failure = new Error("private query failure"); + try { + await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows); + await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure); + expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure); + capture.flush(); + expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2, + responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0, + }); + expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 }); + expect(JSON.stringify(events)).not.toContain(failure.message); + } finally { capture.flush(); } +}); + +it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => { + const events: Array> = []; + let now = 0; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now); + let release!: (value: string[]) => void; + const original = new Promise(resolve => { release = resolve; }); + const pending = capture.execute(() => original, metadata); + now = 1000; + capture.flush(); + release(["result"]); + await expect(pending).resolves.toEqual(["result"]); + capture.flush(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1", + startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0, + }); +}); diff --git a/server/src/__tests__/helpers/full-database-response-capture.ts b/server/src/__tests__/helpers/full-database-response-capture.ts new file mode 100644 index 00000000000..a52831c2adc --- /dev/null +++ b/server/src/__tests__/helpers/full-database-response-capture.ts @@ -0,0 +1,115 @@ +import { randomUUID } from "node:crypto"; +import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture"; + +export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; + +type Capture = ReturnType; +type Metadata = Parameters[1]; +type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; + queryErrors: number; serializationErrors: number; estimatedValues: number }; + +function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } { + const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined; + if (!Array.isArray(value) || !Array.isArray(columns)) { + const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); + return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 }; + } + let bytes = 0, estimatedValues = 0; + for (const row of value) { + const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]); + for (const cell of cells) { + if (cell === null || cell === undefined) continue; + if (typeof cell === "string") bytes += Buffer.byteLength(cell); + else if (typeof cell === "boolean") bytes += 1; + else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell)); + else { + estimatedValues += 1; + const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); + if (serialized !== undefined) bytes += Buffer.byteLength(serialized); + } + } + } + return { bytes, estimatedValues }; +} + +export function createFullDatabaseResponseCapture( + window: ResponseCaptureWindow, + emit: (event: Record) => void, + now: () => number = Date.now, +): Capture { + const capture = createDatabaseResponseCapture(window, () => {}, now); + const id = randomUUID(), entries = new Map(); + let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0; + const send = (event: Record) => { try { emit(event); } catch {} }; + function flush() { + if (closed) return; + closed = true; + clearTimeout(timer); + capture.flush(); + const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes); + for (const [groupIndex, entry] of ordered.entries()) send({ + event: "paperclip.test_db_response_capture.entry.v1", captureId: id, + groupIndex, groupPage: Math.floor(groupIndex / 128), + operation: entry.operation, table: entry.table, callers: entry.callers, + calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes, + maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues, + queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors, + }); + send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id, + windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(), + closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed, + inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128), + overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0), + rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false, + measurementBoundary: "postgres-js results before Drizzle column decoding", + relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true, + taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false, + }); + entries.clear(); + } + const timer = setTimeout(flush, Math.max(0, window.end - now())); + timer.unref(); + function finish(metadata: Metadata, value: unknown, failed: boolean) { + if (!capture.active()) return; + inFlight -= 1; + completed += 1; + let key = JSON.stringify(metadata); + if (!entries.has(key) && entries.size >= 2047) { + overflowCalls += 1; + metadata = { operation: "other", table: "other", callers: [] }; + key = "overflow"; + } + let entry = entries.get(key); + if (!entry) { + entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, + queryErrors: 0, serializationErrors: 0, estimatedValues: 0 }; + entries.set(key, entry); + } + entry.calls += 1; + if (failed) { entry.queryErrors += 1; return; } + entry.rows += Array.isArray(value) ? value.length : 0; + try { + const size = fullResponseValueSize(value); + entry.bytes += size.bytes; + entry.maxBytes = Math.max(entry.maxBytes, size.bytes); + entry.estimatedValues += size.estimatedValues; + } catch { entry.serializationErrors += 1; } + } + function execute(action: () => unknown, metadata: Metadata) { + if (!capture.active()) return action(); + inFlight += 1; + started += 1; + let result: unknown; + try { result = action(); } + catch (error) { finish(metadata, undefined, true); throw error; } + let asynchronous = false; + try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {} + if (asynchronous) return Promise.resolve(result).then( + value => { finish(metadata, value, false); return value; }, + error => { finish(metadata, undefined, true); throw error; }, + ); + finish(metadata, result, false); + return result; + } + return { ...capture, execute, flush }; +} diff --git a/server/src/__tests__/issue-creation-source-read.test.ts b/server/src/__tests__/issue-creation-source-read.test.ts index 2b47f3af580..ba3a74fc2cf 100644 --- a/server/src/__tests__/issue-creation-source-read.test.ts +++ b/server/src/__tests__/issue-creation-source-read.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => { expect(run.issueId).toBe(binding === "scoped" ? sourceId : null); expect(run.nativeIssueId).toBeNull(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture); try { const result = await resolveIssueCreationSource(observed, child); diff --git a/server/src/__tests__/issue-display-status.test.ts b/server/src/__tests__/issue-display-status.test.ts new file mode 100644 index 00000000000..e2f05c26720 --- /dev/null +++ b/server/src/__tests__/issue-display-status.test.ts @@ -0,0 +1,127 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import request from "supertest"; +import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db"; +import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared"; +import { issueDisplayStatusSql } from "../services/issue-display-status.js"; +import { issueService } from "../services/issues.js"; +import { companySearchService } from "../services/company-search.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueRoutes } from "../routes/issues.js"; +import { + describeEmbeddedPostgres, routeApp, + seedCompanyWithBoardAccess, useEmbeddedPostgres, +} from "./helpers/route-test-harness.js"; + +describeEmbeddedPostgres("Monitoring task presentation", () => { + const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", { + resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); }, + }); + + async function fixture() { + const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring"); + const agentId = randomUUID(); + await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} }); + const nextCheckAt = new Date(Date.now() + 60 * 60_000); + const issueId = randomUUID(); + await ctx.db.insert(issues).values({ + id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: company.actor.userId, + monitorNextCheckAt: nextCheckAt, + executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } }, + }); + return { ...company, agentId, issueId }; + } + + async function displayStatus(id: string) { + const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id)); + return row; + } + + it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => { + const f = await fixture(); + const svc = issueService(ctx.db); + expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" }); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]); + expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1); + const app = routeApp(ctx.db, f.actor, issueRoutes); + const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200); + expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" }); + const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" })); + expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" }); + expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1); + expect(result.filterOptionCounts.status.in_progress).toBe(1); + const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning(); + expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]); + // Existing agent/API lifecycle filters keep their original meaning. + expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1); + expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false); + }); + + it("filters before pagination and keeps In Progress separate from Monitoring", async () => { + const f = await fixture(); + await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId }); + const svc = issueService(ctx.db); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]); + expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1); + expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2); + }); + + it("shows the same wait indicator in list blocker summaries and task detail relations", async () => { + const f = await fixture(); + const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning(); + await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" }); + const svc = issueService(ctx.db); + const list = await svc.list(f.companyId, { includeBlockedBy: true }); + expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]); + const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200); + expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]); + }); + + it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => { + const f = await fixture(); + const runId = randomUUID(); + await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" }); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId)); + await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring"); + await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" }); + }); + + it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => { + const f = await fixture(); + await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe(status); + }); + + it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => { + const f = await fixture(); + await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + }); + + it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => { + const f = await fixture(); + await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" }); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`); + expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring"); + await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + }); +}); diff --git a/server/src/__tests__/issue-queued-comments-routes.test.ts b/server/src/__tests__/issue-queued-comments-routes.test.ts index 1d0467105af..fed13ba5780 100644 --- a/server/src/__tests__/issue-queued-comments-routes.test.ts +++ b/server/src/__tests__/issue-queued-comments-routes.test.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agentRuntimeState, @@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)) @@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { }).where(eq(heartbeatRuns.companyId, f.companyId)); const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200); @@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { })), ]); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200); diff --git a/server/src/__tests__/issue-summaries-routes.test.ts b/server/src/__tests__/issue-summaries-routes.test.ts index c832d92af03..f82446cd20c 100644 --- a/server/src/__tests__/issue-summaries-routes.test.ts +++ b/server/src/__tests__/issue-summaries-routes.test.ts @@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => { expect(res.headers["cache-control"]).toBe("private, no-store"); expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]); expect(Object.keys(res.body[0]).sort()).toEqual([ - "companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt", + "companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt", ]); expect(res.body[0].externalConversationState).toBeNull(); expect(forbiddenRead).not.toHaveBeenCalled(); diff --git a/server/src/__tests__/issues-list-query-parsing.test.ts b/server/src/__tests__/issues-list-query-parsing.test.ts index 91024b93d05..9a369ce0afe 100644 --- a/server/src/__tests__/issues-list-query-parsing.test.ts +++ b/server/src/__tests__/issues-list-query-parsing.test.ts @@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => { it("returns every status when ?status is absent", async () => { expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]); }); + it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting"); + await request(routeApp(ctx.db, actor, issueRoutes)) + .get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`) + .expect(422); + }); + + it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting"); + const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`); + await ctx.db.insert(issues).values([ + ...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo", + createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"), + updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })), + { companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") }, + ]); + const app = routeApp(ctx.db, actor, issueRoutes); + const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`; + for (const sortDir of ["asc", "desc"]) { + const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200); + const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200); + expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse()); + } + await request(app).get(`${path}&sortDir=invalid`).expect(400); + }); + }); diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts index 65e0e0cc147..38d748d1a36 100644 --- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts +++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db"; @@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, }).where(eq(heartbeatRuns.id, f.runId)).returning(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let returned: Awaited>; try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); } finally { capture.flush(); } - const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"); const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select"); expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1); expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096); @@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } }, }))!; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let returned: Awaited>; try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly, @@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde ...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}), }); } finally { capture.flush(); } - const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && event.operation === "update"); expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1); expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt, diff --git a/server/src/__tests__/native-environment-settlement-work.test.ts b/server/src/__tests__/native-environment-settlement-work.test.ts index 7a52bc5304d..4fd2448b9a4 100644 --- a/server/src/__tests__/native-environment-settlement-work.test.ts +++ b/server/src/__tests__/native-environment-settlement-work.test.ts @@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { heartbeatService } from "../services/heartbeat.js"; import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js"; @@ -180,7 +180,7 @@ describe("native environment release scheduling", () => { } const events: Record[] = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture); const settle = vi.fn(async () => {}); const work = nativeEnvironmentSettlementWork(db, settle); diff --git a/server/src/__tests__/native-settled-process-release.test.ts b/server/src/__tests__/native-settled-process-release.test.ts index 1028d9abc07..a6805c61582 100644 --- a/server/src/__tests__/native-settled-process-release.test.ts +++ b/server/src/__tests__/native-settled-process-release.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { spawn } from "node:child_process"; import os from "node:os"; import { and, eq, sql } from "drizzle-orm"; @@ -222,11 +222,11 @@ suite("settled native run process identity", () => { const saved = await readRun(runId); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.responseValueBytes).toBeLessThan(8192); const after = await readRun(runId); expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null, diff --git a/server/src/__tests__/plugin-host-services-span.test.ts b/server/src/__tests__/plugin-host-services-span.test.ts index 01b7eb06c7a..a91e6c4e626 100644 --- a/server/src/__tests__/plugin-host-services-span.test.ts +++ b/server/src/__tests__/plugin-host-services-span.test.ts @@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => { }); describe("nested worker timing receipts",()=>{ + it("keeps lease verification beneath its worker and clamps provider state", () => { + mockRecordSpan.mockReset(); + const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 }; + const verifyParent = { ...rootParent, spanId: "2222222222222222" }; + mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent); + const now = Date.now(); + recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [ + { name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [ + { name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20, + attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } }, + ] }, + ] }, { traceparent: VALID_TRACEPARENT }); + expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent }); + expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent, + startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } }); + expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token"); + expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({}); + }); + + it("admits every closed lease phase and rejects dynamic phase names", () => { + mockRecordSpan.mockReset(); + const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup", + "lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell", + "lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"]; + for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT }); + expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([ + ...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other", + ]); + }); + it("uses host-generated parents, clamps content and bounds child clocks",()=>{ mockRecordSpan.mockReset(); const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1}; diff --git a/server/src/__tests__/redaction.test.ts b/server/src/__tests__/redaction.test.ts index 4ab09bbd80e..5d060d67d1a 100644 --- a/server/src/__tests__/redaction.test.ts +++ b/server/src/__tests__/redaction.test.ts @@ -899,3 +899,73 @@ second-line\" status=401`, }); }); }); + +describe("redaction of long letter/digit runs", () => { + const typescriptSource = [ + "export function compactRunLogChunk(chunk: string, maxChars = 65536) {", + " const normalized = redactSensitiveText(chunk);", + " if (normalized.length <= maxChars) return normalized;", + " return normalized.slice(0, maxChars);", + "}", + "", + ].join("\n"); + + function base64OfTypeScriptSource(length: number) { + const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length)); + return Buffer.from(repeatedSource).toString("base64").slice(0, length); + } + + function redactWithinTimeBudget(input: string) { + const startedAt = performance.now(); + const output = redactSensitiveText(input); + expect(performance.now() - startedAt).toBeLessThan(1_000); + return output; + } + + it("redacts secret field names that carry a prefix in plain and escaped JSON", () => { + const cases = [ + [`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`], + [`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`], + [`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`], + [`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`], + [`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`], + [`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`], + [String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`], + [ + String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`, + String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`, + ], + [String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`], + ]; + + for (const [input, expected] of cases) { + expect(redactSensitiveText(input)).toBe(expected); + expect(redactSensitiveText(expected)).toBe(expected); + } + }); + + it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => { + const blob = base64OfTypeScriptSource(48 * 1024); + expect(blob).toMatch(/^[A-Za-z0-9]+$/); + + const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`; + expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand); + + const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } }); + expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine); + const serializedToolCallLine = JSON.stringify(toolCallLine); + expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine); + + const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`); + expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`); + }); + + it("scans long dash-separated runs in linear time", () => { + for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) { + expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`)) + .toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`); + expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`)) + .toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`); + } + }); +}); diff --git a/server/src/__tests__/sandbox-provider-schema-reads.test.ts b/server/src/__tests__/sandbox-provider-schema-reads.test.ts index 55f81f8bfbe..fd82c852bbc 100644 --- a/server/src/__tests__/sandbox-provider-schema-reads.test.ts +++ b/server/src/__tests__/sandbox-provider-schema-reads.test.ts @@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { eq, sql, type SQL } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { createDb, plugins, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared"; import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js"; import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js"; @@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => { const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey }); async function measure(client: Db, operation: () => Promise) { const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now); installDatabaseResponseCapture(client, { plugins }, capture); try { const value = await operation(); capture.flush(); - const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"); return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0), calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) }; } finally { capture.flush(); } diff --git a/server/src/__tests__/tool-effective-assignments.test.ts b/server/src/__tests__/tool-effective-assignments.test.ts index 4b99a23eec2..88534789025 100644 --- a/server/src/__tests__/tool-effective-assignments.test.ts +++ b/server/src/__tests__/tool-effective-assignments.test.ts @@ -5,7 +5,7 @@ import { agents, companies, createDb, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -149,14 +149,14 @@ describeDatabase("effective tool assignments", () => { const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id); expect(before.allowedTools).toHaveLength(102); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture); let scoped: Awaited>; try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); } finally { capture.flush(); } expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0); expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024); - const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_catalog_entries" && event.operation === "select"); expect(catalogQueries).toHaveLength(1); expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 }); @@ -192,13 +192,13 @@ describeDatabase("effective tool assignments", () => { const service = toolAccessService(db), before = await compare(f); const savedRules = await db.select().from(toolProfileEntries); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture); try { const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); capture.flush(); - const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries"); - const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings"); + const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries"); + const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings"); expect(rules).toHaveLength(1); expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096); expect(rules[0]).toMatchObject({ calls: 1, rows: 2 }); diff --git a/server/src/__tests__/voice-call-reports.test.ts b/server/src/__tests__/voice-call-reports.test.ts index fa614a5ed8a..0714158493c 100644 --- a/server/src/__tests__/voice-call-reports.test.ts +++ b/server/src/__tests__/voice-call-reports.test.ts @@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await quiet(); expect(fetchReport).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(deadline - Date.now()); await vi.waitFor(async () => { - await vi.advanceTimersByTimeAsync(1000); + // Keep time fixed during database I/O. Advancing it on every poll can + // make the first reports due again before the backlog has settled. + const next = coordinator!.nextWakeAt(); + if (next !== null && next < deadline + 60_000) { + await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now())); + } expect(idleWorkSnapshot().active).toBe(0); expect(fetchReport).toHaveBeenCalledTimes(105); - }, { timeout: 5000 }); + }, { interval: 0, timeout: 15_000 }); expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort()); expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId))) .every(row => row.attempts === 1)).toBe(true); await quiet(); } finally { reads.mockRestore(); } - }); + }, 30_000); it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => { await session({ state: "active", endedAt: null }); await session({ endedAt: new Date(Date.now() - day - 1000) }); diff --git a/server/src/__tests__/voice-inbound-retention.test.ts b/server/src/__tests__/voice-inbound-retention.test.ts index 6bdedd256d5..6925c795d39 100644 --- a/server/src/__tests__/voice-inbound-retention.test.ts +++ b/server/src/__tests__/voice-inbound-retention.test.ts @@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet(); }); it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000; await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`); const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id }); diff --git a/server/src/app.ts b/server/src/app.ts index 06adc15edb8..17b4e63e127 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -12,6 +12,7 @@ import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; import { createLifecycleDriver } from "./services/agent-lifecycle-driver.js"; import { startAgentLifecycle } from "./services/agent-lifecycle.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; +import { databaseOperationMiddleware } from "./middleware/database-operation.js"; import { staticUiFiles } from "./middleware/static-ui.js"; import { retainedUiAssets } from "./retained-ui-assets.js"; import { createStorageProviderFromConfig } from "./storage/provider-registry.js"; @@ -470,6 +471,7 @@ export async function createApp( }, ) { const app = express(); + app.use(databaseOperationMiddleware); app.use(idleAdmissionMiddleware); const staticUi = express.Router(); app.locals.paperclipDb = db; diff --git a/server/src/middleware/database-operation.ts b/server/src/middleware/database-operation.ts new file mode 100644 index 00000000000..b7cbc7abe01 --- /dev/null +++ b/server/src/middleware/database-operation.ts @@ -0,0 +1,5 @@ +import { withDatabaseOperation } from "@paperclipai/db"; +import type { RequestHandler } from "express"; + +export const databaseOperationMiddleware: RequestHandler = (_req, _res, next) => + withDatabaseOperation("http.request", next); diff --git a/server/src/modules/wake-queue/adapters/release-run-read.test.ts b/server/src/modules/wake-queue/adapters/release-run-read.test.ts index 6a669edf617..4ad78f2b1e0 100644 --- a/server/src/modules/wake-queue/adapters/release-run-read.test.ts +++ b/server/src/modules/wake-queue/adapters/release-run-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; @@ -40,7 +40,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => { const scope = { companyId, runId: before.id }; const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const native = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, scope)); @@ -51,7 +51,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => { expect(native.run).not.toHaveProperty("stdoutExcerpt"); expect(native.run).not.toHaveProperty("stderrExcerpt"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(8192); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before); @@ -97,12 +97,12 @@ describe.skipIf(!support.supported)("wake release run reads", () => { contextSnapshot: { ...context, ...launch }, resultJson: evidence }).returning(); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let selected: Awaited>; try { selected = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, { companyId, runId: before.id })); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(8192); expect(selected!.run?.contextSnapshot).toEqual(context); diff --git a/server/src/redaction.ts b/server/src/redaction.ts index 28693d13fa0..dd3edf95680 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -2,6 +2,7 @@ import { redactCommandText } from "@paperclipai/adapter-utils"; import { isPublicExecutorToolSelector, looksLikeCredentialJwt } from "@paperclipai/adapter-utils/command-redaction"; const SECRET_FIELD_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring|browser[-_]?code|login[-_]?url)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_FIELD_NAME_START = String.raw`(? Promise, patch: (row: Copy, values: Partial) => Promise, environmentRuntime?: EnvironmentRuntimeService) { +export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise, patch: (row: Copy, values: Partial, condition?: SQL) => Promise, environmentRuntime?: EnvironmentRuntimeService) { const store = agentFileStore(db); async function insertCopy(values: typeof copies.$inferInsert) { return db.transaction(async tx => { @@ -503,7 +503,15 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string return row.state === "superseded" || !row.processStoppedAt || completed.has(row.state) && row.receipt?.cleanupPending === false; } + async function settleCompletedCleanupRetry(row: Copy) { + if (completed.has(row.state) && row.processStoppedAt && !row.receipt?.retainedByRunId && row.receipt?.cleanupPending === false && row.nextAttemptAt) { + return patch(row, { nextAttemptAt: null }, sql`${copies.receipt} = ${JSON.stringify(row.receipt)}::jsonb + AND ${copies.processStoppedAt} IS NOT NULL`); + } + return row; + } async function release(row: Copy, target?: AdapterExecutionTarget | null) { + row = await settleCompletedCleanupRetry(row); if (releaseIsNoop(row) || !await owns(row)) return; const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true; if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return; @@ -559,7 +567,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string : eq(environmentLeases.environmentId, row.location.slice("remote:".length)), )); if (leases.length !== 1) return false; - const lease = leases[0]!; + const lease = await resolveRemoteTerminationLease(db, leases[0]!); if (lease.environmentId !== null && row.location !== `remote:${lease.environmentId}`) return false; const remoteCwd = cleanup?.remoteCwd ?? lease.metadata?.remoteCwd; return typeof remoteCwd === "string" && @@ -588,7 +596,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string cleanup?.leaseId ? eq(environmentLeases.id, cleanup.leaseId) : eq(environmentLeases.environmentId, row.location.slice("remote:".length)), )); if (leases.length !== 1) return null; - const lease = leases[0]!; + const lease = await resolveRemoteTerminationLease(db, leases[0]!); const destroyedWithoutEnvironment = lease.environmentId === null && hasRemoteTerminationReceipt(lease) && (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed"; if (row.location !== `remote:${lease.environmentId}` && !destroyedWithoutEnvironment) return null; @@ -658,10 +666,12 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string checkpointWarm: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => current.receipt?.warm === true ? checkpoint(current, target) : Promise.resolve(current), "agent_directory_checkpoint"), collectStopped: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => collectStopped(current, target), "agent_directory_collect"), release: async (row: Copy) => { - const current = await get(row.companyId, row.runId); + let current = await get(row.companyId, row.runId); + if (!current) return; + current = await settleCompletedCleanupRetry(current); // This check changes no receipt or bytes. A concurrent stop merely leaves // cleanup to its owner or the recovery sweep; it grants no new authority. - if (!current || releaseIsNoop(current)) return; + if (releaseIsNoop(current)) return; await serial(current, release, "agent_directory_release"); }, }; diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index 0d990331d4a..586bd63dbfb 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -129,8 +129,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen }; return db.transaction(tx => write(tx as unknown as Db)); } - async function patch(row: Copy, values: Partial) { - const updated = await tryPatch(row, values); + async function patch(row: Copy, values: Partial, condition?: SQL) { + const updated = await tryPatch(row, values, false, condition); // A late cleanup must not overwrite an explicit resolution or a newer // baseline. Canonical content has its own independent head CAS. return updated ?? (await get(row.companyId, row.runId))!; @@ -499,8 +499,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen function directoryCleanupEntry(row: Copy | null): DirectoryCleanupEntry | null { return row && ["saved", "unchanged", "resolved", "unavailable"].includes(row.state) && row.processStoppedAt && isAgentDirectoryCopy(row) && !row.receipt?.retainedByRunId - && row.receipt?.cleanupPending !== false - && (Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true) + && (row.receipt?.cleanupPending === false ? row.nextAttemptAt !== null + : Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true) ? { id: row.runId, companyId: row.companyId, dueAt: row.nextAttemptAt?.getTime() ?? 0 } : null; } function directoryCleanupPredicate() { @@ -508,8 +508,9 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen AND ${copies.processStoppedAt} IS NOT NULL AND ${copies.receipt}->>'schema' = 'paperclip.agent-files.v1' AND coalesce(${copies.receipt}->>'retainedByRunId', '') = '' - AND ${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false' - AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')`; + AND ((${copies.receipt}->>'cleanupPending' = 'false' AND ${copies.nextAttemptAt} IS NOT NULL) + OR (${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false' + AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')))`; } async function loadDirectoryCleanup(scope: DirectoryCleanupScope | undefined, admitted: () => boolean) { const entries: DirectoryCleanupEntry[] = []; diff --git a/server/src/services/budget-tracing.ts b/server/src/services/budget-tracing.ts index c0f525a61b1..f4b9212ca8f 100644 --- a/server/src/services/budget-tracing.ts +++ b/server/src/services/budget-tracing.ts @@ -1,18 +1,27 @@ import { trace, type Span } from "@opentelemetry/api"; +import { withDatabaseOperation } from "@paperclipai/db"; + +type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover" + | "budget.accounting.settlement" | "budget.accounting.transaction" + | "budget.admission.evaluate" | "budget.admission.reserve" + | "budget.cancellation.deliver" | "budget.enforcement.deliver" + | "budget.enforcement.scope" | "budget.exposure.evaluate"; /** Operator-gated OpenTelemetry only; these spans contain no prompt, receipt, * credential, or provider error text. Diagnostics never change accounting. */ -export async function withBudgetSpan(name: string, work: () => Promise, attributes?: Record) { - if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work(); - let operation: Promise | undefined; - const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => { - try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ } +export function withBudgetSpan(name: BudgetOperation, work: () => Promise, attributes?: Record) { + return withDatabaseOperation(name, async () => { + if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work(); + let operation: Promise | undefined; + const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => { + try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ } + }); + try { + const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span)); + // Some external SDK implementations can fail before or after the callback. + // Own the accounting promise so either failure still executes work once. + void Promise.resolve(traced).catch(() => undefined); + } catch { /* Preserve the accounting operation if span creation fails. */ } + return operation ?? run(); }); - try { - const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span)); - // Some external SDK implementations can fail before or after the callback. - // Own the accounting promise so either failure still executes work once. - void Promise.resolve(traced).catch(() => undefined); - } catch { /* Preserve the accounting operation if span creation fails. */ } - return operation ?? run(); } diff --git a/server/src/services/chat-channels/authorization-run-read.test.ts b/server/src/services/chat-channels/authorization-run-read.test.ts index 3d56b695222..5e573a5974e 100644 --- a/server/src/services/chat-channels/authorization-run-read.test.ts +++ b/server/src/services/chat-channels/authorization-run-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -49,7 +49,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => { const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId)); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const selected = await db.transaction(tx => readChatAuthorizationRun(tx, scope, true)); @@ -58,7 +58,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => { expect(selected.run).not.toHaveProperty("runnerProfileJson"); expect(selected.run).not.toHaveProperty("stdoutExcerpt"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBeGreaterThan(0); expect(summary.responseValueBytes).toBeLessThan(4096); const retry = await db.transaction(tx => readChatAuthorizationRun(tx, scope, false)); diff --git a/server/src/services/chat-channels/github/webhook-ingress.test.ts b/server/src/services/chat-channels/github/webhook-ingress.test.ts index ab868d63c5c..975f0b810de 100644 --- a/server/src/services/chat-channels/github/webhook-ingress.test.ts +++ b/server/src/services/chat-channels/github/webhook-ingress.test.ts @@ -2,7 +2,7 @@ import { createHmac } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { applyPendingMigrations, chatActions, createDb } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; import { cleanupIngressFixture, githubIngressFixture as createFixture } from "../../../__tests__/helpers/github-webhook-ingress.js"; @@ -36,7 +36,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh const f = await githubIngressFixture(db, createGitHubWebhookIngress, 768 * 1024); const action = await f.insertAction(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions }, capture); try { expect((await f.service.processGitHubWebhookIngress(action.id))?.status).toBe(202); } finally { capture.flush(); } @@ -59,7 +59,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh // A corrupt body must not let a nonclaimant overwrite a live or delayed claim. const action = await f.insertAction({ status, result, payload: { ...f.payload, bodySha256: "0".repeat(64) } }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions }, capture); let response: Response | null; try { response = await f.service.processGitHubWebhookIngress(action.id); } @@ -67,7 +67,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh expect(response?.status ?? null).toBe(accepted ? 202 : null); expect(await f.readAction(action.id)).toEqual(action); expect(f.calls).toEqual([]); - expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024); + expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024); }); it.each(["received", "failed", "processing"])("settles eligible malformed %s payloads without replay", async status => { diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index fb27743e0d2..17055aa3146 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { externalConversationStateSql } from "./slack-conversation-state.js"; import { and, desc, eq, gte, inArray, isNotNull, isNull, notInArray, or, sql } from "drizzle-orm"; import type { SQL } from "drizzle-orm"; @@ -21,6 +22,7 @@ import { COMPANY_ARTIFACTS_MAX_QUERY_LENGTH, ISSUE_PRIORITIES, ISSUE_STATUSES, + ISSUE_DISPLAY_STATUSES, SYSTEM_ISSUE_DOCUMENT_KEYS, type CompanyArtifact, type CompanySearchArtifactSummary, @@ -50,6 +52,7 @@ type IssueSearchRow = { title: string; description: string | null; status: string; + displayStatus: string; priority: string; assigneeAgentId: string | null; assigneeUserId: string | null; @@ -193,6 +196,7 @@ function emptySearchCounts(): Record { function emptyFilterOptionCounts(): CompanySearchFilterOptionCounts { return { status: {}, + displayStatus: {}, priority: {}, assigneeAgentId: {}, assigneeUserId: {}, @@ -224,7 +228,7 @@ function updatedWithinStart(value: string | undefined, now = new Date()): Date | } function issueOnlyFiltersActive(query: CompanySearchQuery) { - return query.status.length > 0 + return (query.status.length > 0 || query.displayStatus.length > 0) || query.priority.length > 0 || query.assigneeAgentId !== undefined || Boolean(query.assigneeUserId) @@ -236,7 +240,7 @@ function issueOnlyFiltersActive(query: CompanySearchQuery) { function activeIssueFilters(query: CompanySearchQuery): Array<{ key: CompanySearchIssueFilterKey; values: string[] }> { const filters: Array<{ key: CompanySearchIssueFilterKey; values: string[] }> = []; - if (query.status.length > 0) filters.push({ key: "status", values: query.status }); + if (query.status.length > 0 || query.displayStatus.length > 0) filters.push({ key: "status", values: [...query.status, ...query.displayStatus] }); if (query.assigneeAgentId !== undefined) filters.push({ key: "assigneeAgentId", values: [query.assigneeAgentId ?? "null"] }); if (query.assigneeUserId) filters.push({ key: "assigneeUserId", values: [query.assigneeUserId] }); if (query.projectId) filters.push({ key: "projectId", values: [query.projectId] }); @@ -251,6 +255,7 @@ function queryWithoutFilter(query: CompanySearchQuery, key: CompanySearchIssueFi return { ...query, status: key === "status" ? [] : query.status, + displayStatus: key === "status" ? [] : query.displayStatus, priority: key === "priority" ? [] : query.priority, assigneeAgentId: key === "assigneeAgentId" ? undefined : query.assigneeAgentId, assigneeUserId: key === "assigneeUserId" ? undefined : query.assigneeUserId, @@ -265,6 +270,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery return { ...query, status: [], + displayStatus: [], priority: [], assigneeAgentId: undefined, assigneeUserId: undefined, @@ -277,6 +283,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery function issueFilterConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] { const conditions: SQL[] = []; + if (omit !== "status" && query.displayStatus.length) conditions.push(inArray(issueDisplayStatusSql(), query.displayStatus)); if (omit !== "status" && query.status.length > 0) { conditions.push(query.status.length === 1 ? eq(issues.status, query.status[0]!) : inArray(issues.status, query.status)); } @@ -315,6 +322,7 @@ function issueFilterConditions(companyId: string, query: CompanySearchQuery, omi // matched-issues CTE (plain columns, no drizzle table references). function matchedFacetConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] { const conditions: SQL[] = []; + if (omit !== "status" && query.displayStatus.length) conditions.push(sql`m.display_status = ANY(${sqlTextArray(query.displayStatus)})`); if (omit !== "status" && query.status.length > 0) { conditions.push(sql`m.status = ANY(${sqlTextArray(query.status)})`); } @@ -447,6 +455,7 @@ function issueResult(row: IssueSearchRow, prefix: string, normalizedQuery: strin identifier: row.identifier, title: row.title, status: row.status as CompanySearchIssueSummary["status"], + displayStatus: row.displayStatus as CompanySearchIssueSummary["displayStatus"], priority: row.priority as CompanySearchIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -626,7 +635,7 @@ export function companySearchService(db: Db) { conditions.length > 0 ? sql`WHERE ${sql.join(conditions, sql` AND `)}` : sql``; // Count branches must match the result branch's column list; the // trailing NULLs pad the issue data columns. - const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`; + const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`; const branches: SQL[] = []; const wantResultRows = scopeIncludesIssues(scope) @@ -652,6 +661,7 @@ export function companySearchService(db: Db) { (SELECT issue_text.description FROM issues issue_text WHERE issue_text.id = m.id AND issue_text.company_id = ${companyId}) AS description, m.status, + m.display_status AS "displayStatus", m.priority, m.assignee_agent_id AS "assigneeAgentId", m.assignee_user_id AS "assigneeUserId", @@ -690,6 +700,7 @@ export function companySearchService(db: Db) { GROUP BY 2 `; branches.push(facetBranch("facet:status", sql`m.status`, "status")); + branches.push(facetBranch("facet:displayStatus", sql`m.display_status`, "status")); branches.push(facetBranch("facet:priority", sql`m.priority`, "priority")); branches.push(facetBranch("facet:assigneeAgentId", sql`m.assignee_agent_id`, "assigneeAgentId", [sql`m.assignee_agent_id IS NOT NULL`])); branches.push(facetBranch("facet:assigneeUserId", sql`m.assignee_user_id`, "assigneeUserId", [sql`m.assignee_user_id IS NOT NULL`])); @@ -757,6 +768,7 @@ export function companySearchService(db: Db) { title: row.title, description: row.description, status: row.status, + displayStatus: row.displayStatus, priority: row.priority, assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -780,6 +792,9 @@ export function companySearchService(db: Db) { if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; else if (row.kind === "type:document") aggregates.typeCounts.document = count; + else if (row.kind === "facet:displayStatus" && row.value && (ISSUE_DISPLAY_STATUSES as readonly string[]).includes(row.value)) { + aggregates.filterOptionCounts.displayStatus![row.value as keyof NonNullable] = count; + } else if (row.kind === "facet:status" && row.value && (ISSUE_STATUSES as readonly string[]).includes(row.value)) { aggregates.filterOptionCounts.status[row.value as keyof CompanySearchFilterOptionCounts["status"]] = count; } else if (row.kind === "facet:priority" && row.value && (ISSUE_PRIORITIES as readonly string[]).includes(row.value)) { diff --git a/server/src/services/delivery-work-coordinator.ts b/server/src/services/delivery-work-coordinator.ts index 3e1f1b95b2b..42638c33e34 100644 --- a/server/src/services/delivery-work-coordinator.ts +++ b/server/src/services/delivery-work-coordinator.ts @@ -1,4 +1,5 @@ import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db/work-signals"; +import { withDatabaseOperation } from "@paperclipai/db"; import { createWorkScheduler } from "./work-scheduler.js"; import { type DeliveryQueue } from "./delivery-work-notifications.js"; import { beginIdleTrackedWork } from "./task-admission.js"; @@ -70,7 +71,7 @@ export function createDeliveryWorkCoordinator(input: { const attempt = new AbortController(); let runFailed = false; controller = attempt; - running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) + running = withDatabaseOperation(`delivery.${queue}`, () => Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) .then(() => workerStopped || !(input.canRun() || (task.runDuringDrain && input.canReconcile?.())) ? undefined : task.run(attempt.signal).catch(error => { runFailed = true; throw error; })).then(() => workerStopped ? false : task.hasPending()).then(pending => { pendingWork = pending || databaseWorkPending(input.owner, queue); if (pendingWork) schedule(task.retryMs); @@ -98,7 +99,7 @@ export function createDeliveryWorkCoordinator(input: { controller = null; finish(); if (dirty) schedule(0); - }); + })); } function wake() { if (workerStopped) return; diff --git a/server/src/services/durable-chat-wakeup.ts b/server/src/services/durable-chat-wakeup.ts index 2870eac88ba..367386ffaca 100644 --- a/server/src/services/durable-chat-wakeup.ts +++ b/server/src/services/durable-chat-wakeup.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { agentWakeupRequests, chatActions, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db"; -import { and, eq, inArray, or, sql, type SQLWrapper } from "drizzle-orm"; +import { agentWakeupRequests, chatActions, chatConversations, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db"; +import { and, eq, exists, inArray, or, sql, type SQLWrapper } from "drizzle-orm"; import { HttpError } from "../errors.js"; export interface CommittedChatResponseAuthorizationInput { @@ -233,6 +233,22 @@ export interface DeferredChatWakeScope { issueId: string; requestId: string; } + +/** Identify saved input by its canonical receipt, never by a caller's source + * label. This is selection only; the inbound owner rechecks live authority. */ +export function deferredChatInputPredicate(db: Db) { + return exists(db.select({ id: chatActions.id }).from(chatActions) + .innerJoin(chatConversations, and(eq(chatConversations.companyId, chatActions.companyId), + eq(chatConversations.id, chatActions.conversationId))) + .innerJoin(chatEndpoints, and(eq(chatEndpoints.companyId, chatActions.companyId), + eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.id, chatConversations.endpointId))) + .where(and(eq(chatActions.companyId, agentWakeupRequests.companyId), eq(chatActions.id, agentWakeupRequests.id), + eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"), + eq(chatEndpoints.assignedAgentId, agentWakeupRequests.agentId), eq(chatEndpoints.externalExecutionPolicy, "restricted"), + sql`${chatConversations.issueId}::text = ${agentWakeupRequests.payload}->>'issueId'`, + sql`${chatActions.payload}->>'issueId' = ${agentWakeupRequests.payload}->>'issueId'`, + sql`${chatActions.payload}->>'agentId' = ${agentWakeupRequests.agentId}::text`))); +} const deferredChatWakeOwners = new WeakMap Promise>(); /** The live inbound owner reauthorizes the original source. Cleanup owns no chat authority. */ diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index 9ee7f8a189e..4ba7aa69457 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from "node:async_hooks"; +import { withDatabaseOperation } from "@paperclipai/db"; import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { createHash } from "node:crypto"; import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api"; @@ -156,7 +157,7 @@ function measure(runId: string, name: string, work: () => T | Promise, ind ended: false, details: !independent && previous?.runId === runId ? previous.details : new Map(), }; - operation = Promise.resolve().then(() => + operation = withDatabaseOperation(name, () => Promise.resolve().then(() => ownership.run(owned, () => runWithRuntimeParent(owned.parent, work))).then(value => { try { span?.end(); } catch {} owned.ended = true; @@ -166,10 +167,10 @@ function measure(runId: string, name: string, work: () => T | Promise, ind try { span?.end(); } catch {} owned.ended = true; throw error; - }); + })); return operation; }; - if (!endpointEnabled()) return (async () => work())(); + if (!endpointEnabled()) return withDatabaseOperation(name, async () => work()); try { const parent = independent ? undefined : activeParent ?? getRunExecutionTraceContext(runId); const trigger = parent ? undefined : trace.getActiveSpan()?.spanContext(); @@ -202,20 +203,22 @@ export function withRunWorkspaceFinalizationTiming(runId: string, work: () => export function withRunResponsePresentationTiming(runId: string, work: () => T | Promise): Promise { const owned = ownership.getStore(); if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) { - return (async () => work())(); + return withDatabaseOperation("task.response_presentation", async () => work()); } return measure(runId, "task.response_presentation", work, true); } -export async function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise { - const owned = ownership.getStore(); - if (!endpointEnabled() || !owned || owned.ended) return work(); - const count = owned.details.get(phase) ?? 0; - if (count >= 16) return work(); - owned.details.set(phase, count + 1); - // A startup step can be more specific than the owning run phase. Keep that - // causal parent while the child publishes its own context to plugin workers. - return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined); +export function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise { + return withDatabaseOperation(phase, async () => { + const owned = ownership.getStore(); + if (!endpointEnabled() || !owned || owned.ended) return work(); + const count = owned.details.get(phase) ?? 0; + if (count >= 16) return work(); + owned.details.set(phase, count + 1); + // A startup step can be more specific than the owning run phase. Keep that + // causal parent while the child publishes its own context to plugin workers. + return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined); + }); } export function beginRunPhase(phase: ExecutionPhase): (failed?: boolean) => void { diff --git a/server/src/services/heartbeat/native-stop-evidence-read.test.ts b/server/src/services/heartbeat/native-stop-evidence-read.test.ts index d53a5d4c417..9a139a5cc85 100644 --- a/server/src/services/heartbeat/native-stop-evidence-read.test.ts +++ b/server/src/services/heartbeat/native-stop-evidence-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq, sql } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { isAcknowledgedNativeStop } from "../acknowledged-native-stop.js"; @@ -31,7 +31,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read contextSnapshot: { savedInput: bulk }, runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, resultJson: result }).returning(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let evidence: Awaited>; try { evidence = await readNativeStopExecutionEvidence(db, company.id, id); } @@ -68,7 +68,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read runtimeMode: "native", status: "cancelled" }).returning(); await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(saved)}::jsonb where id = ${run.id}`); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const evidence = await readNativeStopExecutionEvidence(db, company.id, run.id); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 18b5033ea39..9f0b0cf7aed 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2299,7 +2299,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`, )); const retry = readTaskRetryIntent(receipt?.payload); - if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || + if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || retry.queuedChatRequestId || retry.actorId !== receipt.requestedByActorId) return null; const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId && durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId; @@ -4369,7 +4369,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), )).then(rows => rows[0] ?? null) : null; - const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false; + // External input owns an exact authorized source batch. Board input + // must retain its separate receipt, including during normal resume. + const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false; // A resumed receipt already passed admission under this issue lock. // Consume it with its successor even when chat keeps other messages // in separate turns, or finalization will deliver it a second time. diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 0553b686a42..4b1c7c7af69 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -763,6 +763,10 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont eq(agentWakeupRequests.requestedByActorType, "user"), )); const intent = readTaskRetryIntent(wake?.payload); + if (intent?.queuedChatRequestId) { + await reconcileTaskChatRetryReceipts(db, { companyId, requestId }); + return; + } if (!wake || !intent || intent.chatActionId || intent.canonicalIntentId || intent.generationId !== wake.id || intent.actorId !== wake.requestedByActorId) return; const issueId = readNonEmptyString(wake.payload?.issueId); @@ -788,6 +792,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`, + sql`${agentWakeupRequests.payload}->'taskRetry'->>'queuedChatRequestId' is null`, notInArray(issues.status, ["done", "cancelled"]))) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const chatEnabled = !retries.some(retry => retry.conversationAgentId) diff --git a/server/src/services/heartbeat/run-state.test.ts b/server/src/services/heartbeat/run-state.test.ts index 28faa0efa6d..7769b779da8 100644 --- a/server/src/services/heartbeat/run-state.test.ts +++ b/server/src/services/heartbeat/run-state.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import type { Db } from "@paperclipai/db"; import { agents, @@ -186,13 +186,13 @@ describePostgres("heartbeat run state database wiring", () => { stdoutExcerpt: payload, stderrExcerpt: payload, }).returning(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); const read = await state.getRunExecutionWait(companyId, run.id); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(4096); expect(read).toMatchObject({ id: run.id, companyId, agentId: agent.id, status: "cancelled", contextSnapshot: context, @@ -228,7 +228,7 @@ describePostgres("heartbeat run state database wiring", () => { resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload, }).returning(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); @@ -236,7 +236,7 @@ describePostgres("heartbeat run state database wiring", () => { id: run.id, companyId, scopeKind: "issue", issueId: issue.id, executionWorkspaceId: workspaceId, }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(1024); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); @@ -260,7 +260,7 @@ describePostgres("heartbeat run state database wiring", () => { }).returning(); const observed = createDb(database.connectionString); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); @@ -270,7 +270,7 @@ describePostgres("heartbeat run state database wiring", () => { runtimeModeResolvedAt: null, startedAt: run.startedAt, createdAt: run.createdAt, contextSnapshot: { issueId: issue.id }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(2048); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); @@ -321,7 +321,7 @@ describePostgres("heartbeat run state database wiring", () => { await state.getRunWorkspaceSettlement(run.id); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); expect(await state.getRunCleanupState(run.id)).toEqual({ id: run.id, companyId, agentId: agent.id, runtimeMode: run.runtimeMode, @@ -334,7 +334,7 @@ describePostgres("heartbeat run state database wiring", () => { contextSnapshot: { timerClaimWasFirstHeartbeat: true }, }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(2000); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); diff --git a/server/src/services/issue-display-status.ts b/server/src/services/issue-display-status.ts new file mode 100644 index 00000000000..5c9bdefb54b --- /dev/null +++ b/server/src/services/issue-display-status.ts @@ -0,0 +1,37 @@ +import { sql, type SQL } from "drizzle-orm"; +import { issues } from "@paperclipai/db"; +import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME, type IssueDisplayStatus } from "@paperclipai/shared"; + +/** Read-only projection, shared by task reads, search, filters and counts. + * The persisted deadline is the scheduler's source of truth. Locks and recovery + * take precedence; a retained monitor alone must not disguise active work. + */ +export function issueDisplayStatusSql(): SQL { + // Drizzle removes column qualifiers in a single-table SELECT. Keep these + // outer references explicitly qualified so correlated reads cannot bind to + // the run/recovery table's own id or company_id instead. + const issueId = sql`"issues"."id"`; + const companyId = sql`"issues"."company_id"`; + return sql`CASE WHEN + ${issues.status} = 'in_progress' + AND ${issues.assigneeAgentId} IS NOT NULL AND ${issues.assigneeUserId} IS NULL + AND ${issues.monitorNextCheckAt} IS NOT NULL + AND ${issues.executionRunId} IS NULL AND ${issues.checkoutRunId} IS NULL + AND ${issues.executionState}->>'currentParticipant' IS NULL + AND COALESCE(${issues.executionState}#>>'{monitor,status}', 'scheduled') = 'scheduled' + AND ${issues.executionPolicy}->'monitor' IS NOT NULL + AND COALESCE(${issues.executionPolicy}#>>'{monitor,serviceName}', '') <> ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME} + AND NOT EXISTS ( + SELECT 1 FROM heartbeat_runs display_run + WHERE display_run.company_id = ${companyId} + AND display_run.issue_id = ${issueId} + AND display_run.status IN ('queued', 'running') + ) + AND NOT EXISTS ( + SELECT 1 FROM issue_recovery_actions display_recovery + WHERE display_recovery.company_id = ${companyId} + AND display_recovery.source_issue_id = ${issueId} + AND display_recovery.status IN ('active', 'escalated') + ) + THEN 'monitoring' ELSE ${issues.status} END`; +} diff --git a/server/src/services/issue-summaries.ts b/server/src/services/issue-summaries.ts index 485bb2250db..645ff8a86c2 100644 --- a/server/src/services/issue-summaries.ts +++ b/server/src/services/issue-summaries.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { and, eq, inArray, isNull, type SQL } from "drizzle-orm"; import { issues, type Db } from "@paperclipai/db"; import { ISSUE_SUMMARIES_MAX_IDS } from "@paperclipai/shared"; @@ -18,6 +19,7 @@ export async function listIssueSummaries( title: issues.title, identifier: issues.identifier, status: issues.status, + displayStatus: issueDisplayStatusSql(), updatedAt: issues.updatedAt, externalConversationState: externalConversationStateSql(), projectId: issues.projectId, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index ecdb45c43a4..9eb64654b46 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,3 +1,5 @@ +import type { IssueDisplayStatus } from "@paperclipai/shared"; +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js"; import { finishedIdleWake } from "./idle-work-proof.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -1900,6 +1902,7 @@ export function parseStatusFilter( export interface IssueFilters { attention?: "blocked"; status?: string | readonly string[]; + displayStatus?: string | readonly string[]; /** * Filter by assignee agent ID. * - `string` (UUID): match issues assigned to that agent. @@ -1944,14 +1947,14 @@ export interface IssueFilters { q?: string; limit?: number; offset?: number; - sortField?: "updated" | "id"; + sortField?: "updated" | "created" | "id"; afterId?: string; sortDir?: "asc" | "desc"; /** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */ updatedSince?: string; } -type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null }; +type IssueRow = typeof issues.$inferSelect & { displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null }; type IssueLabelRow = typeof labels.$inferSelect; type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect; type IssueActiveRunRow = { @@ -3218,6 +3221,11 @@ function issueListOrderBy( }, ) { if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)]; + if (sortField === "created") { + return sortDir === "asc" + ? [asc(issues.createdAt), asc(issues.id)] + : [desc(issues.createdAt), desc(issues.id)]; + } const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId); if (sortField === "updated") { const activityOrder = @@ -3603,6 +3611,7 @@ function appendBlockerAttentionEdges( } type IssueRelationSummaryRow = { + displayStatus?: IssueDisplayStatus; relatedId: string; identifier: string | null; title: string; @@ -3620,6 +3629,7 @@ function summarizeIssueRelationRow( identifier: row.identifier, title: row.title, status: row.status as IssueRelationIssueSummary["status"], + displayStatus: row.displayStatus, priority: row.priority as IssueRelationIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -3735,6 +3745,7 @@ async function terminalExplicitBlockersByRoot( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -5039,6 +5050,7 @@ const issueListSelect = { END `, status: issues.status, + displayStatus: issueDisplayStatusSql(), statusVersion: issues.statusVersion, lastStatusDecisionId: issues.lastStatusDecisionId, workMode: issues.workMode, @@ -5279,6 +5291,7 @@ async function blockedByMapForIssues( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -5301,6 +5314,7 @@ async function blockedByMapForIssues( identifier: row.identifier, title: row.title, status: row.status as IssueRelationIssueSummary["status"], + displayStatus: row.displayStatus, priority: row.priority as IssueRelationIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -6420,6 +6434,8 @@ async function blockedInboxIssueConditions( filters?.lowTrustBoundary, ); if (lowTrustCondition) conditions.push(lowTrustCondition); + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length > 0) { conditions.push( @@ -6872,7 +6888,7 @@ export function issueService(db: Db) { async function getIssueByUuid(id: string) { const row = await retryIdempotentDatabaseOperation(() => db - .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() }) .from(issues) .where(eq(issues.id, id)) .then((rows) => rows[0] ?? null)); @@ -6883,7 +6899,7 @@ export function issueService(db: Db) { async function getIssueByIdentifier(identifier: string) { const row = await retryIdempotentDatabaseOperation(() => db - .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() }) .from(issues) .where(eq(issues.identifier, identifier.toUpperCase())) .then((rows) => rows[0] ?? null)); @@ -7482,6 +7498,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -7502,6 +7519,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -8193,6 +8211,9 @@ export function issueService(db: Db) { if (filters?.sortField === "id" && filters.attention) { throw unprocessable("ID ordering is not supported for blocked attention lists"); } + if (filters?.sortField === "created" && filters.attention) { + throw unprocessable("Creation-time ordering is not supported for blocked attention lists"); + } if (filters?.afterId !== undefined && ( !isUuidLike(filters.afterId) || filters.sortField !== "id" || filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0 @@ -8281,6 +8302,8 @@ export function issueService(db: Db) { filters?.lowTrustBoundary, ); if (lowTrustCondition) conditions.push(lowTrustCondition); + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length === 1) { conditions.push(eq(issues.status, statuses[0])); @@ -8574,6 +8597,8 @@ export function issueService(db: Db) { conditions.push(nonIdleSlackIssueCondition()); } } + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length === 1) conditions.push(eq(issues.status, statuses[0]!)); @@ -8895,6 +8920,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -9522,6 +9548,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, diff --git a/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts b/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts index 7caef7cf216..6f9a222e39e 100644 --- a/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts +++ b/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts @@ -1,5 +1,5 @@ import { heartbeatRuns, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { expect } from "vitest"; @@ -21,11 +21,11 @@ export async function verifyCommittedReplayPayloadReads( const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { await replay(); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBeGreaterThan(0); expect(summary.responseValueBytes).toBeLessThan(100_000); const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); diff --git a/server/src/services/native-runtime/assigned-mcp-tools.test.ts b/server/src/services/native-runtime/assigned-mcp-tools.test.ts index a5708ba1668..cfcef3b18e2 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.test.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.test.ts @@ -19,6 +19,50 @@ describe("assigned MCP runner tools", () => { const searchName = "paperclip_search_assigned_tools"; const callName = "paperclip_call_assigned_tool"; + it.each(["", " \t\n"])("discovers and calls assigned tools through MCP with a blank schema reference %j", async schemaTool => { + const github = descriptor("github_bot:read_pull_request"); + const f = fixture([github, descriptor("calendar.search")]); + const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" }); + const bridge = await startOpenCodeMcpBridge({ tools: assigned.definitions(), handler: call => assigned.execute(call) }); + let id = 0; + const call = async (name: string, args: Record) => { + const response = await fetch(bridge.url, { + method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }), + }); + const result = (await response.json()).result; + expect(result.isError).not.toBe(true); + return JSON.parse(result.content[0].text); + }; + try { + const page = await call(searchName, { query: "github", schemaTool }); + expect(page.tools).toHaveLength(1); + expect(page.tools[0]).toMatchObject({ description: expect.stringContaining(github.name), inputSchema: github.parametersSchema }); + expect(await call(searchName, { query: "github", schemaTool, schemaOffset: 0 })).toEqual(page); + expect((await call(searchName, { query: "", schemaTool })).tools).toHaveLength(2); + await call(callName, { name: page.tools[0].name, arguments: { query: "fixture" } }); + expect(f.executeTool).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ + tool: github.name, parameters: { query: "fixture" }, gatewayPublicId: f.gatewayPublicId, sessionToken: f.bearerToken, + })); + f.listToolsForNamedGateway.mockResolvedValue([]); + expect(await call(searchName, { query: "github", schemaTool })).toEqual({ tools: [], nextOffset: null }); + } finally { + await bridge.close(); + } + }); + + it("explains how to recover from an unknown schema reference or an offset without a reference", async () => { + const f = fixture([descriptor("github_bot:read_pull_request")]); + const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" }); + await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool: "unknown" } })) + .rejects.toThrow("Omit schemaTool to search assigned tools"); + for (const schemaTool of [undefined, "", " \t\n"]) { + await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool, schemaOffset: 1 } })) + .rejects.toThrow("A positive schemaOffset requires an exact inputSchemaRef in schemaTool"); + } + expect(f.executeTool).not.toHaveBeenCalled(); + }); + it.each([undefined, "on_demand"] as const)("advertises usable page limits through MCP discovery (%s)", async delivery => { const f = fixture(Array.from({ length: 25 }, (_, i) => descriptor(`operation_${i}`))); const assigned = await createAssignedMcpTools({ ...f, delivery }); diff --git a/server/src/services/native-runtime/assigned-mcp-tools.ts b/server/src/services/native-runtime/assigned-mcp-tools.ts index 00473c3b5d5..6e5bf20bf23 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.ts @@ -21,8 +21,8 @@ const ON_DEMAND_TOOLS: ToolDefinition[] = [ query: { type: "string", maxLength: 200 }, offset: { type: "integer", minimum: 0 }, limit: { type: "integer", minimum: 1, maximum: 20, default: 5, description: "Maximum results per page (1 to 20; default 5)." }, - schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result." }, - schemaOffset: { type: "integer", minimum: 0 }, + schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result. Omit or leave blank for a normal tool search." }, + schemaOffset: { type: "integer", minimum: 0, description: "Schema chunk offset. A positive offset requires an exact inputSchemaRef in schemaTool." }, }, required: ["query"], }, @@ -116,14 +116,17 @@ export async function createAssignedMcpTools(input: { const offset = args.offset ?? 0; const limit = args.limit ?? 5; const schemaOffset = args.schemaOffset ?? 0; + const hasSchemaTool = typeof args.schemaTool === "string" && args.schemaTool.trim().length > 0; if (typeof args.query !== "string" || args.query.length > 200 || typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 || typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 20 || (args.schemaTool !== undefined && typeof args.schemaTool !== "string") || - typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0 || - (args.schemaOffset !== undefined && args.schemaTool === undefined)) { + typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0) { throw new Error("assigned_mcp_tool_invalid_arguments"); } + if (schemaOffset > 0 && !hasSchemaTool) { + throw new Error("assigned_mcp_tool_invalid_arguments: A positive schemaOffset requires an exact inputSchemaRef in schemaTool. Omit schemaTool and schemaOffset to search assigned tools."); + } // Intersect fresh grants with this session's pinned catalog. Revocations // take effect immediately; newly assigned tools require a new session. const current = new Map(deduplicateToolDescriptors(await input.gateway.listToolsForNamedGateway({ @@ -135,9 +138,9 @@ export async function createAssignedMcpTools(input: { return fresh && permits(tool) && permits(tool, currentWorkMode) && permits(fresh) && permits(fresh, currentWorkMode); }); - if (typeof args.schemaTool === "string") { + if (hasSchemaTool) { const selected = authorized.find(([name]) => name === args.schemaTool); - if (!selected) throw new Error("assigned_mcp_tool_unknown"); + if (!selected) throw new Error("assigned_mcp_tool_unknown: schemaTool must match an authorized inputSchemaRef. Omit schemaTool to search assigned tools, then use an exact inputSchemaRef from the results."); const schema = JSON.stringify(selected[1].parametersSchema); const end = schemaOffset + schemaChunkCharacters; return { diff --git a/server/src/services/native-runtime/chat-attachment-reuse.test.ts b/server/src/services/native-runtime/chat-attachment-reuse.test.ts index d0de5ab04c9..7870a8033d9 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.test.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.test.ts @@ -29,7 +29,7 @@ import { toolApplications, toolConnections, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; @@ -322,7 +322,7 @@ describe("native same-conversation chat attachment reuse", () => { const deliveries = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); const bulk = randomBytes(64 * 1024).toString("base64"); const events: Array> = []; - let capture: ReturnType | undefined; + let capture: ReturnType | undefined; try { await db.update(chatEndpoints).set({ communicationInstructions: bulk, setup: { ...endpoint.setup, unusedContent: bulk } as typeof endpoint.setup, @@ -335,7 +335,7 @@ describe("native same-conversation chat attachment reuse", () => { .where(eq(chatDeliveries.id, delivery.id)); } const now = Date.now(); - capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatConversations, chatEndpoints, chatEndpointResources, chatExternalPrincipals, chatIdentityLinks, companyMemberships }, capture); for (const lockMode of ["read", "blocking", "nonblocking"] as const) { diff --git a/server/src/services/native-runtime/external-chat-wait.integration.test.ts b/server/src/services/native-runtime/external-chat-wait.integration.test.ts index f2951d3b8c0..3bb91b68a38 100644 --- a/server/src/services/native-runtime/external-chat-wait.integration.test.ts +++ b/server/src/services/native-runtime/external-chat-wait.integration.test.ts @@ -8,7 +8,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { and, eq, ne, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agentWakeupRequests, @@ -922,7 +922,7 @@ describe("native external-chat response wait", () => { await authorizeChatConversationForBoundRun(db, fixture, contextSnapshot, "read"); const observed = createDb(temporary?.connectionString ?? externalTestDatabaseUrl!); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { issues, heartbeatRuns }, capture); try { expect(await listExternalReviewWaits(observed, fixture.companyId, [fixture.issueId])).toEqual(governed ? [] : [ @@ -5462,7 +5462,7 @@ describe("native external-chat response wait", () => { if (reply) await db.update(chatActions).set({ result: { ...reply.result, savedProviderOutput: payload } }).where(eq(chatActions.id, reply.id)); const observed = createDb(temporary?.connectionString ?? externalTestDatabaseUrl!); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, chatActions }, capture); try { const context = await new PaperclipRunnerToolAuthority(observed, fixture) diff --git a/server/src/services/native-runtime/native-controller-identity.test.ts b/server/src/services/native-runtime/native-controller-identity.test.ts new file mode 100644 index 00000000000..fd11591da9a --- /dev/null +++ b/server/src/services/native-runtime/native-controller-identity.test.ts @@ -0,0 +1,37 @@ +import { afterEach, expect, it, vi } from "vitest"; +import * as childProcess from "node:child_process"; +import * as hotRestart from "../hot-restart.js"; +import { getServerInfoSnapshot, resetServerInfoCacheForTests } from "../../server-info.js"; +import { currentNativeControllerIdentity } from "./native-restart-recovery.js"; + +vi.mock("node:child_process", async importOriginal => ({ + ...await importOriginal(), + execFileSync: vi.fn(() => { + throw new Error("Git metadata is unavailable; identity must still work"); + }), +})); + +afterEach(() => { + vi.restoreAllMocks(); + resetServerInfoCacheForTests(); +}); + +it.each([true, false])("keeps controller identity independent of Git commands (OS start available=%s)", async available => { + const observed = "2026-11-02T12:00:00.000Z"; + const fallback = getServerInfoSnapshot({ gitCommand: () => { throw new Error("no checkout"); } }).processStartedAt; + resetServerInfoCacheForTests(); + vi.spyOn(hotRestart, "readProcessStartedAt").mockImplementation(async () => { + if (!available) throw new Error("process start unavailable"); + return observed; + }); + const subprocess = vi.mocked(childProcess.execFileSync); + subprocess.mockClear(); + + const first = await currentNativeControllerIdentity(); + const second = await currentNativeControllerIdentity(); + + expect(first).toEqual(second); + expect(first.pid).toBe(process.pid); + expect(first.processStartedAt.toISOString()).toBe(available ? observed : fallback); + expect(subprocess).not.toHaveBeenCalled(); +}); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts index 1c969758ac7..d698855a97f 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { expect, it } from "vitest"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; -it("removes retired backups containing immutable skills without modifying live files or symlink targets", () => { +it("removes retired backups containing immutable skills without modifying live files or symlink targets", async () => { const base = mkdtempSync(join(tmpdir(), "paperclip-backup-cleanup-")); const retired = join(base, "previous"); const nested = join(retired, "claude", "skills", "first-task"); @@ -19,12 +19,12 @@ it("removes retired backups containing immutable skills without modifying live f chmodSync(retired, 0o555); chmodSync(live, 0o555); try { - removeNativeHarnessBackup(retired); + await removeNativeHarnessBackup(retired); expect(existsSync(retired)).toBe(false); expect(readFileSync(join(live, "SKILL.md"), "utf8")).toBe("live instructions"); expect(lstatSync(live).mode & 0o777).toBe(0o555); expect(lstatSync(join(live, "SKILL.md")).mode & 0o777).toBe(0o444); - expect(() => removeNativeHarnessBackup(retired)).not.toThrow(); + await expect(removeNativeHarnessBackup(retired)).resolves.toBeUndefined(); } finally { for (const directory of [retired, join(retired, "claude", "skills"), nested, live]) { if (existsSync(directory)) chmodSync(directory, 0o700); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.ts index d52e50ba5c4..7a803704ea7 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.ts @@ -1,18 +1,23 @@ -import { chmodSync, lstatSync, readdirSync, rmSync } from "node:fs"; +import { chmod, lstat, readdir, rm } from "node:fs/promises"; import { join } from "node:path"; /** Remove a retired controller-owned backup, never a live harness directory. */ -export function removeNativeHarnessBackup(root: string): void { +export async function removeNativeHarnessBackup(root: string): Promise { // Archives preserve immutable skill directory modes. Deleting their children // requires write permission on those directories, even with rm's force flag. // Only change directories in the backup being discarded. lstat deliberately // avoids following provider-created symlinks into live or unrelated trees. - const prepareDirectory = (path: string): void => { - const metadata = lstatSync(path, { throwIfNoEntry: false }); + const prepareDirectory = async (path: string): Promise => { + const metadata = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); if (!metadata?.isDirectory()) return; - chmodSync(path, (metadata.mode & 0o777) | 0o700); - for (const name of readdirSync(path)) prepareDirectory(join(path, name)); + await chmod(path, (metadata.mode & 0o777) | 0o700); + for (const entry of await readdir(path, { withFileTypes: true })) { + if (entry.isDirectory()) await prepareDirectory(join(path, entry.name)); + } }; - prepareDirectory(root); - rmSync(root, { recursive: true, force: true }); + await prepareDirectory(root); + await rm(root, { recursive: true, force: true }); } diff --git a/server/src/services/native-runtime/native-harness-backup-publication.test.ts b/server/src/services/native-runtime/native-harness-backup-publication.test.ts index dd9d940d9df..df1029f55f9 100644 --- a/server/src/services/native-runtime/native-harness-backup-publication.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-publication.test.ts @@ -1,4 +1,6 @@ -import { promises as fs } from "node:fs"; +import { existsSync, promises as fs } from "node:fs"; +import { spawn } from "node:child_process"; +import { createServer, type ServerResponse } from "node:http"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -32,7 +34,17 @@ describe("native harness backup publication", () => { }); it("publishes the completed replacement once and removes its predecessor", async () => { const input = await fixture(); const changes: unknown[] = []; - unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + const previous = path.join(backupRoot, "previous"); + for (const root of [input.current, previous]) { + const skill = path.join(root, "skills", "immutable"); + await fs.mkdir(skill, { recursive: true }); + await fs.writeFile(path.join(skill, "SKILL.md"), "saved instructions", { mode: 0o444 }); + await fs.chmod(skill, 0o555); + } + unsubscribe = onNativeHarnessBackupChange(input.db, change => { + expect(existsSync(previous)).toBe(false); + changes.push(change); + }); await publishNativeHarnessBackup({ ...input, stamp: async current => { expect(changes).toEqual([]); expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); @@ -41,4 +53,85 @@ describe("native harness backup publication", () => { expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]); }); + it("serves an unrelated request while a retired backup is being removed", async () => { + const input = await fixture(); + for (let index = 0; index < 40; index++) { + const directory = path.join(input.current, `skill-${index}`); + await fs.mkdir(directory); + await Promise.all(Array.from({ length: 50 }, (_, file) => + fs.writeFile(path.join(directory, `file-${file}`), "retired instructions", { mode: 0o444 }))); + await fs.chmod(directory, 0o555); + } + let publishing = false; + let probeReceived: (response: ServerResponse | null) => void; + const probeReady = new Promise(resolve => { probeReceived = resolve; }); + const server = createServer((request, response) => { + if (request.url === "/probe") { probeReceived(response); return; } + response.end("ok"); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("HTTP probe did not start"); + const client = spawn(process.execPath, ["--input-type=module", "-e", ` + import { Agent, get } from "node:http"; + const agent = new Agent({ keepAlive: true }); + const request = path => new Promise((resolve, reject) => { + get({ hostname: "127.0.0.1", port: Number(process.argv[1]), path, agent }, response => { + response.resume(); + response.on("end", () => resolve(response.headers["x-publication-active"])); + }).on("error", reject); + }); + await request("/"); + process.send({ kind: "ready" }); + process.once("message", async () => { + const active = await request("/probe"); + process.send({ kind: "result", active }, () => { agent.destroy(); process.disconnect(); }); + }); + `, String(address.port)], { stdio: ["ignore", "ignore", "inherit", "ipc"] }); + const message = (kind: string) => new Promise<{ active?: string }>((resolve, reject) => { + const timer = setTimeout(() => { cleanup(); reject(new Error(`HTTP probe ${kind} timed out`)); }, 10_000); + const received = (value: { kind?: string; active?: string }) => { + if (value.kind === kind) { cleanup(); resolve(value); } + }; + const failed = () => { cleanup(); reject(new Error(`HTTP probe exited before ${kind}`)); }; + const cleanup = () => { + clearTimeout(timer); client.off("message", received); client.off("exit", failed); client.off("error", failed); + }; + client.on("message", received); client.once("exit", failed); client.once("error", failed); + }); + let publication: Promise | undefined; + try { + await message("ready"); + const response = message("result"); + publishing = true; + publication = publishNativeHarnessBackup({ + ...input, + stamp: async () => { + client.send({ kind: "probe" }); + const response = await probeReady; + if (!response) return; + setImmediate(() => { + response.setHeader("x-publication-active", publishing ? "1" : "0"); + response.end("ok"); + }); + }, + }).finally(() => { publishing = false; }); + const result = await response; + await publication; + expect(result.active).toBe("1"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("replacement"); + } finally { + probeReceived!(null); + try { await publication; } + finally { + client.kill(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + } + }, 15_000); }); diff --git a/server/src/services/native-runtime/native-harness-backup-publication.ts b/server/src/services/native-runtime/native-harness-backup-publication.ts index 820303584b0..714dafb5f20 100644 --- a/server/src/services/native-runtime/native-harness-backup-publication.ts +++ b/server/src/services/native-runtime/native-harness-backup-publication.ts @@ -9,7 +9,7 @@ export async function publishNativeHarnessBackup(input: { }) { const currentRoot = resolve(input.backupRoot, "current"); const previousRoot = resolve(input.backupRoot, "previous"); - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); let movedCurrent = false; if (existsSync(currentRoot)) { renameSync(currentRoot, previousRoot); @@ -19,10 +19,10 @@ export async function publishNativeHarnessBackup(input: { renameSync(input.pendingRoot, currentRoot); await input.stamp?.(currentRoot); } catch (error) { - if (existsSync(currentRoot)) removeNativeHarnessBackup(currentRoot); + if (existsSync(currentRoot)) await removeNativeHarnessBackup(currentRoot); if (movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot)) renameSync(previousRoot, currentRoot); throw error; } - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); noteNativeHarnessBackupChange(input.db, { companyId: input.companyId, scopeDigest: input.scopeDigest }); } diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index 96a4112e5a6..a27af96ac3b 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -31,7 +31,7 @@ import { } from "@paperclipai/db"; import { REMOTE_CONTROLLER_LEASE_MS } from "./native-runner-recovery-contract.js"; import { readProcessStartedAt } from "../hot-restart.js"; -import { getServerInfoSnapshot } from "../../server-info.js"; +import { getServerProcessStartedAt } from "../../server-info.js"; import { redactSensitiveText } from "../../redaction.js"; import { issueRecoveryActionService } from "../issue-recovery-actions.js"; import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report.js"; @@ -157,7 +157,7 @@ async function observedProcessStart(pid: number): Promise { export async function currentNativeControllerIdentity(): Promise { const observed = await observedProcessStart(process.pid); - const fallback = new Date(getServerInfoSnapshot().processStartedAt); + const fallback = new Date(getServerProcessStartedAt()); return { bootId: controllerBootId, pid: process.pid, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f3565441c39..93992b4870d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13734,7 +13734,7 @@ async function createRunnerdBackendWithinSessionClaim( } : undefined, }); } finally { - removeNativeHarnessBackup(pendingRoot); + await removeNativeHarnessBackup(pendingRoot); } }, { diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts index bf4f53dd371..5fab72c4c2c 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -19,7 +19,7 @@ import { issueThreadInteractions, issues, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { initializeRunIdentity, reserveSteeredIdentity, reconcileSteeredIdentity } from "../run-identity.js"; import { documentService } from "../documents.js"; @@ -248,7 +248,7 @@ describe("PaperclipRunnerToolAuthority", () => { }).where(eq(heartbeatRuns.id, runId)); const [expanded] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const observed = createDb(temporary!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues, agents, companies }, capture); try { const reader = new PaperclipRunnerToolAuthority(observed, binding); @@ -291,7 +291,7 @@ describe("PaperclipRunnerToolAuthority", () => { await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, runId)); const [expanded] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const observed = createDb(temporary!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues, agents, companies }, capture); try { const reader = new PaperclipRunnerToolAuthority(observed, binding); diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.ts index 2a9a845d01a..eeff010abbe 100644 --- a/server/src/services/plugin-host-services.ts +++ b/server/src/services/plugin-host-services.ts @@ -537,6 +537,9 @@ const KNOWN_PROVIDER_SPAN_NAMES: ReadonlySet = new Set([ "postUploadCommand", "session.open", "session.close", + "lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup", + "lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", + "lease.shell", "lease.create", "lease.expiry", "lease.cleanup", "plugin.worker", "plugin.host.identity.read", "plugin.host.config.read", "plugin.host.progress.read", "plugin.host.progress.write", "plugin.workspace.prepare", "plugin.storage.queue", "plugin.storage.execute", "plugin.runtime.queue", "plugin.runtime.prepare", "plugin.runtime.reconcile", "plugin.runtime.verify", @@ -564,6 +567,7 @@ const PROVIDER_SPAN_ATTR_ALLOWLIST: ReadonlySet = new Set([ SPAN_ATTRS.transferWallMs, SPAN_ATTRS.transferGuardCount, SPAN_ATTRS.transferDirection, + SPAN_ATTRS.state, "plugin.operation", "plugin.reused", "plugin.item_count", "plugin.bytes", "plugin.call_count", ]); @@ -595,6 +599,10 @@ export function clampProviderSpanAttributes( if (!raw) return clamped; for (const [key, value] of Object.entries(raw)) { if (!PROVIDER_SPAN_ATTR_ALLOWLIST.has(key)) continue; + if (key === SPAN_ATTRS.state) { + if (typeof value === "string" && ["started", "stopped", "archived", "error", "other"].includes(value)) clamped[key] = value; + continue; + } if (key === "plugin.operation") { if (typeof value === "string" && ["request", "queue.wait", "launch.wait", "mount.refresh", "guest.setup", "process.execute", "script.workspace", "script.git", "script.settings", "script.publication", "cleanup", "process.start", "hello", "checkpoint.load"].includes(value)) clamped[key] = value; continue; diff --git a/server/src/services/recovery/issue-state-read.test.ts b/server/src/services/recovery/issue-state-read.test.ts index 5a8ad5acf28..b1b4a39a0b5 100644 --- a/server/src/services/recovery/issue-state-read.test.ts +++ b/server/src/services/recovery/issue-state-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, issueRecoveryActions, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -60,7 +60,7 @@ describe.skipIf(!support.supported)("recovery issue state reads", () => { const service = recoveryService(db, { enqueueWakeup }); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { issues, issueRecoveryActions }, capture); try { await service.reconcileStrandedAssignedIssues(); @@ -100,7 +100,7 @@ describe.skipIf(!support.supported)("recovery issue state reads", () => { const service = recoveryService(db, { enqueueWakeup }); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { agents, issues, issueRecoveryActions }, capture); try { expect(await service.reconcileStrandedAssignedIssues()).toMatchObject({ skipped: 1 }); diff --git a/server/src/services/recovery/legacy-disposition-read.test.ts b/server/src/services/recovery/legacy-disposition-read.test.ts index 749a7991879..21ae049d472 100644 --- a/server/src/services/recovery/legacy-disposition-read.test.ts +++ b/server/src/services/recovery/legacy-disposition-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -37,7 +37,7 @@ describe.skipIf(!support.supported)("legacy disposition source reads", () => { errorCode: "adapter_failed", resultJson: { executionRecovery: { providerWorkStarted: true }, stdout: bulk } }).returning(); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await readLegacyDispositionSource(db, { companyId, issueId, runId: before.id })) @@ -77,7 +77,7 @@ describe.skipIf(!support.supported)("legacy disposition source reads", () => { const enqueueWakeup = vi.fn(async () => { throw new Error("native work must not enter legacy recovery"); }); const service = recoveryService(db, { enqueueWakeup }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await service.reconcileLegacyContinuation(before.id)).toBe("skipped"); } finally { capture.flush(); } diff --git a/server/src/services/remote-execution-termination.ts b/server/src/services/remote-execution-termination.ts index f9fae854c69..d30b60ae697 100644 --- a/server/src/services/remote-execution-termination.ts +++ b/server/src/services/remote-execution-termination.ts @@ -1,5 +1,8 @@ import { and, eq } from "drizzle-orm"; import { environmentLeases, type Db } from "@paperclipai/db"; +import { retiredReusableLeasePredecessors } from "./environment-lease-handoff.js"; + +type Lease = typeof environmentLeases.$inferSelect; type LeaseIdentity = { id: string; companyId: string; heartbeatRunId: string | null; @@ -39,12 +42,36 @@ export function remoteLeaseCleanupScope(lease: Pick { + if (hasRemoteTerminationReceipt(lease) || lease.status !== "expired" || + lease.leasePolicy !== "reuse_by_environment" || !lease.provider || !lease.providerLeaseId) return lease; + const history = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.provider, lease.provider), eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + )); + const current = history.find(row => row.id === lease.id); + if (!current || current.companyId !== lease.companyId || current.heartbeatRunId !== lease.heartbeatRunId) return lease; + const tips = history.filter(row => !Object.hasOwn(row.metadata ?? {}, "reusableLeaseSuccessor")); + if (tips.length !== 1) return current; + const tip = tips[0]!; + const receipt = tip.metadata?.remoteExecutionTermination as Record | undefined; + if (!hasRemoteTerminationReceipt(tip) || receipt?.state !== "destroyed") return current; + const predecessors = retiredReusableLeasePredecessors(tip, history.filter(row => row.id !== tip.id)); + if (!predecessors?.some(row => row.id === current.id)) return current; + const termination = remoteTerminationReceipt(current, { providerLeaseId: current.providerLeaseId, state: "destroyed" }); + if (!termination) return current; + return { ...current, metadata: { ...current.metadata, remoteExecutionTermination: { + ...termination, confirmedAt: receipt.confirmedAt, + } } }; +} + export async function stoppedRemoteCleanupScopes(db: Db, companyId: string, runId: string) { const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId), )); - if (leases.length === 0 || !leases.every(hasRemoteTerminationReceipt)) return null; - return [...new Set(leases.map(lease => remoteLeaseCleanupScope(lease)!))]; + if (leases.length === 0) return null; + const resolved = await Promise.all(leases.map(lease => resolveRemoteTerminationLease(db, lease))); + if (!resolved.every(hasRemoteTerminationReceipt)) return null; + return [...new Set(resolved.map(lease => remoteLeaseCleanupScope(lease)!))]; } export async function remoteExecutionHasStopped(db: Db, companyId: string, runId: string) { diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index 07fa20609f1..ef9664d05df 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -7,10 +7,21 @@ import { promisify } from "node:util"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; -const archiveIo = vi.hoisted(() => ({ afterChmod: undefined as ((target: string, mode: number) => void) | undefined })); +const archiveIo = vi.hoisted(() => ({ + afterChmod: undefined as ((target: string, mode: number) => void) | undefined, + beforeOpen: undefined as ((target: string) => Promise) | undefined, + captures: [] as string[], +})); vi.mock("node:fs/promises", async importOriginal => { const actual = await importOriginal(); - return { ...actual, async chmod(target: string, mode: number) { + return { ...actual, async open(...args: Parameters) { + await archiveIo.beforeOpen?.(String(args[0])); + return actual.open(...args); + }, async mkdtemp(...args: Parameters) { + const directory = await actual.mkdtemp(...args); + if (String(args[0]).includes("pc-workspace-capture-")) archiveIo.captures.push(String(directory)); + return directory; + }, async chmod(target: string, mode: number) { await actual.chmod(target, mode); archiveIo.afterChmod?.(target, mode); } }; }); @@ -22,7 +33,10 @@ const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C describe("portable remote workspace archives", () => { let root: string; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-archive-test-")); }); - afterEach(async () => { archiveIo.afterChmod = undefined; await fs.rm(root, { recursive: true, force: true }); }); + afterEach(async () => { + archiveIo.afterChmod = undefined; archiveIo.beforeOpen = undefined; archiveIo.captures = []; + await fs.rm(root, { recursive: true, force: true }); + }); async function source() { const cwd = path.join(root, "source"); await fs.mkdir(cwd); return cwd; } async function capture(cwd: string) { const archivePath = path.join(root, "snapshot.pcws"); return { archivePath, ...await captureRemoteWorkspaceDirectory({ cwd, archivePath }) }; } async function restore(snapshot: Awaited>) { @@ -57,6 +71,76 @@ describe("portable remote workspace archives", () => { const snapshot = await capture(await source()); const restored = await restore(snapshot); expect(await fs.readdir(restored)).toEqual([]); }); + it("captures with room for one archive and bounded metadata without another copy of working files", async () => { + const cwd = await source(), payload = Buffer.alloc(4 * 1024 * 1024, 123); + await fs.writeFile(path.join(cwd, "work.bin"), payload); + const archivePath = path.join(root, "snapshot.pcws"); + async function bytes(directory: string): Promise { + let total = 0; + for (const name of await fs.readdir(directory)) { + const file = path.join(directory, name), stat = await fs.lstat(file); + total += stat.isDirectory() ? await bytes(file) : stat.size; + } + return total; + } + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + let staged = 0; + for (const directory of archiveIo.captures) staged += await bytes(directory); + if (staged + payload.length > payload.length + 128 * 1024) { + throw Object.assign(new Error("No space left for the archive"), { code: "ENOSPC" }); + } + }; + const snapshot = await capture(cwd); + archiveIo.beforeOpen = undefined; + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + expect(digest(await fs.readFile(path.join(await restore(snapshot), "work.bin")))).toBe(digest(payload)); + expect(digest(await fs.readFile(path.join(cwd, "work.bin")))).toBe(digest(payload)); + }); + it.each(["content", "symlink", "added-file"])("rejects a concurrent %s change and removes its incomplete archive", async change => { + const cwd = await source(), file = path.join(cwd, "work.txt"), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(file, "original\n"); + const outside = path.join(root, "outside.txt"); await fs.writeFile(outside, "must stay outside\n"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + if (change === "content") await fs.writeFile(file, "changed\n"); + if (change === "added-file") await fs.writeFile(path.join(cwd, "new.txt"), "new file\n"); + if (change === "symlink") { await fs.unlink(file); await fs.symlink(outside, file); } + }; + await expect(capture(cwd)).rejects.toThrow(); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(outside, "utf8")).toBe("must stay outside\n"); + for (const directory of archiveIo.captures) await expect(fs.access(directory)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("cleans a cancelled capture without deleting a pre-existing archive", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + const controller = new AbortController(), reason = new Error("capture cancelled"); + archiveIo.beforeOpen = async target => { if (target === archivePath) controller.abort(reason); }; + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath, signal: controller.signal })).rejects.toBe(reason); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + archiveIo.beforeOpen = undefined; + await fs.writeFile(archivePath, "existing archive\n"); + await expect(capture(cwd)).rejects.toMatchObject({ code: "EEXIST" }); + expect(await fs.readFile(archivePath, "utf8")).toBe("existing archive\n"); + expect(await fs.readFile(path.join(cwd, "file"), "utf8")).toBe("retained\n"); + }); + it("rejects Git reference changes during archive output and keeps the new reference", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await git(cwd, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base"); + const head = await git(cwd, "rev-parse", "HEAD"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + await git(cwd, "update-ref", "refs/heads/later-work", head); + }; + await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_changed"); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await git(cwd, "rev-parse", "refs/heads/later-work")).toBe(head); + }); it("rejects a Files entry limit before extraction while ordinary restores keep their full contents", async () => { const cwd = await source(); await fs.writeFile(path.join(cwd, "first.txt"), "first\n"); await fs.writeFile(path.join(cwd, "second.txt"), "second\n"); const snapshot = await capture(cwd), targetDir = path.join(root, "limited"); diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index 6d4b1552053..6c910a83ed9 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -120,9 +120,28 @@ async function workspaceArchive(input: { if (state) fail("git_operation_in_progress: finish or abort the Git operation before saving the workspace"); } }; + const readStableFile = async (source: string, consume?: (bytes: Buffer) => Promise) => { + check(); + const original = await fs.lstat(source, { bigint: true }); + const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || original.dev !== before.dev || original.ino !== before.ino || await fs.realpath(source) !== source) fail("changed"); + const digest = crypto.createHash("sha256"); let size = 0; + for await (const bytes of handle.createReadStream({ autoClose: false, signal: input.signal })) { + check(); digest.update(bytes); size += bytes.length; + await consume?.(bytes); + } + const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); + if (size !== Number(before.size) || await fs.realpath(source) !== source + || ["dev", "ino", "size", "mode", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] + || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); + return { sha256: digest.digest("hex"), size, mode: Number(before.mode & 0o777n) }; + } finally { await handle.close(); } + }; const directories: string[] = []; const excludedPaths = new Set(); - const walk = async (root: string, copyTo?: string, skipGit = false) => { + const walk = async (root: string, skipGit = false) => { const entries: Entry[] = []; const visit = async (relative: string, parentIsRepository = false) => { const directory = relative ? path.join(root, relative) : root; @@ -137,7 +156,7 @@ async function workspaceArchive(input: { if (input.exclude?.some(excluded => next === excluded || next.startsWith(`${excluded}/`)) || next === ".paperclip-runtime" || next.startsWith(".paperclip-runtime/")) { excludedPaths.add(next); continue; } - const source = path.join(root, next), target = copyTo ? path.join(copyTo, next) : null; + const source = path.join(root, next); const stat = await fs.lstat(source); if (skipGit && name === ".git") { if (stat.isSymbolicLink()) fail("unsafe_git_link"); directories.push(relative); continue; } if (stat.isDirectory() && name === "node_modules") { @@ -150,30 +169,13 @@ async function workspaceArchive(input: { } if (stat.isDirectory()) { entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 }); - if (target) await fs.mkdir(target, { mode: 0o700 }); await visit(next, inRepository); } else if (stat.isSymbolicLink()) { const link = await fs.readlink(source); if (!safeLink(next, link)) fail("unsafe_symlink"); entries.push({ path: next, type: "symlink", target: link }); - if (target) await fs.symlink(link, target); } else if (stat.isFile()) { - const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); - let output: Awaited> | undefined; - try { - const before = await handle.stat({ bigint: true }); - if (!before.isFile() || BigInt(stat.dev) !== before.dev || BigInt(stat.ino) !== before.ino || await fs.realpath(source) !== source) fail("changed"); - if (target) output = await fs.open(target, "wx", 0o600); - const digest = crypto.createHash("sha256"); let bytes = 0; - for await (const part of handle.createReadStream({ autoClose: false, signal: input.signal })) { - check(); digest.update(part); bytes += part.length; - await output?.writeFile(part); - } - const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); - if (bytes !== Number(before.size) || await fs.realpath(source) !== source || ["dev", "ino", "size", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); - entries.push({ path: next, type: "file", mode: stat.mode & 0o777, size: bytes, sha256: digest.digest("hex") }); - if (target) await fs.chmod(target, stat.mode & 0o777); - } finally { await output?.close(); await handle.close(); } + entries.push({ path: next, type: "file", ...await readStableFile(source) }); } else fail("unsupported_entry"); } }; @@ -188,9 +190,10 @@ async function workspaceArchive(input: { const root = await fs.realpath(input.cwd!); if (!(await fs.lstat(root)).isDirectory()) fail("source_invalid"); const scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-capture-"))); + let archiveCreated = false, completed = false; try { const tree = path.join(scratch, "tree"); await fs.mkdir(tree, { mode: 0o700 }); - const before = await walk(root, tree, true); + const before = await walk(root, true); const repositories = [...new Set(directories)]; const gitSignatures = new Map(); const signature = async (repo: string, settings?: Awaited>) => { @@ -207,6 +210,7 @@ async function workspaceArchive(input: { // private repository. The selected worktree's index and HEAD are separate. await git(root, ["clone", "--mirror", "--no-hardlinks", "--dissociate", "--", repo, normalized]); const destination = path.join(tree, relative, ".git"); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); await fs.rename(normalized, destination); const sourceGit = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim(); const sourceCommon = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim(); @@ -232,35 +236,38 @@ async function workspaceArchive(input: { const [object, ref] = row.split(" "); return `update ${ref} ${object}\n`; }).join(""); if (updates) await git(restoredRepo, ["update-ref", "--stdin"], updates); - await git(restoredRepo, ["repack", "-a", "-d"]); await fs.rm(path.join(destination, "objects", "info", "alternates"), { force: true }); await git(restoredRepo, ["fsck", "--connectivity-only", "--no-dangling"]); if (await signature(repo) !== gitSignatures.get(relative)) fail("changed"); } - directories.length = 0; - const after = await walk(root, undefined, true); - if (JSON.stringify(before) !== JSON.stringify(after)) fail("changed"); - for (const relative of repositories) await assertGitOperationComplete(path.join(root, relative)); - const entries = await walk(tree); - // Keep private staging writable for cleanup; the manifest preserves the - // source modes and restoration applies them only after writing children. const originals = new Map(before.map(entry => [entry.path, entry])); - for (const entry of entries) if (entry.type === "directory") { - const original = originals.get(entry.path); if (original) entry.mode = original.mode; - } + const entries = [...before, ...(await walk(tree)).filter(entry => !originals.has(entry.path))] + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); const manifest = { schema, exclusionPolicy: "paperclip.remote-workspace-exclusions.v1", excluded: [...excludedPaths].sort(), entries }, body = Buffer.from(JSON.stringify(manifest)); if (body.length > 64 * 1024 * 1024) fail("manifest_too_large"); const digest = manifestHash(manifest), length = Buffer.alloc(4); length.writeUInt32BE(body.length); const output = await fs.open(input.archivePath, "wx", 0o600); + archiveCreated = true; try { await output.writeFile(Buffer.concat([magic, length, body])); for (const entry of entries) if (entry.type === "file") { - for await (const bytes of createReadStream(path.join(tree, entry.path), { signal: input.signal })) { check(); await output.writeFile(bytes); } + const observed = await readStableFile(path.join(originals.has(entry.path) ? root : tree, entry.path), bytes => output.writeFile(bytes)); + if (observed.size !== entry.size || observed.sha256 !== entry.sha256 || observed.mode !== entry.mode) fail("changed"); } } finally { await output.close(); } + directories.length = 0; + if (JSON.stringify(before) !== JSON.stringify(await walk(root, true))) fail("changed"); + for (const relative of repositories) { + if (await signature(path.join(root, relative)) !== gitSignatures.get(relative)) fail("changed"); + } if (input.manifestPath) await fs.writeFile(input.manifestPath, body, { flag: "wx", mode: 0o600 }); - return { schema, manifestDigest: digest, ...await hashFile(input.archivePath), entries: entries.length }; - } finally { await fs.rm(scratch, { recursive: true, force: true }); } + const content = await hashFile(input.archivePath); + completed = true; + return { schema, manifestDigest: digest, ...content, entries: entries.length }; + } finally { + if (archiveCreated && !completed) await fs.rm(input.archivePath, { force: true }); + await fs.rm(scratch, { recursive: true, force: true }); + } } if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid"); if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid"); diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index d4fe9d7ce80..669e097c532 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,4 +1,5 @@ import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { createHash } from "node:crypto"; import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { @@ -552,6 +553,7 @@ export function statusCardService( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index a116aeba7fe..71b6b12c292 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -6,7 +6,7 @@ import { z } from "zod"; import { randomUUID } from "node:crypto"; import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, issueComments, runIdentityContexts, issueRecoveryActions, type Db } from "@paperclipai/db"; -import { conflict, forbidden, notFound } from "../errors.js"; +import { conflict, forbidden, notFound, HttpError } from "../errors.js"; import { persistActivity } from "./activity-log.js"; import { createSystemRecovery, readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; @@ -17,6 +17,10 @@ import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-wo import { NativeWorkspaceFinalizationBusyError } from "./native-runtime/native-workspace-finalization-ownership.js"; import { adapterExecutionControls } from "./adapter-execution-control.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; +import { deferredChatInputPredicate, resumeDeferredChatWake } from "./durable-chat-wakeup.js"; +import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; +import { hasPendingRunCleanup } from "./execution-blocker.js"; +import { PersistedExecutionWaitRefusal } from "./deferred-execution-wait.js"; export async function taskRetryMutationAllowed(db: Db, input: { companyId: string; issueId: string; actorId: string; actorSource: TaskRetryIntent["actorSource"]; @@ -45,6 +49,7 @@ const taskRetryIntentSchema = z.object({ schema: z.literal("paperclip.task-retry.v1"), generationId: z.string().uuid(), sourceRunId: z.string().uuid(), actorId: z.string().min(1), actorSource: z.enum(["local_implicit", "session", "board_key", "cloud_tenant"]).default("session"), strategy: z.enum(["evaluate", "continue", "fresh_isolated_workspace", "finalize_result"]), chatActionId: z.string().uuid().optional(), canonicalIntentId: z.string().uuid().optional(), + queuedChatRequestId: z.string().uuid().optional(), boardMessageOrigin: z.object({ commentId: z.string().uuid(), identityContextId: z.string().uuid() }).optional(), }); export type TaskRetryIntent = z.infer; @@ -159,7 +164,7 @@ async function readBoardMessageRetrySource(db: Db, input: { sql`${agentWakeupRequests.payload}->>'issueId' = ${input.issueId}`)); const intent = readTaskRetryIntent(receipt?.payload); if (!intent?.boardMessageOrigin || intent.generationId !== receipt!.id || intent.actorId !== input.actorId || - intent.chatActionId || intent.canonicalIntentId || intent.sourceRunId !== successor.previousRunId) return null; + intent.chatActionId || intent.queuedChatRequestId || intent.canonicalIntentId || intent.sourceRunId !== successor.previousRunId) return null; const [identity] = await db.select({ id: runIdentityContexts.id }).from(runIdentityContexts).where(and( eq(runIdentityContexts.id, successor.identityContextId), eq(runIdentityContexts.companyId, input.companyId), eq(runIdentityContexts.runId, input.sourceRunId), eq(runIdentityContexts.responsibleUserId, input.actorId), @@ -279,6 +284,7 @@ export async function requestTaskRetry(input: { sourceRunId: latest.id, actorId: input.actorId, actorSource: taskRetryIntentSchema.shape.actorSource.parse(input.actorSource ?? "session"), strategy: "evaluate", ...(active && activeIntent?.chatActionId ? { chatActionId: activeIntent.chatActionId } : {}), + ...(active && activeIntent?.queuedChatRequestId ? { queuedChatRequestId: activeIntent.queuedChatRequestId } : {}), ...(active ? { canonicalIntentId: active.id } : {}) }; // Preserve external chat authority through its existing admission owner. // Generic board Retry must never manufacture a chat-inbound request. @@ -290,7 +296,20 @@ export async function requestTaskRetry(input: { ? await readTaskRetryBoardMessageOrigin(db, { companyId: issue.companyId, issueId: issue.id, agentId: issue.assigneeAgentId, sourceRunId: latest.id, actorId: input.actorId }) : null; if (boardMessageOrigin) intent.boardMessageOrigin = boardMessageOrigin; - if (chat && !active && !busy && !boardMessageOrigin && (latest.contextSnapshot?.source === "issue.comment.reopen" || latest.contextSnapshot?.boardMessageRetry === true) && + if (chat && !active && latest.status === "succeeded") { + const [savedInput] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, issue.companyId), eq(agentWakeupRequests.agentId, issue.assigneeAgentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.runId} is null`, + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, deferredChatInputPredicate(db), + )).orderBy(desc(agentWakeupRequests.requestedAt), desc(agentWakeupRequests.id)).limit(1).for("update"); + // A completed turn stays history. Only independently admitted input can + // continue, with its original identity and current inbound authorization. + if (savedInput) { + intent.queuedChatRequestId = savedInput.id; + await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: id }); + } + } + if (chat && !active && !busy && !boardMessageOrigin && !intent.queuedChatRequestId && (latest.contextSnapshot?.source === "issue.comment.reopen" || latest.contextSnapshot?.boardMessageRetry === true) && latest.responsibleUserId === input.actorId) { throw conflict("This saved message cannot be retried safely. Send a new message on the task to continue from saved work.", { code: "chat_failed_run_retry_not_authorized", reason: "saved_board_message_not_replayable" }); @@ -301,7 +320,7 @@ export async function requestTaskRetry(input: { status: active || busy ? "coalesced" : "deferred_issue_execution", runId: active?.runId ?? (busy ? latest.id : null), payload: { issueId: issue.id, taskRetry: intent }, finishedAt: active || busy ? new Date() : null }); - if (chat && !boardMessageOrigin && !active && !busy) { + if (chat && !boardMessageOrigin && !intent.queuedChatRequestId && !active && !busy) { if (!input.prepareChatRetry) throw conflict("Retry is unavailable for this connected conversation."); intent.chatActionId = (await input.prepareChatRetry(db, { companyId: issue.companyId, issueId: issue.id, agentId: issue.assigneeAgentId, failedRunId: latest.id, initiatedByUserId: input.actorId, retryIntentId: id })).actionId; @@ -351,21 +370,58 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId? scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, + or(sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, + sql`${agentWakeupRequests.payload}#>>'{taskRetry,queuedChatRequestId}' is not null`), )).orderBy(agentWakeupRequests.updatedAt).limit(50); for (const wake of pending) { const intent = readTaskRetryIntent(wake.payload); - if (!intent?.chatActionId) continue; + if (!intent || (!intent.chatActionId && !intent.queuedChatRequestId)) continue; + const actionId = intent.queuedChatRequestId ?? intent.chatActionId!; const [chat] = await db.select().from(chatActions).where(and(eq(chatActions.companyId, wake.companyId), - eq(chatActions.id, intent.chatActionId), eq(chatActions.kind, "failed_run_retry"), + eq(chatActions.id, actionId), eq(chatActions.kind, intent.queuedChatRequestId ? "inbound_wakeup" : "failed_run_retry"), sql`${chatActions.payload}->>'issueId' = ${String(wake.payload?.issueId)}`)); - const [receipt] = chat ? await db.select().from(agentWakeupRequests).where(and( + let [receipt] = chat ? await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.id, chat.id), eq(agentWakeupRequests.agentId, wake.agentId))) : []; + let inputDenied = false; + if (intent.queuedChatRequestId && !scope.settleOnly && !receipt?.runId) { + const issueId = String(wake.payload?.issueId); + const admission = await db.transaction(async tx => { + if (intent.generationId !== wake.id || intent.canonicalIntentId || + wake.requestedByActorType !== "user" || wake.requestedByActorId !== intent.actorId || + !wake.idempotencyKey?.startsWith("task-retry:")) return "denied"; + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, issueId), + eq(issues.companyId, wake.companyId))).for("update"); + if (!issue || issue.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(issue.status)) return "denied"; + if (await retireSupersededTaskRetry(tx as unknown as Db, { companyId: wake.companyId, issueId, + requestId: wake.id, sourceRunId: intent.sourceRunId })) return "denied"; + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: wake.companyId, issueId, + actorId: intent.actorId, actorSource: intent.actorSource })) return "denied"; + if (issue.conversationAgentId && (issue.conversationUserId !== intent.actorId || + !(await instanceSettingsService(tx as unknown as Db).getExperimental()).enableAgentChat)) return "denied"; + if (adapterExecutionControls.has(intent.sourceRunId) || + await hasPendingRunCleanup(tx as unknown as Db, wake.companyId, intent.sourceRunId)) return "waiting"; + return "ready"; + }); + inputDenied = admission === "denied" || !chat || !receipt; + if (!inputDenied && admission === "ready" && receipt?.status === "deferred_issue_execution") { + try { + await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, issueId, requestId: actionId }); + } catch (error) { + if (!(error instanceof PersistedExecutionWaitRefusal)) { + if (!(error instanceof HttpError) || ![403, 404, 409].includes(error.status)) throw error; + inputDenied = true; + } + } + [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.id, actionId), eq(agentWakeupRequests.agentId, wake.agentId))); + } + inputDenied ||= Boolean(receipt && ["failed", "skipped", "cancelled"].includes(receipt.status)); + } // A deferred chat receipt is admitted input, not a completed dispatch. // Re-stage that exact action through chat's owner; it rechecks authority // and normal queue policy before updating the existing receipt. - if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && + if (!intent.queuedChatRequestId && !scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" && !receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) { await db.transaction(async tx => { @@ -375,7 +431,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId? eq(chatActions.status, "processed"), eq(chatActions.updatedAt, chat.updatedAt))); }); } - const failed = chat && ["failed", "cancelled"].includes(chat.status); + const failed = inputDenied || (chat && ["failed", "cancelled"].includes(chat.status)); const write = (writer: Pick) => writer.update(agentWakeupRequests).set({ ...(receipt?.runId ? { status: "coalesced", runId: receipt.runId, finishedAt: new Date() } : failed ? { status: "failed", finishedAt: new Date() } : {}), diff --git a/server/src/services/task-search.ts b/server/src/services/task-search.ts index 600e355217e..9f56b5a7996 100644 --- a/server/src/services/task-search.ts +++ b/server/src/services/task-search.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { sql, type SQL } from "drizzle-orm"; import { COMPANY_SEARCH_MAX_QUERY_LENGTH, COMPANY_SEARCH_MAX_TOKENS } from "@paperclipai/shared"; import { visibleIssueCondition } from "./issue-visibility.js"; @@ -134,7 +135,7 @@ export function taskSearchCtes(companyId: string, search: TaskSearch, includeCon // The search page fetches descriptions only for its result window. const flags = (fuzzyMatch: SQL) => sql` SELECT issues.id, issues.identifier, issues.title, - issues.status, issues.priority, issues.assignee_agent_id, issues.assignee_user_id, + issues.status, ${issueDisplayStatusSql()} AS display_status, issues.priority, issues.assignee_agent_id, issues.assignee_user_id, issues.project_id, issues.created_at, issues.updated_at, ${identExact} AS ident_exact, ${identStarts} AS ident_starts, ${phraseMatch(sql`issues.identifier`)} AS ident_phrase, diff --git a/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts b/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts index fcb8588483e..efba0a8996a 100644 --- a/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts +++ b/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts @@ -24,10 +24,12 @@ test.describe("failed-attempt notice", () => { return { background: style.backgroundColor, padding: [style.paddingTop, style.paddingRight, style.paddingBottom, style.paddingLeft].map(parseFloat), + topPadding: parseFloat(getComputedStyle(element.parentElement!).paddingTop), }; }); expect(surface.background).not.toBe("rgba(0, 0, 0, 0)"); expect(surface.padding.every(value => value > 0)).toBe(true); + expect(surface.topPadding).toBeCloseTo(16); const iconBounds = (await icon.boundingBox())!; const textBounds = (await message.boundingBox())!; diff --git a/tests/perf/collections/README.md b/tests/perf/collections/README.md index eda8de20f4c..c88a05532ac 100644 --- a/tests/perf/collections/README.md +++ b/tests/perf/collections/README.md @@ -23,6 +23,31 @@ Request counts and row budgets are acceptance checks. They do not establish an end-to-end latency improvement. Measure cold, warm, and Back timings separately with the same fixture, assets, cache, viewport, and CPU/network settings. +`parent-context.spec.ts` also covers a delayed missing-parent index response, +reuse of an already indexed parent, and row geometry through deferred visible +diagnostics. Parent captions retain the initial reveal boundary. Failed-run +context and parent-to-failure cache transitions are covered by `Inbox.test.tsx`. + +To collect timings, serve pinned baseline and candidate production assets from +two loopback origins that proxy the same disposable API. Then run: + +```sh +INBOX_BASELINE_URL=http://127.0.0.1: \ +INBOX_CANDIDATE_URL=http://127.0.0.1: \ + node --import ./server/node_modules/tsx/dist/loader.mjs \ + tests/perf/collections/measure-mine-context.ts +``` + +The script retains a four-parent, 30-child fixture, archives three completed +parents, and enables the personal Coordinator preference. It runs five alternating +pairs by default. `INBOX_PAIRS`, `INBOX_FIXTURE_FILE`, and `INBOX_OUTPUT` override +the count and evidence paths. Each cold sample starts with empty HTTP and query +caches. Warm Mine navigation and task-to-Mine Back retain those caches. The report +keeps every sample, errors, row membership/order, geometry, assets and API timings. +Keep upstream identity/data timings as controls: a faster cold total alone does +not attribute the gain to parent context. Use the same immutable asset cache policy +on both origins and stop builds and suites during timing runs. + Startup cases also check that shared collection controls do not require the task-detail bundle before reveal, even when a different organization is remembered. Code warms after the collection paints, or on task-link intent. A successful diff --git a/tests/perf/collections/measure-mine-context.ts b/tests/perf/collections/measure-mine-context.ts new file mode 100644 index 00000000000..36108383dee --- /dev/null +++ b/tests/perf/collections/measure-mine-context.ts @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { arch, loadavg, platform } from "node:os"; +import { chromium, expect, request, type APIRequestContext } from "@playwright/test"; + +type Fixture = { + company: { id: string; issuePrefix: string }; + parents: Array<{ id: string }>; + children: Array<{ id: string }>; +}; +type Arm = "baseline" | "candidate"; +declare global { + interface Window { __mineContextMeasure: { first: number | null } } +} + +function loopbackOrigin(value: string | undefined, name: string) { + assert.ok(value, `Set ${name} to a disposable local production UI origin.`); + const url = new URL(value); + assert.ok(["http:", "https:"].includes(url.protocol) + && ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) + && url.pathname === "/" && !url.username && !url.password && !url.search && !url.hash, + `${name} must be an exact loopback origin.`); + return url.origin; +} +const origins = { + baseline: loopbackOrigin(process.env.INBOX_BASELINE_URL, "INBOX_BASELINE_URL"), + candidate: loopbackOrigin(process.env.INBOX_CANDIDATE_URL, "INBOX_CANDIDATE_URL"), +}; +assert.notEqual(origins.baseline, origins.candidate, "Use distinct baseline and candidate UI origins."); +const pairs = Number(process.env.INBOX_PAIRS ?? 5); +assert.ok(Number.isSafeInteger(pairs) && pairs > 0, "INBOX_PAIRS must be a positive integer."); +const fixtureFile = process.env.INBOX_FIXTURE_FILE ?? "test-results/collections/mine-context-fixture.json"; +const output = process.env.INBOX_OUTPUT ?? "test-results/collections/mine-context-timings.json"; +assert.notEqual(path.resolve(fixtureFile), path.resolve(output), "Fixture and report paths must differ."); + +async function api(client: APIRequestContext, method: string, url: string, data?: unknown) { + const response = await client.fetch(`/api${url}`, { method, data }); + assert.ok(response.ok(), `${method} ${url} returned HTTP ${response.status()}.`); + return response.status() === 204 ? null : response.json(); +} +async function retainFixture(client: APIRequestContext): Promise { + const saved = await fs.readFile(fixtureFile, "utf8").then(JSON.parse).catch(error => { + if (error.code !== "ENOENT") throw error; + return null; + }) as Fixture | null; + if (saved) { + assert.ok(saved.company?.id && saved.company.issuePrefix + && saved.parents.length === 4 && saved.children.length === 30, "Invalid retained fixture."); + return saved; + } + const company = await api(client, "POST", "/companies", { name: "Mine parent context performance" }); + const fixture: Fixture = { company: { id: company.id, issuePrefix: company.issuePrefix }, parents: [], children: [] }; + const create = async (title: string, status: string, parentId?: string) => { + const row = await api(client, "POST", `/companies/${company.id}/issues`, { title, status, parentId }); + return { id: row.id as string }; + }; + for (let index = 0; index < 4; index++) { + fixture.parents.push(await create(`Parent ${index}`, index === 0 ? "blocked" : "done")); + } + for (let index = 0; index < 30; index++) { + fixture.children.push(await create(`Inbox child ${index}`, "todo", fixture.parents[index % 4]!.id)); + } + for (const parent of fixture.parents.slice(1)) await api(client, "POST", `/issues/${parent.id}/inbox-archive`, {}); + await fs.mkdir(path.dirname(fixtureFile), { recursive: true }); + await fs.writeFile(fixtureFile, JSON.stringify(fixture, null, 2)); + return fixture; +} + +const clients = { + baseline: await request.newContext({ baseURL: origins.baseline }), + candidate: await request.newContext({ baseURL: origins.candidate }), +}; +const samples: Array> = []; +let expectedRowIds: Array | undefined; +let fixture: Fixture | undefined; +let browser: Awaited> | undefined; +const save = async () => { + await fs.mkdir(path.dirname(output), { recursive: true }); + await fs.writeFile(output, JSON.stringify({ + pairs, fixture, origins, rowSequenceReference: expectedRowIds, + settings: { viewport: "1440x900", httpCache: "fresh context", serviceWorkers: "blocked", + queryCache: "empty for cold, retained for warm and Back", cpu: "native", network: "loopback", + readiness: "cold: browser clock to first Mine row; warm/Back: host clock around browser actions", + node: process.version, platform: platform(), arch: arch(), browser: browser?.version() }, + samples, + }, null, 2)); +}; +try { + for (const arm of ["baseline", "candidate"] as const) { + const response = await clients[arm].get("/"); + assert.ok(response.ok(), `${arm} UI is unavailable.`); + const html = await response.text(); + assert.ok(/src=["']\/assets\/[^"']+\.js["']/.test(html) && !html.includes("/@vite/client"), + `${arm} must serve a production UI build.`); + } + fixture = await retainFixture(clients.candidate); + for (const arm of ["baseline", "candidate"] as const) await api(clients[arm], "GET", `/companies/${fixture.company.id}`); + await api(clients.candidate, "PUT", `/companies/${fixture.company.id}/workspace-preferences/me`, { coordinatorWorkspaceEnabled: true }); + browser = await chromium.launch(); + for (let pair = 0; pair < pairs; pair++) { + const order: Arm[] = pair % 2 ? ["candidate", "baseline"] : ["baseline", "candidate"]; + for (const arm of order) { + const context = await browser.newContext({ viewport: { width: 1440, height: 900 }, serviceWorkers: "block" }); + const page = await context.newPage(); + const errors: string[] = []; + const reads: Array<{ view: string | null; ids: number }> = []; + let rowIds: Array | undefined; + page.on("pageerror", error => errors.push(error.message)); + page.on("request", request => { + const url = new URL(request.url()); + if (url.pathname.endsWith("/issues")) reads.push({ view: url.searchParams.get("view"), ids: url.searchParams.get("ids")?.split(",").length ?? 0 }); + }); + try { + await page.addInitScript(() => { + window.__mineContextMeasure = { first: null }; + const observer = new MutationObserver(() => { + if (window.__mineContextMeasure.first === null && location.pathname.endsWith("/inbox/mine") && document.querySelector("[data-inbox-item]")) { + window.__mineContextMeasure.first = performance.now(); + observer.disconnect(); + } + }); + observer.observe(document, { subtree: true, childList: true }); + }); + await page.goto(`${origins[arm]}/${fixture.company.issuePrefix}/inbox/mine`); + const rows = page.locator("[data-inbox-item]"); + await expect(rows.first()).toBeVisible(); + const cold = await page.evaluate(() => window.__mineContextMeasure.first); + assert.ok(cold !== null, "Cold readiness observer did not see a row."); + rowIds = await page.locator("[data-inbox-issue-link]").evaluateAll(nodes => nodes.map(node => node.getAttribute("href"))); + const firstBox = await rows.first().boundingBox(); + await page.getByRole("tab", { name: "Recent", exact: true }).click(); + await expect(page.getByRole("tab", { name: "Recent", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const start = performance.now(); + await page.getByRole("tab", { name: "Mine", exact: true }).click(); + await expect(page.getByRole("tab", { name: "Mine", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const warm = performance.now() - start; + await page.locator("[data-inbox-issue-link]").first().click(); + await expect(page.getByTestId("issue-detail-header")).toBeVisible(); + const backStart = performance.now(); + await page.goBack(); + await expect(page.getByRole("tab", { name: "Mine", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const back = performance.now() - backStart; + const resources = await page.evaluate(() => performance.getEntriesByType("resource").map(entry => { + const resource = entry as PerformanceResourceTiming; + const url = new URL(resource.name); + return { path: url.pathname, start: resource.startTime, ms: resource.duration, end: resource.responseEnd }; + }).filter(resource => resource.path.startsWith("/api/") || resource.path.startsWith("/assets/"))); + assert.deepEqual(errors, [], "The page reported JavaScript errors."); + expectedRowIds ??= rowIds; + assert.deepEqual(rowIds, expectedRowIds, "Mine row order or membership changed between samples."); + samples.push({ pair, arm, status: "passed", cold, warm, back, rowIds, rowSequenceEquivalent: true, firstBox, reads, resources, errors, load: loadavg() }); + console.log(JSON.stringify({ pair, arm, cold, warm, back, rows: rowIds.length, errors })); + } catch (error) { + samples.push({ pair, arm, status: "failed", error: error instanceof Error ? error.message : String(error), rowIds, errors, reads, load: loadavg() }); + throw error; + } finally { + await save(); + await context.close(); + } + } + } +} finally { + await save(); + await browser?.close(); + await Promise.all(Object.values(clients).map(client => client.dispose())); +} diff --git a/tests/perf/collections/parent-context.spec.ts b/tests/perf/collections/parent-context.spec.ts new file mode 100644 index 00000000000..a7065d340da --- /dev/null +++ b/tests/perf/collections/parent-context.spec.ts @@ -0,0 +1,78 @@ +import { expect, test } from "@playwright/test"; + +let companyId: string | undefined; +test.afterEach(async ({ request }) => { + if (!companyId) return; + const response = await request.delete(`/api/companies/${companyId}`); + expect(response.ok(), await response.text()).toBeTruthy(); + companyId = undefined; +}); + +test("Mine waits for missing parent metadata without loading parent diagnostics", async ({ page, request }) => { + const created = await request.post("/api/companies", { data: { name: "Parent context browser check" } }); + expect(created.ok(), await created.text()).toBeTruthy(); + const company = await created.json() as { id: string; issuePrefix: string }; + companyId = company.id; + const preference = await request.put(`/api/companies/${company.id}/workspace-preferences/me`, { + data: { coordinatorWorkspaceEnabled: true }, + }); + expect(preference.ok(), await preference.text()).toBeTruthy(); + const create = async (title: string, status: string, parentId?: string) => { + const response = await request.post(`/api/companies/${company.id}/issues`, { data: { title, status, parentId } }); + expect(response.ok(), await response.text()).toBeTruthy(); + return response.json() as Promise<{ id: string; identifier: string; title: string }>; + }; + const loaded = await create("Indexed parent context", "blocked"); + const missing = await create("Archived parent context", "done"); + const first = await create("Child of indexed parent", "todo", loaded.id); + const second = await create("Child of archived parent", "todo", missing.id); + const archived = await request.post(`/api/issues/${missing.id}/inbox-archive`, { data: {} }); + expect(archived.ok(), await archived.text()).toBeTruthy(); + + const reads: Array<{ projection: string | null; ids: string[] }> = []; + const contextRead = (url: URL) => url.pathname === `/api/companies/${company.id}/issues` + && url.searchParams.get("view") === "index" + && url.searchParams.get("ids")?.split(",").includes(missing.id) === true; + page.on("request", request => { + const url = new URL(request.url()); + if (url.pathname === `/api/companies/${company.id}/issues` && url.searchParams.has("ids")) { + reads.push({ projection: url.searchParams.get("view"), ids: url.searchParams.get("ids")!.split(",") }); + } + }); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let releaseDetails!: () => void; + const detailsHeld = new Promise(resolve => { releaseDetails = resolve; }); + const detailRead = (url: URL) => url.pathname === `/api/companies/${company.id}/issues` + && url.searchParams.get("view") === "rows" + && url.searchParams.get("ids")?.split(",").includes(second.id) === true; + await page.route(detailRead, async route => { await detailsHeld; await route.continue(); }); + await page.route(contextRead, async route => { await held; await route.continue(); }); + const parentRequest = page.waitForRequest(request => contextRead(new URL(request.url()))); + try { + await page.goto(`/${company.issuePrefix}/inbox/mine`); + await parentRequest; + expect(await page.locator("[data-inbox-item]").count()).toBe(0); + expect(reads.some(read => read.projection === "rows")).toBe(false); + expect(reads.some(read => read.ids.includes(loaded.id))).toBe(false); + } finally { release(); } + + const child = page.locator(`[data-inbox-issue-link][href$="/${second.identifier}"]`); + await expect(child).toBeVisible(); + const node = await child.elementHandle(); + const row = child.locator(".."); + // Visible task diagnostics remain deferred, and their completion must retain + // the title's node, position and row height. + await expect.poll(() => reads.some(read => read.projection === "rows" && read.ids.includes(second.id))).toBe(true); + const before = await row.boundingBox(); + const detailsResponse = page.waitForResponse(response => detailRead(new URL(response.url()))); + releaseDetails(); + await (await detailsResponse).finished(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await expect(page.locator(`[data-inbox-issue-link][href$="/${first.identifier}"]`)).toBeVisible(); + expect(reads.filter(read => read.ids.includes(missing.id)).every(read => read.projection === "index")).toBe(true); + expect(await node!.evaluate(element => element.isConnected)).toBe(true); + const after = await row.boundingBox(); + expect(Math.abs(before!.height - after!.height)).toBeLessThan(1); + expect(Math.abs(before!.y - after!.y)).toBeLessThan(1); +}); diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index a66cf38a374..01d7c354b23 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -181,82 +181,31 @@ backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. -## TestOps completion pilot +## Retired TestOps plans -The separate `Chat completion TestOps` workflow connects only these preserved -catalog cases to the private Paperclip Evals project: +The one-case **Chat feature evals** plan and three-case +**Chat completion pilot — OpenCode 1017** plan are retired. Their scenarios are +included in the [full catalog](#full-catalog-in-testops). Use that plan or select +cases from it; do not recreate the smoke or pilot plans. -- `projects.project-work` -- `projects.project-defaults` -- `instance-operations.restart-service` +The former `chat-feature-testops.yml` and `chat-completion-testops.yml` workflows +are removed. Their bootstrap functions reject registration before any API call +or mutation marker. Shared transport, evidence, export and verification helpers +remain for the full-catalog workflow and retained historical evidence. Retiring a +plan does not delete its cases, results, launches or private evidence. -It evaluates the immutable catalog source in -`chat-completion-testops-contract.mjs`. This keeps the broader catalog on its -integration branch. The source includes the bounded TestOps host prerequisite -from #18515. Changing the pin requires source review. The original DeepSeek -1017 model, case clocks, independent checks, and zero retries remain fixed. -Source registration does not establish a new live result. - -Run the named plan **Chat completion pilot — OpenCode 1017** in TestOps. The -adapter validates the actual project, plan, job, launch and selected cases before -build or provider access. A selected subset runs only those cases. The AWS matrix -has concurrency one and does not cancel sibling cases after a failure. - -The main-only reporter uses the existing protected `ALLURE_TOKEN`; the evaluated -app and worker never receive it. Original evidence stays in the existing private -AWS storage with the unchanged validators, scanners, byte bounds and exact-key -policies. A separate trusted projection carries verdicts, checks, accounting and -private-evidence receipts. The final PNGs are verified in protected storage; -only their checksums and availability are recorded in TestOps. GitHub artifacts -contain metadata only. Raw logs, screenshots, hidden -reasoning, API snapshots, credentials and provider sessions are not attachments. -Original failed grades and cleanup failures remain failed. Same-run readback -requires matching results and complete result-owned attachment metadata within -the existing bounded processing window; it does not claim remote byte hashes. - -Initial setup uses the existing workflow operations: - -1. Dispatch reviewed `master` once with `operation=bootstrap`. This creates or - reconciles only the three exact automated cases and the dedicated plan. It - registers the GitHub job with an empty Allure results directory. It reuses - only the existing enabled GitHub integration from - `CHAT_TESTOPS_INTEGRATION_ID`. No model or fabricated result is created. - The observation receipt captures a closed parameter-shape inspection before - binding. Only the observed single `Branch/defaultValue` definition for - `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases - while preserving that definition; the plan uses its default branch. - Assignment inspection records unassigned and existing-job counts before - binding. A single unassigned bucket must cover the exact selected cases; - foreign jobs and unknown or incomplete buckets stop before writes. -2. Read the closed registration receipt. Store its observed `binding` JSON as - `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. -3. Use TestOps **Run**. For the initial operator launch, set an exact next - workflow run number in `CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER`, set a new - 32-hex `CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE`, then dispatch `launch-plan` - once. Its request produces the TestOps-owned `evaluate` dispatch. Do not - manually dispatch an additional evaluation after an uncertain launch. -4. Verify the fresh TestOps launch, all selected case results, source links, - independent checks, private-evidence receipts and cleanup. Inspect screenshots - through authenticated private-history retrieval. Retain earlier failed attempts. - -Launch preflight searches use TestOps' typed name filter, encoded as Base64 of -UTF-8 JSON in the `search` query parameter. Plain name text returns HTTP 400. -The backlog and completion launchers share this encoding; the full-catalog -launcher uses the completion adapter. Existing nonce and once-only guards still -apply before a plan-run request. - -The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username -metadata. It rejects arbitrary source/model/budget selectors, branch workflows, -replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and -actor identities. The existing one-case backlog plan remains independent. +Retained launch helpers encode TestOps name filters as Base64 of UTF-8 JSON in +`search`. Plain name text returns HTTP 400. Their nonce and once-only guards +remain unchanged; this transport contract does not restore the retired plans. ## Full catalog in TestOps The [Chat catalog TestOps workflow](../../.github/workflows/chat-catalog-testops.yml) registers all 244 scenarios from the 35 YAML areas in a separate **Chat feature catalog — OpenCode 1017 (candidate)** plan. Its source-derived -manifest is `chat-full-catalog-testops-manifest.json`. The existing backlog plan -and three-case completion pilot keep their own jobs, selections and history. +manifest is `chat-full-catalog-testops-manifest.json`. This is the active chat +plan. The former backlog and three-case pilot plans are retired; their cases +and measured history are preserved. Legacy aliases reuse existing TestOps case identities without renaming them. The catalog contains 236 Agent Chat routes, five browser boundaries, one external @@ -322,77 +271,3 @@ and credentials do not enter the public artifact path. Additional credentials reach only cases that declare them. External inputs use the existing selected-case private-input owner, disposable-resource checks and oracle commitments. Unconfigured inputs remain setup blockers. - -## TestOps-backed AWS backlog cell - -The dedicated `Chat feature TestOps` workflow accepts only the shipped default -branch and the fixed OpenCode backlog cell. `evaluate` is the default operation -and requires TestOps' `ALLURE_JOB_RUN_ID`. A protected reporting job verifies the -project, singleton plan, assigned public workflow, selected case and current -GitHub run before any build or provider access. The evaluated process receives -no TestOps token. It retains the original one-query, one-work-run, 120-second -turn, 420-second attempt, 300-second settled-state and zero-retry limits. - -Two metadata operations prepare that route. `bootstrap` registers the actual -public CI job through pinned `allurectl` with an empty results directory and -reconciles only the dedicated case and singleton plan. It produces no passing -test result. After the registered job and protected case/plan/job identifiers -are verified, `launch-plan` sends one TestOps plan-run request and exits. -The TestOps job then dispatches `evaluate`; an operator must not dispatch a -second evaluation manually. Missing or ambiguous registration and selection -metadata stops before inference. - -The workflow reuses the existing AWS fleet and strict private evidence storage. -Its shipped-main authority is separate from manual candidate approval. Original -logs, provider records and failure evidence remain private. The reporting adapter -exports only a source/run-bound normalized result, JUnit and screenshots already -marked by the owned fixture screenshot producer with matching file hashes. -A separate protected reporting job converts that projection to the exact TestOps -case. Failed grades and cleanup failures remain failed. - -Reporting setup uses the master-only `runner-e2e-testops-reporting` environment, -with the existing reporting token delivered through the reviewed sealed-box -bootstrap. `CHAT_TESTOPS_CASE_ID`, `CHAT_TESTOPS_PLAN_ID` and -`CHAT_TESTOPS_JOB_ID` must name the observed dedicated mapping; the numeric -`CHAT_TESTOPS_GITHUB_WORKFLOW_ID` supports TestOps' registered workflow URL. -No arbitrary source, model, case, retry or budget input is accepted. - -The initial `launch-plan` administrator operation also requires a protected -`CHAT_TESTOPS_LAUNCH_RUN_NUMBER`. After registration, the trusted operator creates -this variable once with the reviewed next workflow run number and reads it back. -The launcher compares GitHub's actual run number, attempt, workflow, source and -actors before any TestOps mutation. A later dispatch or replay cannot reuse that -approved number. Missing or uncertain metadata stops; an uncertain request needs -reconciliation rather than a new send. The nonce and launch search provide -additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of -this one-time administrator fence. The launch nonce is protected configuration, -not a dispatch input. Job parameter IDs are opaque. The launcher preserves the -API's exact ID/value pair for `refs/heads/master` and uses the fixed workflow's -`evaluate` default. An operation parameter is used only when the protected -`CHAT_TESTOPS_OPERATION_PARAMETER_ID` matches a returned ID with a `bootstrap` -or `evaluate` value; the launcher then sets it to `evaluate`. Other values, -duplicate IDs and conflicting configuration are rejected. No relationship to -global environment-variable IDs or names is inferred. - -`CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub -integration. The launcher verifies its project, type and enabled state, then binds -only the newly registered public job to it. It never changes integration settings -or credentials. Since the job response does not expose that association, only the -real TestOps-to-GitHub dispatch can qualify it. - -After `allurectl watch` finishes, the same protected reporter checks the actual -TestOps job-run, launch, selected case, result source/status and attachment -metadata. Processing reads are bounded to 12 polls, five-second gaps and a -120-second processing deadline. Unknown or missing results remain unverified. -The check downloads no attachment content and claims no server-side attachment -byte hash. Its closed receipt is retained separately. An upload success alone -is not a passing feature result, and verifying an original failure cannot turn -that failure into a pass. - -Build and worker installs use the existing reviewed report-lock policy. Only its -committed input or output lock is accepted; its authenticated correction is the -only allowed normalization. Dependency manifests, patch inputs and policy must -remain unchanged. Installs are frozen with lifecycle scripts disabled. The -credential-free build rechecks that boundary before packaging, and the worker -matches the source/lock receipt before installing. Arbitrary resolver drift is -rejected before provider access. diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index 5b9f883c935..7d02341af3e 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -135,32 +135,12 @@ export async function inspectPilotCatalog(api) { } export async function bootstrapPilotPlan(api, integrationId, markOnce, context = {}) { + // The pilot plan is retired. Reject before API reads or mutation markers. + if (context.catalog !== 'full') reject('plan_retired'); if (!positive(integrationId)) reject('integration'); const integration = await api(`/api/integration/${integrationId}/project/1`); if (integration.id !== integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) reject('integration'); - if (context.catalog === 'full') return { ...await reconcileFullCatalog(api, markOnce), integrationId }; - const inspected = await inspectPilotCatalog(api); - if (inspected.plans.length && inspected.cases.length !== cases.length) reject('plan_conflict'); - const selected = [...inspected.cases]; - for (const row of cases) { - if (selected.some(value => value.key === row.key)) continue; - await markOnce(`create_case_${row.key}`); - const created = await api('/api/testcase', 'POST', { projectId: 1, name: row.name, fullName: row.fullName, automated: true }); - if (!positive(created?.id)) reject('created_case'); - assertCase(created, { ...row, caseId: created.id }); - selected.push({ ...row, caseId: created.id }); - } - const ordered = cases.map(row => selected.find(value => value.key === row.key)); - let planId = inspected.plans[0]?.id; - if (!planId) { - await markOnce('create_plan'); - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: ordered.map(row => row.caseId) } }); - if (!positive(created?.id)) reject('created_plan'); planId = created.id; - } - const binding = { planId, integrationId, cases: ordered }; - assertPlan(await api(`/api/testplan/${planId}`), binding); - return binding; + return { ...await reconcileFullCatalog(api, markOnce), integrationId }; } export async function completePilotRegistration(api, prepared, official, context, workflowId, markOnce, observe = async () => {}) { diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index 4ee58ebe003..aee4be2527c 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -180,35 +180,9 @@ test('an uncertain launch response is never automatically resent', async () => { assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); }); -test('registration creates only the three missing cases and the dedicated plan after once markers', async () => { - const calls = []; let nextCase = 100; - const get = async (route, method = 'GET', body) => { - calls.push({ route, method, body }); - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - if (route === '/api/testcase' && method === 'POST') return { ...body, id: nextCase++ }; - if (route === '/api/testplan' && method === 'POST') return { ...plan, name: planName }; - if (route === '/api/testplan/10') return plan; - assert.fail(route); - }; - const value = await bootstrapPilotPlan(get, 2, async stage => calls.push({ marker: stage })); - assert.equal(value.cases.length, 3); assert.equal(value.planId, 10); - assert.equal(calls.filter(row => row.method === 'POST').length, 4); - for (const [i, row] of calls.entries()) if (row.method === 'POST') assert.ok(calls[i - 1].marker); -}); - -test('uncertain case creation is never automatically resent', async () => { - let writes = 0, markers = 0; - const get = async (route, method = 'GET') => { - if (method === 'POST') { writes++; throw new Error('uncertain transport'); } - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - assert.fail(route); - }; - await assert.rejects(bootstrapPilotPlan(get, 2, async () => { markers++; }), /uncertain transport/); - assert.equal(writes, 1); assert.equal(markers, 1); +test('retired pilot bootstrap cannot recreate a plan or call the integration API', async () => { + await assert.rejects(bootstrapPilotPlan(() => assert.fail('retired route must not call the API'), 2, + () => assert.fail('retired route must not mark a mutation')), /chat_completion_testops_plan_retired/); }); function api(replace = {}) { diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs index c77d89031a8..c1199bbeb40 100644 --- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -96,37 +96,8 @@ test('the executable guard binds the actual original GitHub run before service o } finally { await rm(root, { recursive: true, force: true }); } }); -test('the real workflow separates metadata authority, provider execution and result reporting', async () => { - const text = await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8'); - const w = parse(text); - assert.deepEqual(Object.keys(w.on.workflow_dispatch.inputs).sort(), ['ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'operation']); - assert.equal(w.concurrency['cancel-in-progress'], false); - assert.equal(w.jobs.chat.strategy['max-parallel'], 1); assert.equal(w.jobs.chat.strategy['fail-fast'], false); - assert.equal(w.jobs.select.environment, 'runner-e2e-testops-reporting'); - assert.equal(w.jobs.chat.environment, 'runner-e2e-private-paid'); - assert.equal(w.jobs.report.environment, 'runner-e2e-private-history'); - assert.equal(w.jobs.upload.environment, 'runner-e2e-testops-reporting'); - for (const [name, job] of Object.entries(w.jobs)) { - assert.ok(job.steps.slice(0, 3).some(step => step.run?.endsWith('chat-completion-testops-cli.mjs guard')), name); - if (name !== 'chat') assert.ok(!JSON.stringify(job).includes('secrets.OPENROUTER_API_KEY'), name); - if (!['select', 'upload'].includes(name)) assert.ok(!JSON.stringify(job).includes('secrets.ALLURE_TOKEN'), name); - } - assert.ok(w.jobs.upload.if.includes("needs.report.result == 'success'")); - assert.ok(w.jobs.report.needs.includes('build')); - assert.ok(w.jobs.report.if.includes("needs.build.result == 'success'")); - assert.ok(w.jobs.report.if.includes("needs.chat.result == 'failure'")); - assert.ok(!text.includes('<<:')); assert.ok(!text.includes('target_branch:')); - assert.equal(w.jobs.build.steps.find(step => step.name?.startsWith('Resolve the fixed')).run.includes('--ignore-scripts'), true); - const paid = w.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); - assert.equal(paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, 'v1'); - assert.equal(paid.env.ALLURE_JOB_RUN_ID, '${{ inputs.ALLURE_JOB_RUN_ID }}'); - assert.ok(paid.run.includes('--max-automatic-retries 0')); - assert.ok(w.jobs.build.steps.some(step => step.run?.includes('chat-completion-testops-host-control.mjs'))); - assert.ok(w.jobs.chat.steps.indexOf(paid) > w.jobs.chat.steps.findIndex(step => step.name?.startsWith('Verify source-bound'))); -}); - test('target checkout cannot populate or overwrite the trusted helper path', async () => { - const definition = parse(await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8')); + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); try { const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 2b4a4f1d049..47fc4939da8 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -53,7 +53,7 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'testops_case_alias_conflict', 'testops_plan_name_conflict', 'testops_plan_identity_invalid', 'testops_integration_identity_invalid', 'testops_bootstrap_identity_invalid', 'testops_plan_without_matching_case', 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', - 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', + 'testops_plan_retired', 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); @@ -139,32 +139,10 @@ export async function inspectChatMetadata(api) { return { projectId: 1, caseConflict: cases.size > 1, cases: [...cases.values()], plans: plans.map(value => ({ id: value.id, name: planName, count: value.testCasesCount })), integrations }; } -export async function ensureChatPlan(api, inspected, markOnce) { - if (inspected.projectId !== 1 || inspected.plans.length > 1) fail('testops_bootstrap_identity_invalid'); - if (inspected.caseConflict === true || inspected.cases.length > 1) fail('testops_case_alias_conflict'); - let selected = inspected.cases[0], plan = inspected.plans[0]; - if (plan && !selected) fail('testops_plan_without_matching_case'); - if (!selected || !plan) await markOnce(); - if (!selected) { - const created = await api('/api/testcase', 'POST', { projectId: 1, name: 'Create a task backlog through chat', fullName, - automated: true, description: 'Canonical scenario: tasks.create-backlog. Readiness and recorded outcomes remain separate.', - expectedResult: 'The real agent creates the requested backlog; public Product E2E assertions and cleanup must pass.' }); - if (!id(created?.id) || created.projectId !== 1 || created.fullName !== fullName) fail('testops_created_case_identity_invalid'); - selected = { id: created.id, fullName, automated: created.automated === true }; - } - if (!plan) { - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: [selected.id] } }); - if (!id(created?.id) || created.projectId !== 1 || created.name !== planName) fail('testops_created_plan_identity_invalid'); - plan = { id: created.id, name: planName }; - } - const actual = await api(`/api/testplan/${plan.id}`); - if (actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== 1 - || actual.treeSelection?.inverted === true || !same(actual.treeSelection?.leafsInclude, [selected.id]) - || (actual.treeSelection?.groupsInclude?.length ?? 0) > 0 || (actual.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('testops_plan_scope_mismatch'); - return { projectId: 1, case: selected, plan: { id: plan.id, name: planName, count: 1 }, - caseUrl: `${endpoint}/project/1/test-cases/${selected.id}`, planUrl: `${endpoint}/testplan/${plan.id}`, - actualResult: null, note: 'Plan preparation is not an executed measurement or a passing result.' }; +// Kept as an explicit retirement boundary for callers of the former smoke lane. +// Catalog registration owns the shared cases; deleting the old plan must persist. +export async function ensureChatPlan() { + fail('testops_plan_retired'); } diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 318c616208c..68f689beee6 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -31,64 +31,14 @@ test('multiple alias identities, foreign project and incomplete metadata stop be await assert.rejects(inspectChatMetadata(async () => ({ content: [], totalPages: 11, totalElements: 1100, number: 0 })), /catalog_incomplete/); }); -test('existing exact plan reuses case and plan without a write', async () => { - const result = await ensureChatPlan(async route => { assert.equal(route, '/api/testplan/8'); return plan; }, - await inspectChatMetadata(inspectApi()), () => assert.fail('must not mark or write')); - assert.equal(result.case.id, 91); - assert.equal(result.plan.id, 8); - assert.equal(result.actualResult, null); -}); - -test('bootstrap creates only the missing exact plan after exclusive marker and reads back scope', async () => { - const calls = []; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - if (method === 'POST') { assert.equal(calls[0].route, 'MARK'); assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [91] } }); } - return plan; - }, { projectId: 1, cases: [caseRow], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(calls.filter(row => row.method === 'POST').length, 1); - assert.equal(result.plan.count, 1); -}); - -test('empty project bootstrap links the genuinely created case into only the new plan', async () => { - const calls = []; - const createdCase = { ...caseRow, id: 92 }; - const createdPlan = { ...plan, treeSelection: { inverted: false, leafsInclude: [92] } }; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - assert.equal(calls[0].route, 'MARK'); - if (route === '/api/testcase') { - assert.equal(body.projectId, 1); assert.equal(body.fullName, fullName); assert.equal(body.automated, true); - return createdCase; - } - if (route === '/api/testplan') { - assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [92] } }); - return createdPlan; - } - assert.equal(route, '/api/testplan/8'); assert.equal(method, undefined); return createdPlan; - }, { projectId: 1, cases: [], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(result.case.id, 92); assert.equal(result.plan.id, 8); - assert.deepEqual(calls.map(call => call.route), ['MARK', '/api/testcase', '/api/testplan', '/api/testplan/8']); -}); - -test('known conflicting plan stops before creating an otherwise missing case or marker', async () => { - await assert.rejects(ensureChatPlan(() => assert.fail('no API write or read needed'), - { projectId: 1, cases: [], plans: [plan] }, () => assert.fail('must not mark')), /plan_without_matching_case/); -}); - -test('new case is created only after alias reconciliation; an uncertain mutation is never resent', async () => { - let writes = 0, markers = 0; - await assert.rejects(ensureChatPlan(async (_route, method) => { - assert.equal(method, 'POST'); writes++; throw new Error('uncertain transport'); - }, { projectId: 1, cases: [], plans: [] }, async () => { markers++; }), /uncertain/); - assert.equal(writes, 1); assert.equal(markers, 1); -}); - -test('existing broad or conflicting plan never expands or changes its scope', async () => { - for (const actual of [{ ...plan, testCasesCount: 2 }, { ...plan, treeSelection: { leafsInclude: [92] } }, - { ...plan, treeSelection: { leafsInclude: [91], inverted: true } }, { ...plan, projectId: 2 }]) { - await assert.rejects(ensureChatPlan(async (_route, method) => { assert.equal(method, undefined); return actual; }, - { projectId: 1, cases: [caseRow], plans: [plan] }, () => assert.fail('no writes')), /scope_mismatch/); +test('retired singleton bootstrap cannot recreate an absent plan or mutate retained metadata', async () => { + for (const inspected of [ + { projectId: 1, cases: [], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [plan] }, + ]) { + await assert.rejects(ensureChatPlan(() => assert.fail('retired route must not call the API'), inspected, + () => assert.fail('retired route must not mark a mutation')), /testops_plan_retired/); } }); diff --git a/tests/runner-e2e/chat-feature-testops-cli.test.mjs b/tests/runner-e2e/chat-feature-testops-cli.test.mjs index 107f046b109..70238eb15f0 100644 --- a/tests/runner-e2e/chat-feature-testops-cli.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-cli.test.mjs @@ -9,7 +9,7 @@ const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/ GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; -test('actual metadata bootstrap caller creates no result or agent and reuses exact case/plan', async t => { +test('actual retired singleton caller refuses plan recreation and creates no result', async t => { const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-bootstrap-'))); t.after(() => rm(temporary, { recursive: true })); const state = path.join(temporary, 'state'), requests = []; @@ -30,9 +30,10 @@ test('actual metadata bootstrap caller creates no result or agent and reuses exa return response({ id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { leafsInclude: [91], inverted: false } }); }; assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, - CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 0); - const plan = JSON.parse(await readFile(path.join(state, 'plan.json'))); - assert.equal(plan.actualResult, null); + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 1); + const failure = JSON.parse(await readFile(path.join(state, 'failure.json'))); + assert.equal(failure.code, 'testops_plan_retired'); + await assert.rejects(readFile(path.join(state, 'plan.json')), { code: 'ENOENT' }); assert.equal(requests.filter(value => value.method === 'POST').length, 1, 'only auth token exchange; no case/plan write'); await assert.rejects(readFile(path.join(state, 'allure-results.json')), { code: 'ENOENT' }); }); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 917d6912f90..65c32e7a7e9 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -244,18 +244,6 @@ test('post-watch foreign backend binding returns safe nonzero and preserves orig assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); }); -test('bootstrap workflow captures official context only after watch and retains exactly named closed receipts', async () => { - const workflow = await readFile(new URL('../../.github/workflows/chat-feature-testops.yml', import.meta.url), 'utf8'); - const start = workflow.indexOf(' - name: Register the public job without an evaluation or a synthetic result'); - const end = workflow.indexOf(' - name: Validate the one real TestOps selection', start); - const step = workflow.slice(start, end); - assert.ok(step.includes("if: needs.authorize.outputs.operation == 'bootstrap'")); - assert.match(step, /allurectl watch -- node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap\n\s+node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap-context/); - assert.equal((step.match(/allurectl watch/g) ?? []).length, 1); assert.ok(!step.includes('continue-on-error')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration.json')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json')); -}); - test('post-watch refuses nonprivate state custody before any command auth or receipt write', async t => { const f = await bootstrapContextFixture(t); await chmod(f.state, 0o755); const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); diff --git a/tests/runner-e2e/testops-chat-workflow.test.ts b/tests/runner-e2e/testops-chat-workflow.test.ts deleted file mode 100644 index 14484cbd30e..00000000000 --- a/tests/runner-e2e/testops-chat-workflow.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { readFileSync } from "node:fs"; -import { parse } from "yaml"; -import { expect, it } from "vitest"; -import { validateChatTestOpsDispatch } from "./testops-chat-dispatch.js"; - -const workflow = parse(readFileSync(new URL("../../.github/workflows/chat-feature-testops.yml", import.meta.url), "utf8")); -function context() { - return { env: { GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", - GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", GITHUB_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", GITHUB_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]" }, - event: { repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, ref: "refs/heads/master", sender: { id: 123 }, inputs: { ALLURE_JOB_RUN_ID: "456" } as Record } }; -} -it("admits the actual TestOps service input and only the closed metadata bootstrap alternative", () => { - const f = context(); expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - f.event.inputs = { operation: "bootstrap" }; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("bootstrap"); - f.event.inputs.operation = "launch-plan"; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("launch-plan"); - f.event.inputs.ALLURE_JOB_RUN_ID = "456"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); - delete f.event.inputs.ALLURE_JOB_RUN_ID; - f.event.inputs.operation = "other"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); -}); -it.each(["unlisted-trigger", "wrong-source", "replay", "extra-input", "missing-jobrun", "foreign-repo"])("rejects %s before any service credential operation", kind => { - const f = context(); let trigger = 123; - if (kind === "unlisted-trigger") trigger = 456; - if (kind === "wrong-source") f.env.GITHUB_REF = "refs/heads/unreviewed"; - if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; - if (kind === "extra-input") f.event.inputs.target_branch = "other"; - if (kind === "missing-jobrun") f.event.inputs = {}; - if (kind === "foreign-repo") f.event.repository.full_name = "foreign/repo"; - expect(() => validateChatTestOpsDispatch(f.env, f.event, trigger)).toThrow(); -}); -it("keeps provider and TestOps service credentials in separate jobs", () => { - const jobs = workflow.jobs; - for (const [id, job] of Object.entries(jobs) as Array<[string, any]>) { - const text = JSON.stringify(job); - if (text.includes("secrets.OPENROUTER_API_KEY")) expect(id).toBe("chat"); - if (text.includes("secrets.ALLURE_TOKEN")) expect(["select", "upload"]).toContain(id); - if (id !== "chat") expect(text).not.toContain("pnpm test:e2e:runner --"); - expect(job.steps.some((step: any) => step.run === "node tests/runner-e2e/testops-chat-dispatch.ts")).toBe(true); - } - expect(jobs.chat.environment).toBe("runner-e2e-private-paid"); - expect(jobs.select.environment).toBe("runner-e2e-testops-reporting"); - expect(jobs.upload.environment).toBe("runner-e2e-testops-reporting"); -}); -it("requires token-isolated selection before the fixed AWS run and keeps bootstrap outside all model jobs", () => { - expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); - expect(Object.keys(workflow.on.workflow_dispatch.inputs)).toEqual(["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"]); - expect(workflow.on.workflow_dispatch.inputs.operation.default).toBe("evaluate"); - expect(workflow.jobs.chat.needs).toContain("select"); expect(workflow.jobs.build.needs).toContain("select"); - expect(workflow.jobs.chat.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.jobs.build.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - const paid = workflow.jobs.chat.steps.find((s: any) => s.env?.OPENROUTER_API_KEY); - expect(paid.run).toContain("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0"); - expect(workflow.jobs.chat.steps.findIndex((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection")).toBeLessThan(workflow.jobs.chat.steps.indexOf(paid)); - expect(workflow.jobs.authorize.outputs.runner).toBe("runs-on/fleet=paperclip-public-pr-x64/env=public-ci"); - expect(workflow.jobs.chat["runs-on"]).toBe("${{ needs.authorize.outputs.runner }}"); -}); -it("uploads only the trusted safe projection after successful strict private report retention", () => { - const steps = workflow.jobs.report.steps; - expect(steps.find((s: any) => s.id === "export").if).toBe("always() && steps.private_report.outcome == 'success'"); - expect(steps.find((s: any) => s.with?.name?.startsWith("runner-e2e-testops-")).with.path).toBe("runner-e2e-testops/"); - expect(JSON.stringify(workflow)).not.toContain("MANUAL_PRIVATE_APPROVALS_JSON"); - expect(Object.values(workflow.jobs).flatMap((j: any) => j.steps).filter((s: any) => s.uses?.startsWith("actions/upload-artifact")).some((s: any) => s.with.path === "tests/runner-e2e/results/")).toBe(false); -}); - -it("initializes the actual TestOps association and never fabricates setup results", () => { - const steps = workflow.jobs.select.steps; - const installer = steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")); - expect(installer.uses).toBe("allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db"); - expect(installer.with["allurectl-version"]).toBe("2.23.1"); - const bootstrap = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs bootstrap")); - expect(bootstrap.run).toContain("mkdir -m 700"); - expect(bootstrap.run).toContain("allurectl watch -- node"); - expect(bootstrap.env.ALLURE_RESULTS).toBe("${{ runner.temp }}/chat-testops-bootstrap-results"); - const select = steps.find((s: any) => s.id === "selection"); - expect(select.run).toContain("allurectl watch -- node"); - for (const name of ["CHAT_TESTOPS_CASE_ID", "CHAT_TESTOPS_PLAN_ID", "CHAT_TESTOPS_JOB_ID"]) - expect(select.env[name]).toBe(`\${{ vars.${name} }}`); - const metadata = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-setup-")); - expect(metadata.if).toBe("always() && needs.authorize.outputs.operation == 'bootstrap'"); - expect(metadata.with.path).not.toContain("results"); -}); -it("authenticates the same-run selection again before converting and uploading", () => { - const steps = workflow.jobs.upload.steps; - const receipt = steps.find((s: any) => s.with?.path === "chat-testops-selection"); - expect(receipt.with["artifact-ids"]).toBe("${{ needs.select.outputs.artifact_id }}"); - const upload = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs upload")); - expect(upload.run).toContain("allurectl watch -- node"); - expect(upload.env.EXPECTED_SELECTION_SHA256).toBe("${{ needs.select.outputs.sha256 }}"); - expect(steps.indexOf(receipt)).toBeLessThan(steps.indexOf(upload)); - expect(JSON.stringify(workflow.jobs.select)).not.toContain("OPENROUTER_API_KEY"); - expect(JSON.stringify(workflow.jobs.upload)).not.toContain("OPENROUTER_API_KEY"); -}); - -it("keeps the one plan launch outside Allure watch and every AWS/model job", () => { - const steps = workflow.jobs.select.steps; - const launch = steps.find((s: any) => s.run?.endsWith("testops-chat-reporter.mjs launch-plan")); - expect(launch.if).toBe("needs.authorize.outputs.operation == 'launch-plan'"); - expect(launch.run).not.toContain("watch"); - expect(launch.env.CHAT_TESTOPS_OPERATION_PARAMETER_ID).toBe("${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}"); - expect(launch.env.CHAT_TESTOPS_INTEGRATION_ID).toBe("${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_NONCE).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}"); - expect(launch.env.TESTOPS_TRIGGERING_ACTOR_ID).toBe("${{ needs.authorize.outputs.triggering_actor_id }}"); - expect(launch.env.TESTOPS_ALLOWED_ACTOR_IDS).toBe("${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - expect(launch.env.GH_TOKEN).toBe("${{ github.token }}"); - expect(launch.run).not.toContain("sleep"); - expect(steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")).if).toBe("needs.authorize.outputs.operation != 'launch-plan'"); - for (const name of ["build", "chat"]) expect(workflow.jobs[name].if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.concurrency["cancel-in-progress"]).toBe(false); -}); - -it("accepts the documented TestOps username metadata without treating it as an actor", () => { - const f = context(); f.event.inputs.ALLURE_USERNAME = "service-metadata"; - expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - expect(() => validateChatTestOpsDispatch(f.env, f.event, 456)).toThrow(); - expect(JSON.stringify(workflow.jobs)).not.toContain("inputs.ALLURE_USERNAME"); -}); - -it("includes every native Node helper control in the normal unit owner", () => { - const launcher = readFileSync(new URL("./run-unit.mjs", import.meta.url), "utf8"); - const config = readFileSync(new URL("./vitest.config.ts", import.meta.url), "utf8"); - for (const file of ["chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"]) { - expect(launcher).toContain(`"tests/runner-e2e/${file}"`); - expect(config).toContain(`"${file}"`); - } -}); - -it("verifies the actual server outcome after watch without masking an original failed upload", () => { - const steps = workflow.jobs.upload.steps; - const watch = steps.find((s: any) => s.id === "allure_upload"); - const verify = steps.find((s: any) => s.id === "verify_upload"); - expect(watch.run).toContain("allurectl watch -- node"); - expect(watch["continue-on-error"]).toBeUndefined(); - expect(verify.run).toBe("node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops"); - expect(verify.if).toBe("always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')"); - expect(verify.env).toEqual(watch.env); - expect(steps.indexOf(verify)).toBeGreaterThan(steps.indexOf(watch)); - const proof = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-verification-")); - expect(proof.with.path.split("\n").filter(Boolean)).toHaveLength(2); - expect(proof.with.path).not.toContain("allure-results"); -}); - -it("carries only the existing source-reviewed lock through build and worker install", () => { - const build = workflow.jobs.build.steps; - const install = build.find((s: any) => s.run?.includes("pnpm install --frozen-lockfile")); - expect(install.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(install.run).not.toContain("--lockfile-only"); - const archive = build.find((s: any) => s.run?.includes("tar -czf chat-testops-build")); - expect(archive.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(archive.run).toContain("chat-testops-lock.json pnpm-lock.yaml"); - const worker = workflow.jobs.chat.steps.find((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection"); - expect(worker.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json"); - expect(worker.run).toContain("cmp chat-testops-lock.json chat-testops-worker-lock.json"); - expect(worker.run.indexOf("cmp chat-testops-lock.json")).toBeLessThan(worker.run.indexOf("pnpm install --frozen-lockfile")); -}); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 91c0e3e9f29..ca84b9de5ab 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -606,8 +606,6 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -893,24 +891,7 @@ describe("public repository paid workflow security", () => { for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); - if (["chat-feature-testops.yml", "chat-completion-testops.yml"].includes(filename)) { - const guard = filename === "chat-feature-testops.yml" ? "testops-chat-dispatch.ts" : "chat-completion-testops-cli.mjs guard"; - expect(other).toContain(`run: node tests/runner-e2e/${guard}`); - expect(other).toContain("ref: ${{ github.sha }}"); - expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; - expect(chat).toContain("environment: runner-e2e-private-paid"); - expect(chat).toContain("needs: [authorize, select, build]"); - expect(chat).toContain(" - *guard"); - expect(chat).toContain("verify-selection"); - expect(chat).not.toContain("ALLURE_TOKEN"); - expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); - const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; - expect(report).toContain("environment: runner-e2e-private-history"); - expect(report).toContain("manual-private-transfer.ts report-session"); - expect(report).not.toContain("ALLURE_TOKEN"); - continue; - } + for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index abfc428b39a..739487a577c 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -194,6 +194,7 @@ export type IssueListFilters = { projection?: "index" | "rows"; attention?: "blocked"; status?: string; + displayStatus?: string; projectId?: string; parentId?: string; assigneeAgentId?: string; @@ -220,7 +221,7 @@ export type IssueListFilters = { q?: string; limit?: number; offset?: number; - sortField?: "updated" | "id"; + sortField?: "updated" | "created" | "id"; afterId?: string; sortDir?: "asc" | "desc"; }; @@ -229,6 +230,7 @@ function issueListSearchParams(filters?: IssueListFilters) { const params = new URLSearchParams(); if (filters?.attention) params.set("attention", filters.attention); if (filters?.status) params.set("status", filters.status); + if (filters?.displayStatus) params.set("displayStatus", filters.displayStatus); if (filters?.projectId) params.set("projectId", filters.projectId); if (filters?.parentId) params.set("parentId", filters.parentId); if (filters?.parentIds) params.set("parentIds", filters.parentIds.join(",")); diff --git a/ui/src/api/search.test.ts b/ui/src/api/search.test.ts new file mode 100644 index 00000000000..9f2d39324e5 --- /dev/null +++ b/ui/src/api/search.test.ts @@ -0,0 +1,14 @@ +import { expect, it, vi } from "vitest"; +import { parseSearchQuery } from "../lib/search-query-parser"; +import { searchApi } from "./search"; +import { api } from "./client"; + +vi.mock("./client", () => ({ api: { get: vi.fn() } })); + +it("keeps scheduled monitor waits in is:open search requests", async () => { + const parsed = parseSearchQuery("is:open deployment", {}); + await searchApi.search("company-1", { q: parsed.query, ...parsed.filters }); + const url = new URL(vi.mocked(api.get).mock.calls[0]![0], "http://localhost"); + expect(url.searchParams.getAll("displayStatus")).toEqual(["backlog", "todo", "in_progress", "monitoring", "in_review", "blocked"]); + expect(url.searchParams.get("q")).toBe("deployment"); +}); diff --git a/ui/src/api/search.ts b/ui/src/api/search.ts index 5715d8685b8..b9e97435685 100644 --- a/ui/src/api/search.ts +++ b/ui/src/api/search.ts @@ -1,4 +1,4 @@ -import type { CompanySearchResponse, CompanySearchScope, CompanySearchSort, IssuePriority, IssueStatus } from "@paperclipai/shared"; +import type { CompanySearchResponse, CompanySearchScope, CompanySearchSort, IssuePriority, IssueDisplayStatus } from "@paperclipai/shared"; import { api } from "./client"; export interface CompanySearchParams { @@ -6,7 +6,7 @@ export interface CompanySearchParams { scope?: CompanySearchScope; limit?: number; offset?: number; - status?: IssueStatus[]; + status?: IssueDisplayStatus[]; priority?: IssuePriority[]; assigneeAgentId?: string | null; assigneeUserId?: string; @@ -28,7 +28,7 @@ export const searchApi = { if (params.scope) search.set("scope", params.scope); if (params.limit !== undefined) search.set("limit", String(params.limit)); if (params.offset !== undefined) search.set("offset", String(params.offset)); - appendMulti(search, "status", params.status); + appendMulti(search, "displayStatus", params.status); appendMulti(search, "priority", params.priority); if (params.assigneeAgentId !== undefined) search.set("assigneeAgentId", params.assigneeAgentId ?? "null"); if (params.assigneeUserId !== undefined) search.set("assigneeUserId", params.assigneeUserId); diff --git a/ui/src/components/BlockedInboxView.tsx b/ui/src/components/BlockedInboxView.tsx index 0cb145a359f..fb854d377df 100644 --- a/ui/src/components/BlockedInboxView.tsx +++ b/ui/src/components/BlockedInboxView.tsx @@ -376,7 +376,7 @@ function BlockedInboxRow({ showDivider={presentation === "legacy"} statusSlot={presentation === "task" ? showStatusColumn - ? + ? :