diff --git a/.github/workflows/chat-catalog-testops-case.yml b/.github/workflows/chat-catalog-testops-case.yml new file mode 100644 index 00000000000..38229c9f070 --- /dev/null +++ b/.github/workflows/chat-catalog-testops-case.yml @@ -0,0 +1,318 @@ +name: Chat catalog TestOps case + +on: + workflow_call: + inputs: + protocol: + type: string + required: true + cell: + type: string + required: true + target_sha: + type: string + required: true + runner: + type: string + required: true + selection_artifact_id: + type: string + required: true + selection_sha256: + type: string + required: true + build_artifact_id: + type: string + required: true + ALLURE_JOB_RUN_ID: + type: string + required: true + secrets: + OPENROUTER_API_KEY: + required: false + ANTHROPIC_API_KEY: + required: false + ALLURE_TOKEN: + required: false + +permissions: + contents: read + actions: read + +env: + NODE_OPTIONS: '' + NODE_PATH: '' + AWS_EC2_METADATA_DISABLED: 'true' + +jobs: + chat: + name: Execute the original selected case + runs-on: ${{ inputs.runner }} + timeout-minutes: ${{ fromJSON(inputs.cell).timeout_minutes }} + environment: runner-e2e-private-paid + permissions: + contents: read + actions: read + id-token: write + env: &retention + TARGET_SHA: ${{ inputs.target_sha }} + MANUAL_PRIVATE_IDS: ${{ fromJSON(inputs.cell).private_ids }} + MANUAL_PRIVATE_APPROVALS_JSON: ${{ fromJSON(inputs.cell).private_policy }} + RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} + steps: + - &checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - &node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - &guard + name: Verify the exact reusable protocol and main-only caller + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + CHAT_TESTOPS_PROTOCOL: ${{ inputs.protocol }} + run: | + test "$CHAT_TESTOPS_PROTOCOL" = streaming-v1 + node tests/runner-e2e/chat-completion-testops-cli.mjs guard + - &pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: 'false' + NPM_CONFIG_FUND: 'false' + NPM_CONFIG_UPDATE_NOTIFIER: 'false' + with: + version: 9.15.4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.target_sha }} + persist-credentials: false + - name: Reject target-owned trusted helper paths + id: target_boundary + run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + path: .testops-trusted + persist-credentials: false + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ inputs.build_artifact_id }} + path: chat-completion-build + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ inputs.selection_artifact_id }} + path: chat-completion-testops-selection + - name: Verify source-bound build and authenticated selection before credentials + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + run: | + set -euo pipefail + test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" + node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json + (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) + tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" + pnpm install --frozen-lockfile --ignore-scripts + (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") + - name: Qualify the pinned provider runtime and preinstalled browser + run: | + node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs + node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' + google-chrome --version + pnpm exec playwright install ffmpeg + - name: Bind exact private evidence under the authenticated TestOps authority + id: private + env: + TARGET_ROOT: ${{ github.workspace }} + SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare + - name: Obtain only this cell's exact-key storage session + id: cell_credentials + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.private.outputs.session_policy }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Verify private custody before delivering the model key + env: &cell_session + SELECTED_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }} + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify + - name: Execute only the selected original case with zero retries + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ contains(fromJSON(inputs.cell).required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }} + PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json + PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).executionId }} + PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration + PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome + run: pnpm test:e2e:runner -- --id "${{ fromJSON(inputs.cell).executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 + - name: Retain original evidence after success or failure + if: always() && steps.cell_credentials.outcome == 'success' + env: *cell_session + run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell + + report: + name: Retain private originals and build the exact safe TestOps projection + needs: chat + if: always() && !cancelled() && (needs.chat.result == 'success' || needs.chat.result == 'failure') + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: runner-e2e-private-history + permissions: + contents: read + actions: read + id-token: write + env: *retention + outputs: + bundle_id: ${{ steps.bundle.outputs.artifact-id }} + steps: + - *checkout + - *node + - *guard + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ inputs.selection_artifact_id }} + path: chat-completion-testops-selection + - *pnpm + - name: Install trusted reporting dependencies with lifecycle scripts disabled + run: | + node tests/runner-e2e/prepare-report-lock.mjs + pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" + - name: Prepare unchanged exact-key report and cell policies + id: session + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session + - name: Obtain exact first-cell reader + id: reader_0 + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 + with: + role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} + aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} + output-env-credentials: false + output-credentials: true + disable-retry: true + action-timeout-s: 60 + - name: Download the first exact original + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + SELECTED_CELL_INDEX: '0' + AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} + AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} + AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} + AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell + - name: Verify every selected original and its completeness receipt + env: + PRIVATE_REPORT_ROOT: ${{ github.workspace }} + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads + - name: Build the existing report while preserving failed source grades + id: normalize + continue-on-error: true + env: + PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e + PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized + PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ fromJSON(inputs.cell).private_ids }} + PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 + PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ inputs.target_sha }} + PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration + run: | + mkdir -p runner-e2e-merged-report + pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 + - name: Export only exact reviewed synthetic fixture results + id: export + if: always() && steps.reader_0.outcome == 'success' + run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts + - name: Require a complete safe projection even after a failed normalization + if: always() + env: + EXPORT_OUTCOME: ${{ steps.export.outcome }} + run: test "$EXPORT_OUTCOME" = success + - name: Freeze one safe Allure bundle before any TestOps transmission + id: prepare + if: always() && !cancelled() && steps.export.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }} + CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }} + ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure + run: | + mkdir -m 700 "$RUNNER_TEMP/chat-catalog-case-bundle" + cp -R runner-e2e-testops "$RUNNER_TEMP/chat-catalog-case-bundle/projection" + node tests/runner-e2e/chat-completion-testops-cli.mjs prepare-case-upload "$RUNNER_TEMP/chat-catalog-case-bundle/projection" + cp "$RUNNER_TEMP/chat-completion-testops-prepare-case-upload/prepared.json" "$RUNNER_TEMP/chat-catalog-case-bundle/prepared.json" + - name: Retain the immutable metadata-only case bundle + id: bundle + if: always() && !cancelled() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-catalog-case-bundle-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }} + path: ${{ runner.temp }}/chat-catalog-case-bundle/ + retention-days: 30 + if-no-files-found: error + + upload: + name: Publish only this original case without closing the shared job run + needs: report + if: always() && !cancelled() && (needs.report.result == 'success' || needs.report.result == 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + environment: runner-e2e-testops-reporting + steps: + - *checkout + - *node + - *guard + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ inputs.selection_artifact_id }} + path: chat-completion-testops-selection + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + artifact-ids: ${{ needs.report.outputs.bundle_id }} + path: ${{ runner.temp }}/chat-catalog-case-bundle + - name: Admit the frozen singleton and submit it at most once + id: publication + env: + GH_TOKEN: ${{ github.token }} + TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + EXPECTED_SELECTION_SHA256: ${{ inputs.selection_sha256 }} + CHAT_TESTOPS_EXECUTION_ID: ${{ fromJSON(inputs.cell).executionId }} + CHAT_TESTOPS_PREPARED_FILE: ${{ runner.temp }}/chat-catalog-case-bundle/prepared.json + ALLURE_RESULTS: ${{ runner.temp }}/chat-catalog-case-bundle/allure + run: node tests/runner-e2e/chat-completion-testops-cli.mjs upload-case "$RUNNER_TEMP/chat-catalog-case-bundle/projection" + - name: Retain only the closed singleton publication receipt + if: always() && !cancelled() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: chat-catalog-case-publication-${{ github.run_id }}-1-${{ fromJSON(inputs.cell).index }} + path: | + ${{ runner.temp }}/chat-completion-testops-upload-case/publication.json + ${{ runner.temp }}/chat-completion-testops-upload-case/failure.json + ${{ runner.temp }}/chat-completion-testops-upload-case/once-upload_case.json + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/chat-catalog-testops.yml b/.github/workflows/chat-catalog-testops.yml index db340509033..30eff3032a5 100644 --- a/.github/workflows/chat-catalog-testops.yml +++ b/.github/workflows/chat-catalog-testops.yml @@ -23,7 +23,7 @@ permissions: actions: read concurrency: - group: chat-catalog-testops + group: chat-catalog-testops-streaming-v1 cancel-in-progress: false env: @@ -78,8 +78,7 @@ jobs: - *checkout - *node - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' + - if: needs.authorize.outputs.operation != 'launch-plan' uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 with: allurectl-version: 2.23.1 @@ -204,107 +203,24 @@ jobs: fail-fast: false max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }} matrix: ${{ fromJSON(needs.select.outputs.native_matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: ${{ matrix.timeout_minutes }} - environment: runner-e2e-private-paid permissions: contents: read actions: read id-token: write - env: &retention - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - MANUAL_PRIVATE_IDS: ${{ matrix.private_ids }} - MANUAL_PRIVATE_APPROVALS_JSON: ${{ matrix.private_policy }} - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - steps: &catalog_worker_steps - - *checkout - - *node - - *guard - - *pnpm - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-completion-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - name: Verify source-bound build and authenticated selection before credentials - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" - node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json - (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) - tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") - - name: Qualify the pinned provider runtime and preinstalled browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Bind exact private evidence under the authenticated TestOps authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only this cell's exact-key storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify private custody before delivering the model key - env: &cell_session - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the selected original case with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }} - PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain original evidence after success or failure - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell + uses: ./.github/workflows/chat-catalog-testops-case.yml + with: + protocol: streaming-v1 + cell: ${{ toJSON(matrix) }} + target_sha: ${{ needs.authorize.outputs.target_sha }} + runner: ${{ needs.authorize.outputs.runner }} + selection_artifact_id: ${{ needs.select.outputs.artifact_id }} + selection_sha256: ${{ needs.select.outputs.selection_sha256 }} + build_artifact_id: ${{ needs.build.outputs.artifact_id }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + secrets: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} chat_external: name: ${{ matrix.executionId }} @@ -314,102 +230,29 @@ jobs: fail-fast: false max-parallel: 1 matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: ${{ matrix.timeout_minutes }} - environment: runner-e2e-private-paid permissions: contents: read actions: read id-token: write - env: *retention - steps: *catalog_worker_steps - - report: - name: Retain private originals and build the exact safe TestOps projection - needs: [authorize, select, build, chat, chat_external] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure') - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - strategy: - fail-fast: false - max-parallel: 4 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies with lifecycle scripts disabled - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" - - name: Prepare unchanged exact-key report and cell policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain exact first-cell reader - id: reader_0 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the first exact original - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Verify every selected original and its completeness receipt - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing report while preserving failed source grades - id: normalize - continue-on-error: true - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ matrix.private_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Export only exact reviewed synthetic fixture results - id: export - if: always() && steps.reader_0.outcome == 'success' - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts - - name: Require a complete safe projection even after a failed normalization - if: always() - env: - EXPORT_OUTCOME: ${{ steps.export.outcome }} - run: test "$EXPORT_OUTCOME" = success - - name: Retain the metadata-only TestOps projection - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-catalog-testops-cell-${{ github.run_id }}-${{ matrix.key }} - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error + uses: ./.github/workflows/chat-catalog-testops-case.yml + with: + protocol: streaming-v1 + cell: ${{ toJSON(matrix) }} + target_sha: ${{ needs.authorize.outputs.target_sha }} + runner: ${{ needs.authorize.outputs.runner }} + selection_artifact_id: ${{ needs.select.outputs.artifact_id }} + selection_sha256: ${{ needs.select.outputs.selection_sha256 }} + build_artifact_id: ${{ needs.build.outputs.artifact_id }} + ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} + secrets: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} aggregate: name: Validate and merge every selected metadata-only projection - needs: [authorize, select, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && (needs.report.result == 'success' || needs.report.result == 'failure') + needs: [authorize, select, build, chat, chat_external] + if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure' || needs.chat_external.result == 'success' || needs.chat_external.result == 'failure') runs-on: ubuntu-latest timeout-minutes: 15 outputs: @@ -429,8 +272,15 @@ jobs: path: chat-completion-testops-selection - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - pattern: chat-catalog-testops-cell-${{ github.run_id }}-* - path: catalog-projections + pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-* + path: catalog-bundles + - name: Gather only safe singleton projections from immutable bundles + run: | + mkdir -m 700 catalog-projections + for bundle in catalog-bundles/*; do + test -d "$bundle/projection" + cp -R "$bundle/projection" "catalog-projections/${bundle##*/}" + done - name: Authenticate selection and validate all singleton exports before aggregation env: GH_TOKEN: ${{ github.token }} @@ -449,9 +299,9 @@ jobs: retention-days: 30 if-no-files-found: error - upload: - name: Record and independently verify each TestOps result - needs: [authorize, select, chat, chat_external, report, aggregate] + verify: + name: Verify final coverage without uploading any result again + needs: [authorize, select, aggregate] if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.aggregate.result == 'success' runs-on: ubuntu-latest timeout-minutes: 10 @@ -468,10 +318,17 @@ jobs: with: artifact-ids: ${{ needs.select.outputs.artifact_id }} path: chat-completion-testops-selection - - *allure - - name: Upload original outcomes, checks and private-evidence receipts - id: allure_upload - env: &upload_env + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: chat-catalog-case-bundle-${{ github.run_id }}-1-* + path: catalog-bundles + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: chat-catalog-case-publication-${{ github.run_id }}-1-* + path: catalog-publications + - name: Authenticate every frozen publication and verify the exact original selection + id: verification + env: GH_TOKEN: ${{ github.token }} TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} @@ -479,20 +336,14 @@ jobs: ALLURE_PROJECT_ID: '1' ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results - run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops - - name: Verify actual selected results and complete result-owned attachments - id: verification - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *upload_env - run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops - - name: Retain closed TestOps verification receipts - if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') + CHAT_TESTOPS_BUNDLES: ${{ github.workspace }}/catalog-bundles + CHAT_TESTOPS_PUBLICATIONS: ${{ github.workspace }}/catalog-publications + run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-stream runner-e2e-testops + - name: Retain closed final coverage verification + if: always() && !cancelled() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: chat-completion-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-verify-upload/ - ${{ runner.temp }}/chat-completion-testops-upload/exported.json + path: ${{ runner.temp }}/chat-completion-testops-verify-stream/ retention-days: 30 if-no-files-found: error diff --git a/.github/workflows/chat-completion-testops.yml b/.github/workflows/chat-completion-testops.yml deleted file mode 100644 index 7016f76dfec..00000000000 --- a/.github/workflows/chat-completion-testops.yml +++ /dev/null @@ -1,504 +0,0 @@ -name: Chat completion TestOps -run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Run the exact TestOps selection, register its plan, or launch it once - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps job-run identity; source, model and clocks are fixed - ALLURE_USERNAME: - type: string - required: false - description: Reserved service metadata; never grants authority - -permissions: - contents: read - actions: read - -concurrency: - group: chat-completion-testops - cancel-in-progress: false - -env: - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - operation: ${{ steps.guard.outputs.operation }} - target_sha: ${{ steps.guard.outputs.target_sha }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify actual main-only dispatch and both actors - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard - - select: - name: Register the dedicated plan or validate its real TestOps selection - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-testops-reporting - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} - execution_ids: ${{ steps.selection.outputs.execution_ids }} - matrix: ${{ steps.selection.outputs.matrix }} - policy: ${{ steps.selection.outputs.policy }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register three exact cases with no synthetic test result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap - node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context - - name: Validate case, plan, job, launch and same-run source before provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection - - name: Request the one reviewed plan launch and exit - if: needs.authorize.outputs.operation == 'launch-plan' - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }} - CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }} - CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }} - run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan - - name: Retain token-free registration and launch receipts - if: always() && needs.authorize.outputs.operation != 'evaluate' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-bootstrap/ - ${{ runner.temp }}/chat-completion-testops-launch-plan/ - retention-days: 30 - if-no-files-found: error - - name: Retain the exact authenticated selection - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-selection-${{ github.run_id }}-1 - path: testops-pilot-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable catalog source without service or model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - name: Resolve the fixed candidate graph with lifecycle scripts disabled - run: | - pnpm install --lockfile-only --ignore-scripts - pnpm install --frozen-lockfile --ignore-scripts - - name: Build existing TypeScript and native Runner outputs - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - - name: Exercise the real workflow and pinned source host contract before model access - run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs - - name: Build native Runner outputs - run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package the resolved lock and exact source outputs - env: - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$TARGET_SHA" - tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-build-${{ github.run_id }}-1 - path: | - chat-completion-build.tar.gz - chat-completion-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: ${{ matrix.executionId }} - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - strategy: - fail-fast: false - max-parallel: 1 - matrix: ${{ fromJSON(needs.select.outputs.matrix) }} - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - actions: read - id-token: write - env: &retention - TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} - MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }} - MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }} - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - steps: - - *checkout - - *node - - *guard - - *pnpm - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.target_sha }} - persist-credentials: false - - name: Reject target-owned trusted helper paths - id: target_boundary - run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)" - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-completion-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - name: Verify source-bound build and authenticated selection before credentials - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA" - node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json - (cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256) - tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE" - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store") - - name: Qualify the pinned provider runtime and preinstalled browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Bind exact private evidence under the authenticated TestOps authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only this cell's exact-key storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify private custody before delivering the model key - env: &cell_session - SELECTED_EXECUTION_ID: ${{ matrix.executionId }} - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the selected original case with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1 - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }} - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain original evidence after success or failure - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and build the exact safe TestOps projection - needs: [authorize, select, build, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure') - runs-on: ubuntu-latest - timeout-minutes: 20 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies with lifecycle scripts disabled - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store" - - name: Prepare unchanged exact-key report and cell policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact first-cell reader - id: reader_0 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the first exact original - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact second-cell reader - id: reader_1 - if: fromJSON(steps.session.outputs.cell_count) > 1 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_1 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the second exact original - if: steps.reader_1.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '1' - AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Obtain exact third-cell reader - id: reader_2 - if: fromJSON(steps.session.outputs.cell_count) > 2 - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_2 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the third exact original - if: steps.reader_2.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '2' - AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - - name: Verify every selected original and its completeness receipt - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing report while preserving failed source grades - id: normalize - continue-on-error: true - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only exact reviewed synthetic fixture results - id: export - if: always() && steps.private_report.outcome == 'success' - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts - - name: Require a complete safe projection even after a failed normalization - if: always() - env: - EXPORT_OUTCOME: ${{ steps.export.outcome }} - run: test "$EXPORT_OUTCOME" = success - - name: Retain the metadata-only TestOps projection - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Record and independently verify each TestOps result - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: chat-completion-testops-projection-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-completion-testops-selection - - *allure - - name: Upload original outcomes, checks and private-evidence receipts - id: allure_upload - env: &upload_env - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results - run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops - - name: Verify actual selected results and complete result-owned attachments - id: verification - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *upload_env - run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops - - name: Retain closed TestOps verification receipts - if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-completion-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-completion-testops-verify-upload/ - ${{ runner.temp }}/chat-completion-testops-upload/exported.json - retention-days: 30 - if-no-files-found: error diff --git a/.github/workflows/chat-feature-testops.yml b/.github/workflows/chat-feature-testops.yml deleted file mode 100644 index 826a340ae91..00000000000 --- a/.github/workflows/chat-feature-testops.yml +++ /dev/null @@ -1,451 +0,0 @@ -name: Chat feature TestOps -run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }} - -on: - workflow_dispatch: - inputs: - operation: - type: choice - options: [evaluate, bootstrap, launch-plan] - default: evaluate - description: Fixed evaluation, zero-result registration, or one TestOps plan launch - ALLURE_JOB_RUN_ID: - type: string - required: false - description: TestOps supplies its job-run identity; no test/source selector is accepted - ALLURE_USERNAME: - type: string - required: false - description: Reserved TestOps service metadata; never used as authorization - -permissions: - contents: read - -concurrency: - group: chat-feature-testops - cancel-in-progress: false - -jobs: - authorize: - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - sha: ${{ steps.guard.outputs.sha }} - operation: ${{ steps.guard.outputs.operation }} - triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }} - runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci - steps: - - &checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ github.sha }} - persist-credentials: false - - &node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 - with: - node-version: 24 - - &guard - name: Verify fixed default-source dispatch and both actor identities - id: guard - env: - GH_TOKEN: ${{ github.token }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - NODE_OPTIONS: '' - NODE_PATH: '' - run: node tests/runner-e2e/testops-chat-dispatch.ts - - select: - name: Validate TestOps selection or register zero-result setup - needs: authorize - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - outputs: - artifact_id: ${{ steps.selection_upload.outputs.artifact-id }} - sha256: ${{ steps.selection.outputs.sha256 }} - steps: - - *checkout - - *node - - *guard - - &allure - if: needs.authorize.outputs.operation != 'launch-plan' - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 - with: - allurectl-version: 2.23.1 - allure-endpoint: https://paperclip.testops.cloud - allure-project-id: '1' - allure-token: ${{ secrets.ALLURE_TOKEN }} - - name: Register the public job without an evaluation or a synthetic result - if: needs.authorize.outputs.operation == 'bootstrap' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap - node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context - - name: Validate the one real TestOps selection before any build or provider access - id: selection - if: needs.authorize.outputs.operation == 'evaluate' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results - run: | - mkdir -m 700 "$ALLURE_RESULTS" - allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection - - name: Start exactly the validated TestOps plan and exit without waiting for its child - if: needs.authorize.outputs.operation == 'launch-plan' - env: - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }} - CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }} - CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }} - CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }} - CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }} - CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }} - CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }} - CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - GH_TOKEN: ${{ github.token }} - run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan - - name: Retain only closed metadata from zero-result setup - if: always() && needs.authorize.outputs.operation == 'bootstrap' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-setup-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json - ${{ runner.temp }}/chat-testops-bootstrap-state/once.json - ${{ runner.temp }}/chat-testops-bootstrap-state/plan.json - ${{ runner.temp }}/chat-testops-bootstrap-state/job.json - ${{ runner.temp }}/chat-testops-bootstrap-state/failure.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration.json - ${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain the one plan-launch receipt and stop state - if: always() && needs.authorize.outputs.operation == 'launch-plan' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-plan-launch-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-launch-plan-state/once.json - ${{ runner.temp }}/chat-testops-launch-plan-state/launch.json - ${{ runner.temp }}/chat-testops-launch-plan-state/failure.json - retention-days: 30 - if-no-files-found: error - - name: Retain only the token-free exact selection receipt - id: selection_upload - if: steps.selection.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-selection-${{ github.run_id }}-1 - path: testops-selection.json - retention-days: 30 - if-no-files-found: error - - build: - name: Build the immutable shipped native candidate without model credentials - needs: [authorize, select] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 45 - outputs: - artifact_id: ${{ steps.bundle.outputs.artifact-id }} - steps: - - *checkout - - *node - - *guard - - &pnpm - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 - env: - NPM_CONFIG_AUDIT: 'false' - NPM_CONFIG_FUND: 'false' - NPM_CONFIG_UPDATE_NOTIFIER: 'false' - with: - version: 9.15.4 - - name: Install only the source-reviewed graph without lifecycle scripts - run: | - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - pnpm install --frozen-lockfile --ignore-scripts - - name: Build the existing TypeScript and native runner owners - run: | - pnpm --filter @paperclipai/paperclip-runner build:typescript - pnpm --filter @paperclipai/paperclip-runner build:runner-binaries - - name: Package source-bound outputs and resolved lock - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$GITHUB_SHA" - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json - tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar - sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256 - - name: Retain credential-free immutable outputs - id: bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1 - path: | - chat-testops-build.tar.gz - chat-testops-build.tar.gz.sha256 - retention-days: 1 - if-no-files-found: error - - chat: - name: One real native OpenCode backlog request on AWS - needs: [authorize, select, build] - if: needs.authorize.outputs.operation == 'evaluate' - runs-on: ${{ needs.authorize.outputs.runner }} - timeout-minutes: 32 - environment: runner-e2e-private-paid - permissions: - contents: read - id-token: write - env: &retention - PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1 - TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }} - TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} - TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }} - TARGET_SHA: ${{ needs.authorize.outputs.sha }} - MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog - RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} - NODE_OPTIONS: '' - NODE_PATH: '' - AWS_EC2_METADATA_DISABLED: 'true' - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Check out the same admitted source for trusted retention - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - ref: ${{ needs.authorize.outputs.sha }} - path: .testops-trusted - persist-credentials: false - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact_id }} - path: chat-testops-build - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - name: Verify immutable outputs and token-free TestOps selection - env: - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - run: | - set -euo pipefail - (cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256) - tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE" - node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json - node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json - cmp chat-testops-lock.json chat-testops-worker-lock.json - cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml - pnpm install --frozen-lockfile --ignore-scripts - (cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store") - - name: Materialize and qualify pinned OpenCode runtime and browser - run: | - node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")' - google-chrome --version - pnpm exec playwright install ffmpeg - - name: Prepare private evidence with shipped-main authority - id: private - env: - TARGET_ROOT: ${{ github.workspace }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare - - name: Obtain only the exact private cell storage session - id: cell_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.private.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Verify strict private custody before the model key - env: &cell_session - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify - - name: Execute only the fixed original request with zero retries - env: - OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} - PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json - PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }} - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog - PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome - run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1 - - name: Retain exact original grade and private evidence, including failures - if: always() && steps.cell_credentials.outcome == 'success' - env: *cell_session - run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell - - report: - name: Retain private originals and export only reviewed fixture results - needs: [authorize, select, chat] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 15 - environment: runner-e2e-private-history - permissions: - contents: read - actions: read - id-token: write - env: *retention - steps: - - *checkout - - *node - - *guard - - *pnpm - - name: Install trusted reporting dependencies only - run: | - node tests/runner-e2e/prepare-report-lock.mjs - pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store" - - name: Prepare exact private report and cell-read policies - id: session - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session - - name: Obtain private report writer - id: report_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.session_policy }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Obtain exact private cell reader - id: read_credentials - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 - with: - role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} - aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - inline-session-policy: ${{ steps.session.outputs.read_policy_0 }} - output-env-credentials: false - output-credentials: true - disable-retry: true - action-timeout-s: 60 - - name: Download the exact private original and verify all inventories - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - SELECTED_CELL_INDEX: '0' - AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: | - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell - node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads - - name: Build the existing normalized campaign without exposing original logs - env: - PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e - PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized - PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]' - PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1 - PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }} - PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }} - run: | - mkdir -p runner-e2e-merged-report - pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1 - - name: Retain the complete merged report privately even when its grade failed - id: private_report - if: always() && steps.report_credentials.outcome == 'success' - env: - PRIVATE_REPORT_ROOT: ${{ github.workspace }} - PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report - AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }} - AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }} - AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }} - AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report - - name: Export only the bound normalized result, JUnit and marked PNGs - id: export - if: always() && steps.private_report.outcome == 'success' - env: - TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }} - run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts - - name: Upload only the sanitized TestOps projection - if: always() && steps.export.outcome == 'success' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops/ - retention-days: 30 - if-no-files-found: error - - upload: - name: Report the original outcome to its exact TestOps case - needs: [authorize, select, chat, report] - if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' - runs-on: ubuntu-latest - timeout-minutes: 10 - environment: runner-e2e-testops-reporting - permissions: - contents: read - actions: read - steps: - - *checkout - - *node - - *guard - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: runner-e2e-testops-${{ github.run_id }}-1 - path: runner-e2e-testops - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.select.outputs.artifact_id }} - path: chat-testops-selection - - *allure - - name: Verify and upload only the bound safe case outcome - id: allure_upload - env: &testops_reporter - ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} - ALLURE_ENDPOINT: https://paperclip.testops.cloud - ALLURE_PROJECT_ID: '1' - ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} - EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }} - ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results - run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops - - name: Verify the actual TestOps result after upload processing - id: verify_upload - if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure') - env: *testops_reporter - run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops - - name: Retain only closed actual TestOps verification metadata - if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure') - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: chat-testops-verification-${{ github.run_id }}-1 - path: | - ${{ runner.temp }}/chat-testops-verify-upload-state/verification.json - ${{ runner.temp }}/chat-testops-verify-upload-state/failure.json - retention-days: 30 - if-no-files-found: error diff --git a/DESIGN.md b/DESIGN.md index 068a7e4530b..9d53a3c844b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -117,8 +117,8 @@ completes without a provider reply. Preserve the reply through persistence. Task chat shows execution errors and waits only while they remain relevant. Exhausted attempts use a padded, rounded notice with a destructive-tinted background, -an error icon, and foreground text. The surrounding layout supplies separation from -the conversation. Automatic retry countdowns keep their quieter inline styling. +an error icon, and foreground text. A wrapper adds top padding above the tinted +surface. Automatic retry countdowns keep their quieter inline styling. Completing or cancelling a task hides its old execution notices. A newer attempt by the same agent or an explicit successor supersedes earlier run notices; an unresolved execution hold remains visible. Historical turns keep their responses, diff --git a/cli/vitest.config.ts b/cli/vitest.config.ts index f624398e8de..13b72103824 100644 --- a/cli/vitest.config.ts +++ b/cli/vitest.config.ts @@ -3,5 +3,7 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", + hookTimeout: 30_000, + teardownTimeout: 30_000, }, }); diff --git a/doc/DATABASE.md b/doc/DATABASE.md index e617f321dcc..04698b00f19 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -522,23 +522,59 @@ leave database construction unchanged. An expired window stays inactive after a restart; remove the setting after the investigation. The diagnostic measures awaited Postgres.js `unsafe` responses before Drizzle -column decoding. It retains the Drizzle builder caller for normal, prepared, -transaction, and dedicated-pool execution. It issues no queries. -It logs the public server caller locations, static schema table, operation, -query/row counts, and response value sizes. It emits at most 2,048 aggregate -groups and a summary. Measurement stops at the deadline. Environment-enabled +column decoding. It issues no queries. Configuration is read when each database +client is created; setting the environment on an already-running process does +not install the capture on existing clients. Prepare the deployment artifact +before choosing the UTC window, then start the window after rollout and warm-up. + +Server entry points use `withDatabaseOperation(owner, action)` to name the work +that executes queries. Owners are fixed semantic labels, never request paths, +company or run IDs, user input, or SQL. The execution owner is captured for each +query and retained through completion, including transactions and dedicated +pools. A prepared query reused by different operations keeps the owner of each +execution. The builder caller is separate from that execution owner. +Owner context is active while a valid future window is armed, up to 30 minutes +before its start, so work that crosses the start boundary keeps its owner. Query +accounting starts only at the window start. With no armed window, or after +expiry, the owner helper calls the action directly without a context scope. + +The capture aggregates owner, static schema table, operation, calls, returned +rows, query errors, and total/maximum completion wall time. `totalWallMs` and +`maxWallMs` include pool, database, network, and event-loop waits; they are not +CPU measurements, and concurrent query durations overlap. Timing excludes the +capture's response sizing. Rank calls and returned rows both per second and per +completed service operation; a busy owner is not necessarily wasteful. + +Response sizing samples successful results 1, 17, 33, and so on per aggregate +group (`sampleEvery: 16`). It examines at most +256 cells, 65,536 string code units, and 65,536 value bytes per sampled result; +an individual string over 16,384 code units is not measured. Parsed objects and +other complex values are not serialized. Oversized, complex, or unexamined +values leave an incomplete sample. `sampledValueBytes` is a lower bound over +sampled primitive values, not a total response-size estimate or network bytes. +Do not extrapolate it as the total bytes returned by the database. Inspect +`sampledResults`, `sampledRows`, `measuredValues`, `unmeasuredValues`, and +`incompleteSamples` alongside it. `sizingWallMs` records the diagnostic's sizing +work separately. Use a separate capture-disabled CPU profile to measure normal +processing cost. + +The diagnostic emits at most 2,048 aggregate groups and a summary. Measurement +stops at the deadline. Environment-enabled captures emit at most 64 records per second and can take another 33 seconds to finish writing the bounded output. Wait for the summary before collecting it. It never logs SQL, parameters, result values, connection strings, or query errors. In-flight queries continue normally. -For a bundled server, keep the linked source maps and enable Node's -`--enable-source-maps` option so the diagnostic can resolve public caller -locations. Restore the prior Node options after capture. Sizes still count when -caller locations are unavailable. +Semantic owners work in a bundled server without source maps. For queries +without an explicit owner, keep the linked source maps and enable Node's +`--enable-source-maps` option to resolve public caller locations. Restore prior +Node options after capture. Known semantic owners avoid repeated stack capture. +Queries without an available owner/caller still contribute to the totals. -Look for `paperclip.db_response_capture.entry.v1` and -`paperclip.db_response_capture.summary.v1` in the process logs. A log provider +Look for `paperclip.db_response_capture.entry.v2` and +`paperclip.db_response_capture.summary.v2` in the process logs. Version 2 uses +sampled, bounded primitive-value sizing; do not interpret it as the version 1 +all-result byte estimate. A log provider can store `event` as a structured field instead of message text. Filter by that field or read the bounded window from the exact deployment. Check the summary and group totals, including the unattributed overflow group. @@ -552,22 +588,26 @@ Fetch a complete capture as follows: from zero through `groupPages - 1`, filtering on the entry event, `captureId`, and `groupPage`. Request at least `groupPageSize` matching records per page. A structured query can use - `@event:paperclip.db_response_capture.entry.v1 AND @captureId: AND @groupPage:0`. + `@event:paperclip.db_response_capture.entry.v2 AND @captureId: AND @groupPage:0`. 3. If the provider's record limit is smaller than a page, fetch each `groupIndex` separately with the same capture ID. Each index matches one entry. Do not split by timestamp: entries can share a timestamp. 4. Deduplicate by `(captureId, groupIndex)`. Require every index from zero through `groups - 1`, with no gaps. Entry count must equal `groups`, summed - `calls` must equal `completedQueries`, and summed `responseValueBytes` must - equal the summary total. A missing page is an incomplete measurement. + `calls` must equal `completedQueries`, and summed `rows`, `sampledResults`, + and `sampledValueBytes` must equal their summary totals. A missing page is an + incomplete measurement. Keep overflow and unattributed groups visible. Tagged raw-driver calls, streams, unawaited raw-driver dispatch, and protocol overhead are not measured. Relational queries use a driver-level caller fallback. -Queries still in flight at the deadline are reported separately. Sizes describe -driver values before column decoding, not billed network bytes. Complex parsed -values use JSON size estimates, counted in `estimatedValues`. Reconcile them with -provider metering and socket measurements. Serialization adds temporary CPU and memory work; -measure the final transfer rate after the window expires. +Queries still in flight at the deadline are reported separately. The census does +not include every PostgreSQL statement or internal transaction command, so its +calls are not interchangeable with database-wide commit counters. It does not +measure SQL compilation, driver parsing CPU, or Drizzle row-mapping CPU. Use the +ranked owners to choose a narrow follow-up profile or synchronous-stage +measurement; do not attribute CPU by subtracting process CPU usage across an +asynchronous query. Remove the capture setting after collecting its final +summary; the absolute deadline already prevents further measurement. ### Push the schema diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index c0827418fe7..106e8412cf1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1058,8 +1058,55 @@ For normal issue work, start with the smallest targeted check that proves the ch #### Embedded Postgres template cache +To run the suite with one owned PostgreSQL server and a separate disposable +database for each fixture, use: + +```sh +node cli/node_modules/tsx/dist/cli.mjs packages/db/src/test-postgres-command.ts +node cli/node_modules/tsx/dist/cli.mjs packages/db/src/test-postgres-command.ts -- pnpm exec vitest run --project @paperclipai/db +``` + +The command starts an embedded server, prepares a migrated database template, +passes it to the child command, and waits for server shutdown before returning. +It preserves the child command's exit status. On cancellation it stops and waits +for the test process group before shutting down PostgreSQL. Database, server, +and CLI fixtures all use the common implementation. Tests of embedded startup itself continue to +start dedicated clusters. + +To reuse a disposable server that you already manage, set +`PAPERCLIP_TEST_POSTGRES_ADMIN_URL` to its administrative PostgreSQL URL. The role +must be able to create databases. Each fixture creates a random database and +closes only its clients before dropping it. It never stops the external server. +`DATABASE_URL` does not select this test path. The optional +`PAPERCLIP_TEST_POSTGRES_TEMPLATE` can name an existing migrated database with +the form `paperclip_test_template_`; `0` disables template reuse. + +On a supported platform, PostgreSQL startup failures fail test collection. +Resource exhaustion, unavailable configured servers, and broken native binaries +do not make integration tests skip. A genuinely unsupported platform or CPU can +still skip embedded PostgreSQL coverage. Startup retries apply only to port +collisions. Teardown waits for process exit, requests fast shutdown first, and +uses immediate PostgreSQL shutdown if needed. A failed stop fails the hook and +retains its data directory. Worker termination also requests owned server +shutdown. A forced `SIGKILL` can bypass these handlers; the separate host cleanup +job remains the backstop for stale macOS semaphore sets. +Database and CLI hooks use the same 30-second deadline as server hooks. This +allows the ten-second shutdown protocol and client/database cleanup to finish +before the test runner ends a worker. A shutdown timeout still fails the hook. +Template publication uses `pg_ctl` to request clean fast shutdown on every +supported platform and allows thirty seconds for it. A template is never +published after immediate shutdown or a crash. Cancelling the shared-server +command also waits for its owned cluster-directory cleanup after the test +processes and PostgreSQL exit. Direct worker termination stops PostgreSQL but +retains its disposable files for inspection; the synchronous exit guard cannot +await directory removal. +Stopped cluster directories are removed asynchronously, and cleanup awaits their +removal. Large suites can leave many relation files; deleting them must not block +the worker's event loop. The idle-sleep suite uses the existing 90-second fixture +budget for its final directory cleanup after hundreds of table truncations. + Suites that call `startEmbeddedPostgresTestDatabase` (from `@paperclipai/db`) -get their own embedded Postgres cluster. The first such suite on a host builds +without an administrative URL get their own embedded Postgres cluster. The first such suite on a host builds one fully migrated cluster and publishes its data directory under `$TMPDIR/paperclip-embedded-postgres-templates-/`. Every later suite copies that template and starts Postgres on the copy instead of running @@ -1669,6 +1716,10 @@ marker is gone. Denied permissions and object locks keep a retry obligation. The sandbox holds current working files. PostgreSQL records its location, writer epoch, and committed snapshot; encrypted immutable snapshots live in S3. +S3 multipart uploads stream up to four parts concurrently. One producer reads +the source in order, and bounded stream buffers preserve backpressure. Failed +uploads cancel and settle their active requests before multipart cleanup. +Snapshot publication still requires full object hash and size verification. Preparation creates a saved baseline, and native finalization saves the stopped writer's files before releasing its persistence obligation. Replacing a webserver reconnects to the recorded allocation without copying files back to @@ -1677,6 +1728,15 @@ host, sandbox, or Git remote. Files changed since the last save still depend on the live sandbox: losing that allocation can lose those edits. There is no periodic workspace backup in this release. +Files reads use the current committed saved version when remote authority is +clean and has no active writer, even while its old allocation remains recorded. +The saved loader verifies the company, workspace, snapshot, and archive before +reading. Dirty or writer-owned authority uses the live allocation and cannot +substitute an older save if that allocation is unavailable. Reads retain their +authority-change fence and remove private saved-file copies after use. +Embedders that omit the optional saved-file loader keep valid live reads. They +still report unavailability when neither a live location nor a loader exists. + Existing local task workspaces require an explicit import. Run from a source checkout with the deployment's database, S3, and key configuration loaded: @@ -2621,6 +2681,16 @@ connections, including provider-native tools. Company permissions, approval gates, and workspace isolation still apply. Explicit restrictive modes remain restrictive; omitted settings use full auto. +Assigned app tools can use `paperclip_search_assigned_tools` and +`paperclip_call_assigned_tool` instead of direct tool definitions. Search with +`{"query":"github"}` and call an exact returned tool name. Omitted, empty or +whitespace-only `schemaTool` values perform normal discovery, including when +`schemaOffset` is zero. To read a large schema, set `schemaTool` to the exact +`inputSchemaRef` from discovery and follow `nextSchemaOffset`. A positive +`schemaOffset` requires a nonblank reference. Unknown or revoked references +return guidance to search again; discovery still checks current grants and work +modes against the session's pinned catalog. + Test questions, saved plan revisions, approval before task handoff, status lookups, and `/new` preserving chat history. Hiring additionally requires the operator-controlled [runner API tools](runner-api-tools.md) rollout; enabling diff --git a/doc/PRODUCT.md b/doc/PRODUCT.md index cba3bec4ae2..62390a783de 100644 --- a/doc/PRODUCT.md +++ b/doc/PRODUCT.md @@ -97,6 +97,11 @@ When an agent creates a subtask or follow-up task, its source task's feed shows a linked card at creation time with the current title, task number, and status. The Tasks tab keeps the full relationship list. +Tasks waiting between automatic checks show **Monitoring** with a blue clock. +Task lists, board columns, filters, search, recent navigation and task properties +use the same read-only display state. The task retains its execution lifecycle +status, and the monitor banner shows when the next check is due. + Automatic monitor checks pause when a task is blocked. A person can use Check now to request one check of a retained monitor through normal execution admission. Dependencies, recovery, pause, budgets and ownership still apply. Provider-usage @@ -141,9 +146,15 @@ choice stays selected while task data refreshes. The task Filters menu also offers **Routine tasks**. It selects tasks with `originKind=routine_execution`, including manually triggered routine runs, even when other automatic tasks are hidden. The routines list's **View all tasks** -link opens `/issues?view=all&originKind=routine_execution` with that checkbox +link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox selected. The URL retains the selection across reloads and task navigation. -Search and pagination apply the routine origin filter on the server. +Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides +saved collection sorting, and the routines link opens newest-created tasks first. +Shared sort links work across Coordinator, Streamlined UI, and Classic UI, including +sort fields without a visible column in the selected interface. Search relevance +and board-column membership keep their existing server order. +Search and pagination apply the routine origin filter on the server. Creation-time +sorting also runs before pagination, with task ID as a stable tie-breaker. Tasks and project lists hide agent-created and system-created tasks by default. The Filters menu's **Show agent-created tasks** checkbox includes routine runs, diff --git a/doc/SPEC-implementation.md b/doc/SPEC-implementation.md index 59874d54e8b..810faf75d9d 100644 --- a/doc/SPEC-implementation.md +++ b/doc/SPEC-implementation.md @@ -423,6 +423,7 @@ retain their titles and default to no generation request. - `title` text not null - `description` text null - `status` enum: `backlog | todo | in_progress | in_review | done | blocked | cancelled` +- Task reads additionally expose read-only `displayStatus`: the lifecycle status, or `monitoring` for an agent-owned scheduled ordinary monitor wait with no active execution, locks, review participant or active recovery action. Board collection/search `displayStatus` filters and counts use the same projection before pagination. This field is not persisted and cannot authorize a status transition; existing `status` filters retain their lifecycle semantics. - `priority` enum: `critical | high | medium | low` - `review_policy` nullable enum: `anyone | not_creator | human_only`; null is equivalent to `anyone` - `assignee_agent_id` uuid fk `agents.id` null @@ -2712,9 +2713,16 @@ its upstream authors; operators can pause updates to review changes manually. The task Filters menu also offers **Routine tasks**. It selects tasks with `originKind=routine_execution`, including manually triggered routine runs, even when other automatic tasks are hidden. The routines list's **View all tasks** -link opens `/issues?view=all&originKind=routine_execution` with that checkbox +link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox selected. The URL retains the selection across reloads and task navigation. -Search and pagination apply the routine origin filter on the server. +Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides +saved collection sorting, and the routines link opens newest-created tasks first. +Shared sort links retain their order across task interfaces. Workflow sorting +requests the existing authorized dependency-edge projection. Search and board +queries keep their existing server order. Blocked-attention lists reject +`sortField=created` with `422` because they use their own activity order. +Search and pagination apply the routine origin filter on the server. Creation-time +sorting also runs before pagination, with task ID as a stable tie-breaker. Tasks and project collections default to hiding agent/system-created tasks and provide a saved **Show agent-created tasks** filter. Classify creation using diff --git a/doc/connections/AI-CONNECTIONS.md b/doc/connections/AI-CONNECTIONS.md index 1bbb8ad26fb..2a47847f46d 100644 --- a/doc/connections/AI-CONNECTIONS.md +++ b/doc/connections/AI-CONNECTIONS.md @@ -98,6 +98,23 @@ grant through encrypted vault secret references, with existing consumer bindings Safe provider-reported account identity is optional; secret references, tokens, and authentication paths are never account labels. +Permissions shows account details separately from the connection's maintenance +controls: the provider account email and organization when available, the sign-in +method, and personal or company-shared access. A Paperclip owner name is not +evidence of the provider login. Anthropic setup tokens can permit inference while +denying profile access; in that case the screen explains that Anthropic did not +permit reading the account email. A profile denial does not mark a connected +credential as disconnected. + +`GET /api/companies/:companyId/ai-connections/:connectionId/identity?grantId=...` +checks company membership, connection-management authority, and the selected +credential's human audience before opening the credential. The response contains +only safe account labels and bounded status messages. The Anthropic profile read +has a 15-second headers/body deadline and no redirects. It runs separately from +the account/grants reads, so profile loading or failure cannot block account +maintenance. The UI reuses an observation for five minutes, scoped to the company, +connection, grant, and grant update time. Reconnect requires a new observation. + Company-scoped `/api/companies/:companyId/ai-connections` operations provide list, API-key creation/reconnect, personal defaults, completed login references, and active-run attribution. The list includes `canManageConnections`, evaluated by diff --git a/doc/evals.md b/doc/evals.md index 4071cf01866..d0ea2fa5ec3 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -57,8 +57,9 @@ OpenCode Product E2E cells. They verify saving an unstarted task and delegating a saved announcement back to chat, using ordinary production instructions, real browser messages and independent state/provenance checks. A separately selected native OpenCode/DeepSeek1017 -backlog cell measures TestOps integration acceptance with the same oracle and -original bounds. Its source registration is not an AWS or TestOps pass. The +backlog cell supplied historical TestOps integration acceptance with the same +oracle and original bounds. Its former smoke plan and the three-case completion +pilot are retired; their cases and measurements remain in history. The [full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops) maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and fixtures use a preserved immutable catalog revision; trusted workflow helpers run diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index dd0378688f5..13f907f61f0 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -112,6 +112,22 @@ The issue is actively owned work. For agent-owned issues, `in_progress` should not be allowed to become a silent dead state. +### Monitoring presentation + +An agent-owned `in_progress` task displays **Monitoring** with a blue clock while +it has a persisted ordinary monitor deadline and no active run, execution or +checkout lock, review participant, or active recovery action. The persisted +status remains `in_progress`. Monitor admission, checkout, finalization, recovery, +and agent lifecycle filters keep their existing contracts. Review, blocked, +terminal, and provider-quota recovery states keep their own presentation. + +Task reads expose optional, read-only `displayStatus`. Board task collections and +search can filter by `displayStatus`; filtering happens before pagination and +counts. Existing `status` filters retain their lifecycle meaning. Monitoring is +not a manually selectable task status or a board drag target. An overdue check +retains its visible deadline warning; the label does not attest that a provider +is running or that a check succeeded. + ### `blocked` The issue cannot proceed until something external changes. @@ -186,6 +202,14 @@ and review run. The server reads these records from saved review state; it does not depend on the parent session remembering a separate review session. These records are evidence and do not grant permission to resolve another review. +An automatic chat-completion wake retains its exact delivery receipt when +admission waits on a dependency, budget, or pause. It does not merge into a +replaceable dependency signal. The same gates still prevent execution. +Workspace readiness accepts a completed operation enclosed by a successful +finalize from the same company, workspace, issue, and run. A nested snapshot +that finishes before that finalize finishes does not leave a permanent wait. +Unfinished operations and work after the finalize still require a later save. + ### Answered Slack conversations An accepted scheduled-monitor yield remains unfinished task work. Its committed @@ -710,6 +734,20 @@ A workspace-coherent adapter path means: - the effective cwd exists or is provider-reachable, according to the workspace provider - when the adapter or workspace strategy relies on git state, the cwd is git-valid for the selected workspace: it resolves to the expected repository root, required base refs or branch metadata can be resolved, and runtime-created worktrees are still registered or explicitly recoverable +Fresh remote-workspace recovery fingerprints the admitted remote strategy. A +temporary strategy used to seed a controller directory must not become the saved +remote configuration. An unchanged follow-up reuses that configuration without +reporting strategy drift. Real strategy, source, mode and environment changes +retain their existing replacement checks. + +Older remote Git workspaces can retain the temporary `project_primary` +fingerprint. That fingerprint alone cannot distinguish the old preparation bug +from a genuinely configured `project_primary` strategy; the remote workspace row +uses `git_worktree` for either Git source. Do not rewrite those fingerprints in +place or suppress their replacement checks. After saving and inspecting the +retained work, an explicitly admitted fresh recovery creates a workspace with the +correct fingerprint. Ordinary reuse of the old workspace does not migrate it. + Adapter-backed liveness also requires control-plane reachability from the agent's actual mutation surface, not just from the host adapter process. If the agent is expected to use Bash, shell tools, runtime helpers, or in-sandbox command execution to update issues, create comments, upload artifacts, or submit review decisions, the `PAPERCLIP_API_URL` and `PAPERCLIP_API_KEY` visible to that surface must route to Paperclip successfully. For sandbox-backed local adapters, Paperclip may satisfy that contract with a run-scoped in-sandbox bridge. The host adapter keeps the real run JWT on the host side, injects only the bridge URL/token into the sandbox tool environment, and forwards allowed Paperclip API requests with the run id attached. The bridge credentials are execution plumbing, not user-facing context: they must not be written into prompts, visible comments, issue documents, restored workspace files, or durable logs. Agents and skills must use the env vars available in Bash/curl rather than assuming that the host's localhost API URL is reachable from browser or web-extraction tools inside the sandbox. @@ -764,6 +802,15 @@ conflict detection. Legacy adapter snapshots remain inline. Snapshot-only worker do not load the adapter process-launch helpers. The directory merge loads those helpers when it resolves its instance-scoped lock root. +Portable remote workspace capture stages only self-contained Git metadata. It +streams working files into the archive without a second working-tree copy and +does not repack an already dissociated mirror. Each streamed file must match its +recorded hash, size and mode and retain its filesystem identity while read. +Capture rechecks the complete source inventory and Git state before returning a +receipt. A changed source or cancelled capture removes its incomplete archive; +an existing archive remains untouched. Temporary space must still hold the +normalized Git metadata and the complete archive together. + ### Remote Codex model compatibility Fresh remote Codex Runner runs check the selected image CLI before saving the @@ -862,6 +909,15 @@ The task thread exposes the guarded Retry action for failed or timed-out convers A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed. +A successful connected run can still have newer admitted input waiting behind it. +Task Retry resumes that input's existing durable chat receipt through the inbound +owner, instead of replaying the successful run as a failed attempt. The original +actor, source, result, and publication history remain intact. Duplicate clicks +follow one canonical receipt. Current board permission, task ownership, source +authorization, and prior execution cleanup are rechecked before normal admission; +pause, budget, approval, and dependency gates remain effective. Completion or +newer execution supersedes an unadmitted retry. + A valid recovery action must name: - the source issue and company @@ -1682,6 +1738,20 @@ sources without this proof remain ineligible. The preservation owner still requires an independently stopped, exclusively bound allocation and no host writer; it saves files and resumes the existing finalizer without another provider turn. +Both the legacy native copyback descriptor and the remote-workspace descriptor +keep an unfinished export under the execution hold. Task Retry saves its intent +behind that hold; an accepted successful result resumes its existing finalizer +after verified preservation instead of admitting another task run. Pending human +review remains effective when the result commits. + +If a later failure supersedes the original recovery action, an explicit board +`recover-workspace` request can still preserve that exact historical export. +The cancelled action remains historical and the current action retains attention +ownership. Preservation checks current writers and source identity, records a +durable snapshot, and leaves the task's newer workspace binding unchanged. It +does not apply an older accepted result over a newer run or authorize continuation. +Cancelled history without a saved explicit recovery request remains ineligible. + Startup and the retained-workspace sweep reconstruct these obligations through the retained workspace index. Committed Retry, recovery admission, and recovery progress writes wake a separate preservation worker. It follows the persisted @@ -1926,6 +1996,24 @@ That is the execution contract Paperclip should present to operators. ### Cancellation during native startup +Core supports verified cleanup for explicitly recorded startup obligations. This +support does not create startup contexts, advertise transport support, or capture +a transport binding. Existing callers retain their legacy payloads and startup +budgets. This support alone makes no startup-latency claim. + +Restart recovery uses the claim breadcrumb only while the current run has no +first session checkpoint and the prior controller and context deadlines have +expired. A later preparation breadcrumb requires the stronger preparation-stop +proof. A failed cleanup-intent write retains the original typed failure and +rearms the existing restart retry after 30 seconds; the running run continues to +block idle sleep. Recovery queues cleanup rather than replaying a provider. A completed result +or a newer owner wins. The existing adopted-authentication and chat-workspace +ownership holds retain their separate behavior. The startup budget does not bound +the duration of remote cleanup. Existing Stop receipts and process cleanup still gate +destructive lifecycle or binding changes. Operator plugin restarts and external +machine changes can still interrupt startup. + + Cancellation records a preparation fence while holding the run row lock. Native runtime selection checks that fence, the running status, and the current startup controller lease in the same transaction that creates the native coordinator. @@ -1964,6 +2052,15 @@ preserving history, unknown action outcomes, and attempt counts. Pauses, approva budgets, task ownership, and terminal task status still gate admission. No automatic provider replay is authorized by a cancelled startup. +A failed cleanup attempt saves its next retry time in the existing control-work +deadlines. The startup marker retains the original admission deadline. Both +candidate batches use stable deadline order, and pending cleanup cannot enter +ordinary forced finalization. A late replacement-task reply remains owned until +its resource identity is durable. It is queued for cleanup instead of attached +after cancellation. The old task's Stop receipt cannot settle cleanup of a new +replacement task. A lost reply with no known resource identity remains unresolved +until provider inspection or operator reconciliation establishes its outcome. + ### Delivering queued messages after a legacy run stops The legacy queued-message Interrupt action accepts a null `targetRunId` when @@ -2446,6 +2543,13 @@ accessible repository work products do not require an attachment. Publication failure calls for continued work or a concrete blocker, not a human confirmation that the task is complete. +Native completion reports can cite the relative task-document links returned by +the server. Each link must resolve to this company's current task and an attached +document whose current revision has a matching native publication receipt. +The same server-owned link projection supplies completion guidance and evidence +validation. A published document link can accompany a requested file, but it +does not replace that file's verified delivery evidence. + For an explicit file output in the current request, an empty report, a verification-only reference, or an unregistered URL cannot satisfy delivery. The report must cite an attachment verified by the current run's durable diff --git a/doc/idle-sleep-safety.md b/doc/idle-sleep-safety.md index 81e5c6c262f..73af0c52bbf 100644 --- a/doc/idle-sleep-safety.md +++ b/doc/idle-sleep-safety.md @@ -148,6 +148,15 @@ agent/company/user, waiting state, no checkout, active run, accounting, monitor or pending delivery, and a finished wake for every accepted user-comment outbox entry. An idle presentation state does not exempt an ordinary runnable task. +Sandbox creation records in `pending` or `needs_reconciliation` remain blockers, +including manual uncertainty after company deletion. The pending-only catalog +omission predates the new startup-cleanup recovery path; no complete-instance +unsafe sleep outcome has been demonstrated. A late identified resource transfers +its obligation to the exact cleanup lease, and only the owning successful cleanup +can resolve that creation. Settled startup cancellations without a committed +native result manifest remain conservatively blocked by native finalization; +their cleanup marker alone is not a history exemption. + Workspace runtime inspection also reads the owner's process-local resource state. Startup reconstruction must succeed before that probe becomes quiet. Adopted/live services, provisioning/control/replacement work, live allocation @@ -224,6 +233,17 @@ fingerprint and RPC company scope for later read-only verification. Retained allocations, unfinished runs/accounting/retries, export intents and live setup credentials continue to block sleep. +For a reused remote allocation, a completed predecessor can resolve termination +through the final owner's destruction receipt only when the full host-recorded +lease handoff chain matches. Company, agent, task, workspace, environment, provider +and allocation identities must agree. A stopped successor, broken handoff or +unrelated owner grants no cleanup authority. This resolution does not rewrite +historical leases, execute remotely or claim that uncollected files were saved. +The normal agent-file recovery worker retains the no-save outcome and completes +cleanup after confirmed destruction. A completed cleanup receipt with a stale +retry deadline is reconciled once through the existing deadline worker. Clearing +that deadline compares the receipt so a concurrent cleanup obligation is kept. + Pending human-only connection setup can wait asleep when no active enabled connection of the requested purpose exists. Available access reblocks sleep so the existing reconciliation path can run; answering a setup card continues to diff --git a/doc/mcp-discovery-performance.md b/doc/mcp-discovery-performance.md index 219079cf3e5..ac696b01187 100644 --- a/doc/mcp-discovery-performance.md +++ b/doc/mcp-discovery-performance.md @@ -25,6 +25,41 @@ The stateless gateway and runtime-tools MCP endpoints return HTTP 405 with The runtime-tools endpoint still validates its caller before that response. See the [MCP transport contract](https://modelcontextprotocol.io/specification/2025-03-26/basic/transports). +## New-tool review projection + +Pending-tool review builds covered application and connection sets from each +profile's include rules. An included catalog entry covers its connection and its +non-null application only while that entry exists in the supplied catalog. +Explicit application selectors retain null application matching. A second index +records all exact selectors, including exclusions, so already reviewed tools do +not appear as new. The catalog filter retains its order, status check, strict +review watermark, and returned metadata. + +The indexes live only for one projection. Profile listings, new-tool review, and +app attention still read current company-scoped database state on each request. +This replaces nested catalog-by-rule scans with linear indexing and filtering. +It changes neither database query counts nor response contracts. + +The PostgreSQL reproduction uses 200 selected profiles, 1,900 active tools, and +76,000 rules. Profiles alternate between 65 and 695 rules. Half reference 695 +removed catalog entries, while half select current tools. This exercises both +scope checks and explicit-review checks. It compares complete ordered results +from the real profile listing and app-attention services before timing them. + +Save baseline `profiles/projection.ts`, `profiles/service.ts`, and +`catalog/attention.ts` beside their current versions as `.baseline-projection.ts`, +`.baseline-service.ts`, and `.baseline-attention.ts`. Change the two baseline +consumers' projection imports to `.baseline-projection.js` with the same relative +path. Then run from `server/`: + +```sh +node --expose-gc --import tsx scripts/benchmark-profile-review-projection.ts --baseline-profiles "$PWD/src/services/tool-access/profiles/.baseline-service.ts" --baseline-attention "$PWD/src/services/tool-access/catalog/.baseline-attention.ts" +``` + +Remove the three baseline files after the comparison. The probe alternates five +samples per implementation and reports process CPU, elapsed time, actual query +count, and returned rows. No provider calls or hosted traffic are generated. + ## Reproduce the measurements Build workspace dependencies, then run from `server/`: @@ -71,3 +106,35 @@ Catalog payload memory and policy-matching CPU still grow with the catalog. The measurements do not establish a safe production heap limit or the share of active installations affected. Verify restart bursts and sustained traffic on the actual deployment before reducing its temporary heap allowance. + +## Profile selector matching + +Effective profile resolution and profile summaries index each profile's include +and exclude selectors once per decision. Catalog rows then use exact set +membership checks for application, connection, catalog entry, tool name, and +risk level. Exclusions still take precedence within a profile; effective profiles +still contribute the union of their allowed tools. Null application selectors +retain their exact-match behavior. + +These indexes live only for the current computation. Each resolver call still +reads current company-scoped bindings, profiles, and rules. Query counts and +returned database rows are unchanged. This reduces JavaScript matching work when +a profile selects many individual tools; it does not cache authorization or +remove the runtime assignment digest check. + +For a real PostgreSQL comparison, save `projection.ts` and `resolution.ts` from +the baseline commit beside their current versions as `.baseline-projection.ts` +and `.baseline-resolution.ts`. Change only the baseline resolver's projection +import to `./.baseline-projection.js`. Then run from `server/`: + +```sh +node --import tsx scripts/benchmark-tool-profile-matching.ts --baseline "$PWD/src/services/tool-access/profiles/.baseline-resolution.ts" +``` + +Remove the two baseline files afterward. The probe uses 1,890 catalog tools, +1,575 individual include/exclude rules, one effective profile, and 12 agents. It +checks full assignment-result equality, warms both versions, and alternates +seven samples of ten resolutions. It reports actual query and returned-row +counts, process CPU, and elapsed time. The installation dependency returns an +empty list for this fixture, which has no installed connections. It does not +measure provider work, whole-run latency, or hosted performance. diff --git a/doc/observability.md b/doc/observability.md index a96ae96058b..abadb0e6538 100644 --- a/doc/observability.md +++ b/doc/observability.md @@ -1900,6 +1900,49 @@ name to `sandbox.daytona.`. The host maps any other value to daytona provider emits these spans today, so the segment is the literal `daytona`. +### Daytona lease acquisition and resume + +The worker invocation carries nested, monotonic timing receipts for lease work. +The host also admits the following fixed `lease.*` names and prefixes them with +`sandbox.daytona.`. These use the existing opt-in endpoint and host-owned parent +context; they do not create an exporter or first-party Telemetry events. + +```text +execution.plugin_dispatch +└── plugin.worker + ├── sandbox.daytona.lease.wait_for_teardown (only when teardown blocks resume) + ├── sandbox.daytona.lease.lookup + ├── sandbox.daytona.lease.refresh (missing-container confirmation) + ├── sandbox.daytona.lease.channel_cleanup (before restart) + ├── sandbox.daytona.lease.start (or lease.recover) + └── sandbox.daytona.lease.verify + ├── sandbox.daytona.lease.workspace + ├── sandbox.daytona.lease.sentinel + └── sandbox.daytona.lease.shell +``` + +`lease.lookup` includes cached-handle lookup and any required provider refresh. +`lease.start` and `lease.recover` measure the actual SDK operation, including its +internal readiness wait. They also cover starts triggered by execution or sync. +They do not separate SDK-internal requests or provider-side queue time. +Their `paperclip.sandbox.startup.state` attribute records the state before the +call: `started`, `stopped`, `archived`, `error`, or `other`. Both the plugin and +host restrict this value; no provider error text or resource identity is sent. + +Fresh acquisition emits `lease.create`, `lease.workspace`, `lease.shell`, +`lease.expiry`, and `lease.sentinel`. `lease.create` covers the existing creation +helper, including qualification-only snapshot resolution and creation-failure +cleanup. Setup-failure deletion is a separate `lease.cleanup` sibling, so it +does not inherit a failed workspace or verification phase. Existing deadlines, +ownership, cleanup and retry behavior are unchanged. Failed awaited steps close +with error status; the receipt contains no exception message. + +Children nest beneath the current `plugin.worker` receipt and verification +children nest beneath `lease.verify`. Never add parent and child durations. +Receipt budgets and clock validation still apply. Older workers without +`tracer.withSpan` keep the original behavior without these detail spans. Tracing +failures preserve the operation's result or error and never repeat it. + The `sandbox.daytona.*` spans use this closed attribute allowlist. The host drops every other key, so a command, an argument, a path, an id, a standard output, or a standard error never rides a provider span. The host records only @@ -1908,6 +1951,7 @@ the attributes that the producer sends for one span. | Attribute | Type | Optional | Meaning | | ------------------------------------------------ | ------ | -------- | --------------------------------------------------------------------------------------------------------- | | `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. | +| `paperclip.sandbox.startup.state` | string | yes | State before start/recovery: `started`, `stopped`, `archived`, `error`, or `other`. | | `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). | | `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.daytona.pack` span. | | `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.daytona.transfer` span. | diff --git a/doc/performance/native-backup-publication.md b/doc/performance/native-backup-publication.md new file mode 100644 index 00000000000..b461ea49d76 --- /dev/null +++ b/doc/performance/native-backup-publication.md @@ -0,0 +1,53 @@ +# Native backup publication benchmark + +Retiring a native harness backup can involve many immutable directories and +small files. Cleanup repairs directory permissions before removing the retired +tree. Publication waits for cleanup, preserves the previous backup when lease +stamping fails, and emits its completion notification after cleanup succeeds. +Cleanup uses asynchronous filesystem operations so unrelated requests can +continue while it runs. Directory preparation is serial and does not follow +symlinks into live trees. + +Run the focused correctness tests from an installed checkout: + +```sh +pnpm --filter @paperclipai/server exec vitest run src/services/native-runtime/native-harness-backup-cleanup.test.ts src/services/native-runtime/native-harness-backup-publication.test.ts +``` + +Run the performance comparison from the repository root with the Node version +used by the target server. Use a quiet machine and an explicit baseline commit: + +```sh +node server/scripts/benchmark-native-backup-publication.mjs \ + --baseline-ref 774ddc72713d6aa7cb516691c71f1fc23bcfa5ff \ + --output /tmp/native-backup-publication.json +``` + +The script compiles the real publication service and its dependencies for both +the baseline commit and the working tree. Each run uses a separate server +process. Fixture preparation occurs before measurement. The default fixture has +100 immutable directories with 100 small files in each of the `current` and +`previous` backups: 20,000 files to remove. Both backups also have a symlink to a +read-only live directory outside the retired trees. The new pending backup has +a distinct manifest. Every completed run checks the new manifest, complete +removal of retired and pending trees, and unchanged live contents and mode. + +A separate process sends requests to a small loopback HTTP endpoint every five +milliseconds. Arrivals use a fixed clock and include overdue requests. The +report includes request completion and error counts, generator lateness, p95 +and maximum foreground latency, requests served during publication, publication +wall time, process CPU time, and maximum event-loop delay. Three pairs alternate +baseline and treatment order. Change the fixture with `--pairs`, `--directories`, +`--files-per-directory`, or `--interval-ms`. + +The result records Node/platform, baseline SHA, treatment HEAD and uncommitted +service diff. Compiled modules and source maps stay in the printed evidence +directory. Successful fixtures are removed; failed fixtures remain for diagnosis. + +This is a service benchmark. The HTTP endpoint is a responsiveness probe, not +the assembled Paperclip API. The lease-stamp callback resolves without database +work. It does not prove a hosted latency or total-CPU improvement. Process CPU +during publication includes foreground requests that the synchronous baseline +cannot serve until cleanup completes. Async cleanup can consume more CPU while +keeping the event loop responsive. Qualify the affected run/checkpoint/recovery +journey and compare matched hosted windows before claiming deployed acceptance. diff --git a/doc/plans/2026-10-31-zero-downtime-web-deploys.md b/doc/plans/2026-10-31-zero-downtime-web-deploys.md index 71e4bf658b2..fac22b56194 100644 --- a/doc/plans/2026-10-31-zero-downtime-web-deploys.md +++ b/doc/plans/2026-10-31-zero-downtime-web-deploys.md @@ -1,10 +1,201 @@ # Zero-downtime web deployments -Date: 2026-10-31. Status (2026-11-01 15:39 UTC): existing-volume staging enrollment -and manual sleep/wake preserve saved files and read/save behavior. The full local -active-agent journey now passes on `cc23019691`; hosted browser continuity and -the ten-deploy staging campaign remain unqualified. Natural database autosuspension -passes after abandoned test-resource cleanup. No production canary yet. +Current state (2026-11-02 18:31 UTC): cleanup recovery PR +[#18633](https://github.com/paperclipai/paperclip/pull/18633) merged as +`60a99307b3bb1062078d35af6677e3f377d8bab8`. Its final head passed all PR CI, +full local typecheck and build, and seven focused cleanup regressions. Greptile +reviewed that head at 5/5; both review threads are resolved. The duplicate full +local test run was stopped after six chat shards passed, while general suites +were still running. Its incomplete local result is preserved separately from +successful full PR CI. + +Canonical Linux release verification failed on the unchanged two-second busy-row +startup test on several consecutive revisions, including the cleanup merge. +The held-row diagnostic completed in 420 ms on macOS without a database lock +wait. Inspection found that `currentNativeControllerIdentity` always called the +server-info snapshot, which synchronously probes Git three times even when the +OS process birth time is available. Controlled one-second delays in real Git +commands reproduced the exact startup assertion failure with real PostgreSQL. +The follow-up separates the constant fallback process birth time from diagnostic +Git metadata. New regressions cover both available and unavailable OS birth time. +The two-second startup limit and release gates remain unchanged. Linux release +verification must pass before the new source can enter hosted qualification. + +The follow-up is [#18649](https://github.com/paperclipai/paperclip/pull/18649). +Its first head received two 5/5 reviews and passed every CI test shard except +an existing voice-retention precision test. That test advances its fake clock +automatically while asserting behavior across a one-millisecond boundary. A +controlled 60 ms database-entry delay reproduced the exact failure. Holding the +clock fixed makes the same assertion pass with that delay; the final correction +only disables automatic clock advancement in that one precision test. The +original CI failure and both controlled results are retained. No production +voice code or assertion was weakened. + +The earlier ten-roll result remains evidence for its recorded images. Neither +the cleanup fix nor the controller-identity follow-up has reached the staging +fixture. Final acceptance still requires a qualified compatible image pair, +fresh staging replacements, natural idle and browser sleep/wake, followed by the +isolated production canary. Cloud source is `b44304603d`; its private packages +require comparison against the new Core SDK and shared archive. Earlier candidate +Core `42c64d4ec4` differs in SDK; `b9e94b85a6` differs in shared types. Neither is +a valid rollback pair for the cleanup merge. No release gate was bypassed. + +The production browser was signed in as Dotta B on the ordinary new-organization +form. No organization was submitted and no production config changed. The only +hosted target changed so far is the pinned staging qualification fixture. +Disposable local PostgreSQL and Moto S3 fixtures passed an independent object +read/write preflight. Docker subsequently stopped; local container qualification +requires restoring that dependency. Hosted acceptance uses real object storage. + +Earlier retained evidence and failure history follows. + +Date: 2026-10-31. Status (2026-11-02 17:23 UTC): Core #18554 is merged as +`1a6017d6460267ad89d7e1784e2b521c28d8e9ac`. The ten-roll and native-agent +proof used Cloud `461c4aea28`. Separately reviewed Cloud #1091/#1092 subsequently +reached staging as `89dd0cfec2`; their changes concern workspace cleanup and +Agent Computer resource fencing. Railway, fleet and deployment workflow paths +are unchanged. Production remains on its original controller revision; the +next manual controller deployment must use current reviewed master. +Both immutable Cloud distributions built from that fixed source passed packaged +Linux rehearsals through the Railway wrapper: 260 reads, 53 saves, zero failures, +legacy-volume migration, an unready replacement, and removal of both predecessor +homes. Each fixture cleaned its containers. These rehearsals do not exercise an +active agent or an actual browser. The two distributions have distinct image +digests but identical application source; their compatible rollback qualifies +artifact replacement, not compatibility between different source revisions. + +The final Core PR has two 5/5 reviews and all required checks passed. One legacy +browser CI shard failed because its deferred-wake fixture observed no queued wake; +the unchanged case passed locally on isolated PostgreSQL and its single CI retry +passed. The original failure remains in the PR evidence. The full local live-agent +journey passed on `cc23019691` before the final non-suspended dispatch refinement; +that refinement passed its three focused Linux regressions, server typecheck, +build and final CI. Do not describe the earlier live run as a rerun of the merge. + +The fixed canonical package baseline passed on the pinned migrated staging fixture +using the allowed one-time maintenance path. An actual Chromium tab running the +older UI reconnected, retained an unsaved title edit, and saved it through the new +server; an independent API read confirmed the title. That old tab opened the +Agents page, and all four previously captured asset hashes still match after the +old deployment was removed. Baseline maintenance interruptions are preserved and +are not counted as zero-downtime qualification. + +The first ten-roll attempt was cancelled at step one, before Railway replacement. +The controller's pre-existing hot-upgrade allowlist named an older test stack, +so it drained rather than negotiated preservation of the new fixture's active run. +Continuous read/save and browser probes remained healthy, but this is NOT active +execution continuity evidence. The original command completed normally once; its +downloaded output bytes and native terminal were independently verified. The +owned fixture has been added to the existing allowlist without changing fleet +defaults. The normal staging controller workflow is applying that correction. +The rollout harness now checks both canary policy and positive tenant readiness +before submitting a campaign. The cancelled attempt remains failed evidence, +with no automatic campaign retry. +The configuration deployment then failed before Railway submission when Sentry +release registration failed or timed out. Its source checks passed; the original +logs are retained. One unchanged whole-workflow retry preserves attempt-bound +preflight requirements. That retry failed at the same pre-Railway Sentry request. +A read-only Sentry lookup confirms the release exists (HTTP 200). The canary-only +configuration was then applied by redeploying the existing successful controller +deployment: the source and replacement image digests are identical, and health +attests unchanged Cloud `461c4aea28`. Both failed workflow attempts are retained; +no Sentry gate was disabled and no new application release was built in this +configuration recovery. The new active-command attempt and gated sequence are +starting with the corrected controller. The first preflight stopped before task +admission on a missing optional plan's 404. `useIssuePlanDocument` explicitly +maps that response to null; an independent read confirmed no plan on the owned +completed task. The raw response remains recorded and the browser gate permits +only that verified path. The configuration redeploy's actual browser reconnect +was 1.22 seconds, with no probe read/save failure. + +All ten Railway replacements are independently qualified. Final continuous proof +passed 12,521 reads and 1,566 saves, with every acknowledged save present exactly +once and all 34 file hashes matching. There were zero HTTP failures. Actual +browser reconnects across the ten deployments ranged from 1.16 to 1.29 seconds. +The old UI assets and original agent artifact remain readable after deployment +ten and provider cleanup. Both traffic observers have stopped cleanly. The first +replaced the webserver while the original eight-minute remote command was +running. Its one execution ID started before replacement and completed afterward +with exit zero. The second replaced the server after completion; a later follow-up +retained the same native/provider session and runner, with accepted native +terminals and finalized accounting for both runs. The 91-byte artifact was +downloaded and independently checked for exactly one start and completion. +Actual browser reconnects were 1.28, 1.16 and 1.25 seconds; continuous reads and +saves have no failures so far. Previously unopened Projects and Skills pages rendered in the +retained old tab. A title draft survived the third cutover and was saved through +the UI, then independently read back from the replacement. + +The harness stopped after deployment three on another owned task's expected +missing optional-plan 404. The UI explicitly treats this as null. Both exact task +paths were independently checked, and the provider deployment, package eligibility, +serving identity, removal of the predecessor, browser timing and continuous probe +were revalidated. The failed original receipt remains unchanged alongside an +explicit classification review. The remaining seven replacements continue without +redeploying or retrying deployment three. An announcement also obscured one +sidebar click; its ordinary Dismiss control resolved that retained interaction +timeout. A separate large run-log diagnostic timed out; a narrower public-events +read succeeded and proves the original command's single execution identity. +A later bounded 64-KiB log read succeeded from shared storage after both the +originating webserver and remote sandbox were removed. + +The native recovery event records SIGTERM without a retry run, then adoption of +the same runner at controller generation 2. Each of the two runs has exactly one +accepted result and terminal event. The independent accounting inspector has no +findings or pending work; the task totals 52.941 cents across two runs. Including +the earlier no-replacement attempt, staging has three estimated cost events totaling +113.84284 cents. The temporary agent is paused, its secret removed and the prior +company budget restored; all three provider resources are independently confirmed absent. The first delete +was refused while the sandbox stopped; that failure remains alongside the +successful cleanup after a fresh stopped-state observation. Both owned local +PostgreSQL/object-storage containers are removed; test evidence is retained. + +Deployment nine also completed, but the local test reader observed the browser +snapshot while its writer had truncated the file. The campaign response was +already parsed and successful; the only non-atomic JSON read in that failing +checkpoint is the local browser report. A concurrent-write regression reproduced +the problem. The reader now retries only local JSON syntax errors for at most +100 ms; persistent corruption still fails, and HTTP/acceptance failures are never +retried. Deployment nine was independently revalidated without redeployment. +Both raw failed sequence receipts and their explicit classification reviews remain; +the ten provider deployments themselves have not been retried. The final staging +driver stopped at the harness failure and is being resumed from the retained +sequence. The normal UI's 256,000-byte run-log read also passes after replacement. + +The first repair check stopped before mutation because an unrelated staging +campaign was active. That campaign has finished, and the pinned canary remains +active and unlocked on the qualified final image. Both one-target configuration repair campaigns passed: backup policy was +restored, the temporary marker removed, and durable files/shared assets remained +enabled on the same volume-free image. The current-image eight-minute natural-idle check failed with all browser pages and +traffic probes closed. Bounded SQL diagnosis found three completed native runs +still in agent-file collection recovery. Normal archive of both completed test +workspaces independently passed close-readiness and preserved saved workspace +files. It produced provider-confirmed destruction receipts for the latest leases. +Two copies then completed no-save cleanup, but retained stale retry deadlines; +the earlier run of a reused sandbox still lacks direct termination evidence even +though its validated successor allocation is destroyed. These are source recovery +gaps, not an idle-proof pass. No historical rows were manually rewritten. The +failed idle and cancelled browser lifecycle receipts remain unchanged. Fix +ownership belongs to remote termination proof resolution and agent-directory +cleanup scheduling; no polling cadence or sleep-safety check should be relaxed. + +The isolated source branch is now based on current master `97c4dc969e`. The +recovery fix resolves the destroyed allocation through the existing validated +handoff-chain contract at read time; it does not stamp historical leases or infer +stop from task status. Agent-file cleanup also reconciles stale retry deadlines +with a receipt comparison. Both owning service suites passed 169 tests, and the +server typecheck passed. The first new deadline test caught a public guard that +skipped settlement; that failed attempt is retained and the corrected five-case +selection passed. The final seven-case selection passed, including a concurrent receipt change and +cross-company rejection. Repository-wide checks and PR review are pending. +This source has not been deployed and is not covered by the prior ten-roll proof. + +Next: review and ship the cleanup fix, complete current-image idle/sleep/wake, then the +isolated production forward/rollback canary. No production deployment has occurred. +Dotta completed the authorized GitHub sign-in; the production browser is back on +the ordinary organization list. The production canary still waits for staging. Its new organization form is +prepared but unsubmitted. Production remains on its original controller revision, +with no campaigns, the single-volume default and continuous deployment disabled. Earlier +staging sleep/wake and natural database-idle proofs remain valid for their image. Core #18543 merged as `7751fc3107` after 55 successful CI checks, Greptile 5/5 and no unresolved threads. It supplies runtime admission, immutable-cache @@ -202,7 +393,7 @@ Measure one-time setup separately from repeated attempts. Keep failed evidence. | Item | State | Next action | | --- | --- | --- | | Prior prerequisites | #18403 and all five storage workstreams merged | Retain the ephemeral filesystem contract | -| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 reviewed at 5/5 with CI pending; local packaged migration, browser overlap, cold restoration and durable command proofs recorded below | Publish and qualify the exact fixed release after CI/review merge | +| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 merged with all required checks and 5/5 reviews; both exact fixed images passed packaged migration and deletion; active hosted output verified | Finish hosted continuation and accounting reconciliation | | Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged; #1102 serving on staging | Verify repeated Railway overlap behavior after Core recovery qualification | | Staging | Original enrollment complete; migrated data and signing key preserved; volume detached and retained | Sleep/wake and natural database idle pass; verify repair and ten app-only replacements with browser continuity | | Production | No canary mutation performed | Proceed only after the staging and active-agent acceptance gates pass | diff --git a/maps/agent-access-map/app-permissions.md b/maps/agent-access-map/app-permissions.md index 5d91fa39f81..78877ca1bf1 100644 --- a/maps/agent-access-map/app-permissions.md +++ b/maps/agent-access-map/app-permissions.md @@ -30,7 +30,10 @@ Review: `operation-reviewed`. API: `present` — `GET /api/tool-connections/{connectionId}/grants`; JSON REST. Agent access: `available` — agent REST and default native `search_api` / `call_api`. -Policy: `tools:manage_connections`; [shared management policy](../../doc/plans/2026-10-13-connection-api-parity.md#authorization-contract). Resolved resources and every affected target are checked; personal ownership remains the responsible user. +Policy: `tools:manage_connections`; [shared management policy](../../doc/plans/2026-10-13-connection-api-parity.md#authorization-contract). Resolved resources and every affected target are checked; personal ownership remains the responsible user. Human capability descriptions +reuse retained-target checks within one request for direct tool and AI connections. +Pools with referenced accounts and delegated actors retain the full evaluator; +mutations always reauthorize. Evidence: `source-only` — [route](../../server/src/routes/tool-access.ts); reviewed in the working tree based on the source commit above. Verification: `not-run` — individual operation not executed in this audit; shared authorization and representative family flows are exercised by the authenticated integration suite. Follow the route schema, inspect persistence/audit, and assert owner, scope and revoked-authority denial. Gaps: This individual operation has source review but no recorded HTTP verification. Provider availability and live setup remain unqualified. @@ -311,6 +314,15 @@ Capability ID: `app-permissions/account-maintenance`. Review: `operation-reviewed`. +### `get-companies-ai-connections-identity` + +API: `present` — `GET /api/companies/{companyId}/ai-connections/{connectionId}/identity`, optional `grantId` query; JSON REST. +Agent access: `available` — agent REST and default native `search_api` / `call_api` through the registered connection-management operation. +Policy: `connections:read` under current connection-management authority, active company membership, and the selected grant's human audience. Manager status alone cannot read a coworker's personal provider profile. Revoked or disabled accounts do not open credentials. No raw provider IDs or credentials are returned. +Evidence: `verified` — local PostgreSQL and actual key/JWT REST requests in [connection-management integration](../../server/src/__tests__/connection-management.integration.test.ts), with provider fixtures. [AI connection tests](../../server/src/__tests__/ai-connections.test.ts) cover private-user and unauthenticated denial. Native API discovery is registered in [OpenAPI](../../server/src/routes/openapi.ts); native tool execution is source-only. +Verification: `passed` — safe email/organization for an authorized selected grant; another company, another user, invalid grant, and missing authority denied before provider access. Profile permission denial remains an explicit unavailable observation rather than a failed connection. Fixtures do not qualify live provider profile access. +Gaps: Provider profile access depends on the saved token's scopes; an inference-only setup token cannot identify its account email. + ### `get-companies-tools-apps-attention` API: `present` — `GET /api/companies/{companyId}/tools/apps/attention`; JSON REST. diff --git a/maps/agent-access-map/chat-channels.md b/maps/agent-access-map/chat-channels.md index 16413dd1eff..da6e5478a37 100644 --- a/maps/agent-access-map/chat-channels.md +++ b/maps/agent-access-map/chat-channels.md @@ -1028,6 +1028,13 @@ receipt preserves the original skipped receipt and older failed native result. Current access and normal native stop/cleanup guards still govern execution; board wake metadata alone cannot manufacture chat authority or replay old work. +After a successful connected run, board Retry can resume a newer deferred inbound +receipt through the original chat owner. It rechecks the board's current task +permission and the saved source's current authority, and waits for prior cleanup. +It grants no permission to replay the successful result or manufacture a chat +request. Concurrent clicks share the canonical retry receipt; completed tasks +and superseding execution retire unadmitted retry intent. + Deferred chat Retry excludes independent board queues from its pinned batch. An explicit board Retry can restore a historical mixed batch only with exact never-started evidence, current source authority, unchanged board comments and diff --git a/maps/agent-access-map/documents-artifacts.md b/maps/agent-access-map/documents-artifacts.md index 0d95367b9c7..4d840550cc5 100644 --- a/maps/agent-access-map/documents-artifacts.md +++ b/maps/agent-access-map/documents-artifacts.md @@ -44,6 +44,14 @@ CLI / MCP. Policy: Task/resource visibility, mutation/checkout ownership, revision/lock guards and permitted deliverable context. +Native completion evidence accepts the server's relative link only for a document +attached to the current company/task with a matching current-revision publication +receipt. It can accompany a separately verified requested file; it cannot replace +that file. The [native feedback regression](../../server/src/services/native-runtime/native-completion-feedback.test.ts) +exercises the native write/register/finish boundary with local PostgreSQL and a +controlled remote file reader. It does not qualify the whole document API or a +live provider run. + Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L10299) at the reviewed commit above. diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index d551c6b1824..18894d62573 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -326,6 +326,12 @@ API: `present` — POST /api/issues/{id}/recovery-actions/recover-workspace with Agent access: `restricted` — board task-read and issue:mutate; not runtime:manage. Agents denied through REST/native fallback. +An explicit request may also preserve a cancelled export-repair action that a +newer failure superseded. It retains the exact original source checks, current +writer exclusion and bounded maintenance budget. It does not change the newer +task workspace or recovery owner, apply an older result over newer execution, +or admit a provider turn. A cancelled record without that request is ineligible. + Policy: Exact terminal retained source/lease/reference; independent stop and physical ownership checks; no newer writer. Bounded queued repair preserves newer files and has retry state. Matching key/recovered state dedupes; permanent blocked source refuses. Recovery starts no provider turn. For native export repair, a reusable lease is eligible only with its exact descriptor and the finalizer-selected accepted successful result under the same company/task/run/completion contract. This does not change actor access or permit another allocation owner or host writer. Invalid Git metadata is a non-retryable persistence failure requiring service repair; it cannot use unsafe-archive omission to complete the result. Run-detail/activity projections retain the allowlisted code, including bounded large-result retrieval, and cancelled-preservation discovery honors the same hold. Evidence: `source-only` — [recovery routes](../../server/src/routes/issues.ts), [retained recovery](../../server/src/services/workspace-recovery.ts); reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`. No live provider proof. diff --git a/maps/agent-access-map/runs-adapters.md b/maps/agent-access-map/runs-adapters.md index a16aef4da4d..74728827d57 100644 --- a/maps/agent-access-map/runs-adapters.md +++ b/maps/agent-access-map/runs-adapters.md @@ -214,7 +214,9 @@ use the server-owned authenticated relay. The provider sees bounded then uses pinned aliases. Search describes its 1–20 page limit and default of 5 in the tool and parameter descriptions. It intersects fresh grants and work modes; calls retain the run's original gateway policy, validation, approvals, revocation and -audit path. Prompt arguments cannot select gateway authority. Assigned gateway +audit path. Blank schema references perform normal discovery; positive schema +offsets require an exact reference, and unavailable references explain how to +search again. Prompt arguments cannot select gateway authority. Assigned gateway launch credentials stay on the controller. Former direct-catalog OpenCode checkpoints refresh; other provider fingerprints stay unchanged. The [projection controls](../../server/src/services/native-runtime/assigned-mcp-tools.test.ts) @@ -318,6 +320,18 @@ session-control access. Deployed diagnostic usefulness remains unverified. ## `session` +Fresh remote Retry preparation retains the admitted strategy in its saved +fingerprint. An unchanged follow-up must not report strategy drift; actual policy +changes retain the replacement guard. This changes no retry, session-reset, +workspace or credential authority. The +[preparation regression](../../server/src/services/heartbeat/workspace-preparation.test.ts) +uses PostgreSQL and controlled runtime acquisition. Live provider resumption +remains a separate check in the [session recipe](../feature-map/runs-adapters.md#session). + +Older strategy fingerprints retain their existing guard until an explicitly +admitted fresh recovery. A remote Git row alone does not authorize rewriting its +configuration history or bypassing a real policy change. + Scoped source evidence: restart preparation transfers its lease renewal to the native executor after the executor acquires its own reference. Completion stops renewal before coordinator release, while real ownership loss retains the detach diff --git a/maps/agent-access-map/tasks.md b/maps/agent-access-map/tasks.md index 7f80a1314c0..07604564b8a 100644 --- a/maps/agent-access-map/tasks.md +++ b/maps/agent-access-map/tasks.md @@ -74,7 +74,7 @@ API: `present` — GET /api/companies/{companyId}/issues; optional view=compact. Agent access: `partial` — REST/call_api full query; CLI issue list exposes status/agent/project and local --match on one fetched page; native search_tasks query up to 500 characters, statuses, limit 1..200 (default 50), and a nextCursor in ascending task-ID order. Responses expose effective limit, returnedCount and hasMore. Exhaust nextCursor with the same filters before treating an inventory as complete. Each page checks current access; concurrent task changes require a fresh scan. Project/label/workspace knobs remain REST-only. -Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|id, sortDir asc|desc, afterId. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results. +Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|created|id, sortDir asc|desc, afterId. Blocked attention lists reject created and id ordering with 422. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results. Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [issues.ts](../../server/src/routes/issues.ts), [issues.ts](../../server/src/services/issues.ts), [paperclip-runner-tool-authority.ts](../../server/src/services/native-runtime/paperclip-runner-tool-authority.ts), [issue.ts](../../cli/src/commands/client/issue.ts), [issues-list-query-parsing.test.ts](../../server/src/__tests__/issues-list-query-parsing.test.ts), [issues-creation-visibility.test.ts](../../server/src/__tests__/issues-creation-visibility.test.ts), [issue-list-assignee-filter-routes.test.ts](../../server/src/__tests__/issue-list-assignee-filter-routes.test.ts), [private-json-etag.test.ts](../../server/src/__tests__/private-json-etag.test.ts). No live execution claimed. @@ -120,7 +120,10 @@ Agent access: `not-applicable` — no agent layout API; permitted underlying tas Policy: IssuesList loads/saves browser preferences by company/collection, with separate agent-created visibility. The Tasks page also owns a routine-only checkbox and `originKind=routine_execution` URL state; it uses the existing -company-scoped task-list origin filter before pagination. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred. +company-scoped task-list origin filter before pagination. Sort controls write +`sortField` and `sortDir` to the URL; explicit URL sorting overrides browser +preferences. `created` sorting runs on the server before pagination, with ID as +a stable tie-breaker. The routines task link requests `created` descending. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred. Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [IssuesList.tsx](../../ui/src/components/IssuesList.tsx), [Issues.tsx](../../ui/src/pages/Issues.tsx), [Issues.test.tsx](../../ui/src/pages/Issues.test.tsx). No live execution claimed. @@ -252,6 +255,8 @@ Gaps: Watchdog execution/recovery acceptance remains [recovery](recovery.md), be Capability ID: `tasks/discussion`. +Native deferred-reply recovery: the server can publish an accepted ordinary-task `needs_review` reply from its completed final-message event after workspace finalization. It uses the same company/run/turn, result-digest and publication authorization checks as completed-task recovery. It does not publish a semantic summary as a substitute, resolve a pending review, or start provider work. Conversation and external governed-response rules retain their separate gates. The [database recovery suite](../../server/src/services/native-runtime/agent-chat-response-recovery.test.ts) covers publication once, pending-review preservation, and rejection of missing/wrong-turn output, invalid digest and suppressed comments. This is local integration evidence; live acceptance is recorded in the pull request. + Review: `operation-reviewed`. Read, append/retry, author-delete or queued-cancel messages, attach allowed files, and distinguish human feedback from agent reporting. @@ -522,3 +527,9 @@ Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7 Verification: `not-run` — Compare eligible owner/admin ordinary denied with explicit read/audit+notice; ordinary operator/agent/foreign denied, grants unchanged. Gaps: Local implicit/instance admin bypass cannot establish ordinary two-user private isolation. + +Task reads expose optional read-only `displayStatus`. `monitoring` describes a +scheduled ordinary monitor wait; it is not accepted as a writable task `status`. +Existing lifecycle `status` filters retain their meaning. Board collection and +search readers can use the separate `displayStatus` filter before pagination. +Agents keep using the existing monitor and semantic completion tools. diff --git a/maps/agent-access-map/workspaces.md b/maps/agent-access-map/workspaces.md index c2b261a7f4f..551170cf4f7 100644 --- a/maps/agent-access-map/workspaces.md +++ b/maps/agent-access-map/workspaces.md @@ -74,6 +74,9 @@ without a controller checkout fallback. Current authorization and responsible-us checks still apply; read-only cross-project referenced folders and explicitly concurrent shared writers are rejected. Managed mutable same-project repositories are supported. +Portable capture streams validated working files while staging self-contained +Git metadata. The smaller temporary-space requirement grants no new agent access +or execution authority. Changed sources still fail before a save receipt. An inherited isolated workspace or operator branch can be reserved through the task's current persisted binding. The original source task remains provenance. Reservation checks task/run, binding, trust, and retained source visibility before @@ -293,6 +296,9 @@ workspace metadata, and operation logs retain their app-data privacy. For enrolled remote workspaces the same board file-resource endpoints read the authorized live allocation or a verified committed saved version. Returned `source.kind` is `live` or `saved`, with `savedAt` for a saved copy. An unavailable +clean allocation does not prevent reads of its current committed save once +writer authority is clear; saved reads validate the exact company, workspace, +and snapshot even while a remote location remains recorded. An unavailable dirty allocation cannot silently fall back to stale saved bytes. Remote paths never become controller filesystem authority. Confined path/symlink checks, bounded search/preview, download streaming, and cancellation remain enforced by diff --git a/maps/feature-map/app-permissions.md b/maps/feature-map/app-permissions.md index 524cc9b4feb..0fd4b55a1df 100644 --- a/maps/feature-map/app-permissions.md +++ b/maps/feature-map/app-permissions.md @@ -97,6 +97,20 @@ still need live account qualification. Automated: [app detail tests](../../ui/src/pages/apps/AppDetail.test.tsx) cover test-dialog/UI contracts. Real provider execution and revocation require the selected provider’s live check. +AI account details share the Permissions page's grants query. Account and grants +reads have a 15-second deadline. The AI panel retries a transient failure once, +then shows an error with Retry. A background failure keeps loaded controls visible. +[AI account loading tests](../../ui/src/components/ai-connections/ManagedAiConnectionDetails.test.tsx) +exercise stalled account/grants reads, cancellation, retry recovery and cached grants. + +The account-details section shows the provider email and organization when +available, beside the sign-in method and Paperclip ownership. Anthropic profile +permission denial shows an explicit unavailable message without changing the +connection's connected state. Account identity loads independently of maintenance +controls, has a bounded wait and a separate retry for temporary failures, and +is refreshed after a credential grant changes. [Account-details tests](../../ui/src/components/ai-connections/AiConnectionAccountDetails.test.tsx) +cover these states and prevent one grant's email from appearing for another. + Manual: Run a read-only action with explicit inputs and inspect returned data and activity. Refresh discovery and review new actions. Revoke or disconnect only the disposable credential, then verify subsequent use fails with an actionable reconnect state. For Daytona, opening an older saved catalog discovers the current native diff --git a/maps/feature-map/chat-channels.md b/maps/feature-map/chat-channels.md index 45802a29aeb..7eb26bbc117 100644 --- a/maps/feature-map/chat-channels.md +++ b/maps/feature-map/chat-channels.md @@ -278,6 +278,14 @@ After successful admission, reload and verify that the prior skipped receipt doe not strand a later real provider reply or tree resume. Provider fixtures support this behavior; live fixed-source runtime and destination proof remain separate. +Also complete an older review successfully while a newer authorized PR event +remains deferred. Use Retry and verify that the existing saved-input receipt +starts exactly one new turn, repeated clicks share that turn, and the older +accepted result remains unchanged. Revoke the connection or change the source +binding before dispatch and verify no run starts. Keep prior cleanup pending, +then settle it and verify the same receipt resumes once. These database-backed +provider fixtures do not establish live GitHub publication at the fixed revision. + Also defer Retry behind cleanup, queue an independent board message, then complete cleanup. Verify one run receives the exact GitHub batch and the board message enters its own turn. For an older mixed batch rejected before provider diff --git a/maps/feature-map/documents-artifacts.md b/maps/feature-map/documents-artifacts.md index a7813b17f1b..7174eed7bdd 100644 --- a/maps/feature-map/documents-artifacts.md +++ b/maps/feature-map/documents-artifacts.md @@ -94,6 +94,13 @@ Manual: Edit a paragraph, save, reload, and compare revisions. Add an annotation Automated: [attachment routes](../../server/src/__tests__/issue-attachment-routes.test.ts) and [work products](../../server/src/__tests__/work-products.test.ts) cover data/access behavior; [artifact arrival browser suite](../../tests/e2e/artifact-tab-arrival.spec.ts) targets arrival. [Artifact uploads](../../tests/e2e/artifact-uploads.spec.ts) drives file selection/drop, filename collisions, retry, and preservation of paused/unassigned tasks. +[Native completion feedback](../../server/src/services/native-runtime/native-completion-feedback.test.ts) +checks a real saved-document link alongside a verified CSV attachment, including +attachment download query and fragment links. It rejects foreign task/company +links, missing documents, unproven current revisions, and workspace files. The +document alone cannot satisfy the requested file. These local tool/Postgres +checks do not establish hosted delivery or browser rendering. + Manual: Upload a small Markdown file and CSV through Artifacts. Markdown becomes a versioned task document; other files become attachments. Read the document, download the CSV and compare bytes, then reload. Upload the same Markdown filename diff --git a/maps/feature-map/inbox-search.md b/maps/feature-map/inbox-search.md index e6904ca8d2e..eb11c0d5f85 100644 --- a/maps/feature-map/inbox-search.md +++ b/maps/feature-map/inbox-search.md @@ -91,7 +91,11 @@ Cold-open verification: load Mine with the task-index response delayed while approvals/runs resolve. Only the stable list placeholder should appear until the initial sources settle; a failed-run row must not jump when tasks arrive. Release visible-row details separately and compare row/title bounds. Mine reads its own -100-row index, with parent/run context by ID; sidebar badges use the count endpoint. +100-row index. Missing parents use the same index projection for their captions; +parents already in the index are reused. Failed-run tasks still load rich row +context by ID, including when they are also parents or already indexed, so resolved +interruptions stay hidden. The initial reveal waits for both context sources. +Sidebar badges use the count endpoint. Coordinator desktop and mobile navigation share `/attention/summary` for the outstanding count. Needs You owns its complete feed; Tasks → Needs attention owns a compact complete task-ID index. Navigation reuses either page response. Coordinator keeps the optimized Inbox even when diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index b97ac0d9a08..1ba9594a4d7 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -429,6 +429,17 @@ Verify saved and newer local files, one preservation receipt, and no additional agent turn. The PostgreSQL regression covers this boundary; the opt-in provider probe also uses this runtime path with the protection marker. +Repeat Task Retry with a completed native result whose unfinished export uses +the remote-workspace descriptor. Before preservation, require the original run +to remain the only task run. After preservation, verify restored file bytes and +Git HEAD, commitment of that same result, and an unanswered human review keeping +the task In Review. For an action superseded by a newer failure, request exact +historical file recovery through the board API: verify a committed snapshot, +unchanged newer workspace binding and attention owner, no old-result commitment, +and no provider turn. An active current writer or no explicit request keeps the +historical source waiting. The PostgreSQL/HTTP regression is in +[workspace recovery](../../server/src/services/workspace-recovery.test.ts). + For a missing managed allocation, verify a fresh Cloud-signed absence observation under the exact physical recovery owner. The existing Retry may then select an independent workspace through normal admission. Verify that the original run diff --git a/maps/feature-map/routines.md b/maps/feature-map/routines.md index 75f8e21a926..bb830f726a8 100644 --- a/maps/feature-map/routines.md +++ b/maps/feature-map/routines.md @@ -30,7 +30,7 @@ Open `/routines`, create one, and inspect `/routines/:routineId` and its edit se Use the routine's trigger controls and Runs/Recent Runs/Audit links; lists also offer Run now. **View all tasks** opens the Tasks list with **Routine tasks** -selected, using `/issues?view=all&originKind=routine_execution`. Uncheck the +selected, using `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc`. Uncheck the filter to return to other tasks. **History** records configuration versions, not executions. ## Driving it with Playwright and the Paperclip CLI diff --git a/maps/feature-map/runs-adapters.md b/maps/feature-map/runs-adapters.md index 11aa505a521..51092b077e0 100644 --- a/maps/feature-map/runs-adapters.md +++ b/maps/feature-map/runs-adapters.md @@ -146,6 +146,18 @@ must still reset the session and preserve the stored workspace owner. The checks this with PostgreSQL and synthetic provider execution; it is not live warm-process or latency evidence. +After an isolated remote Retry, send a follow-up with unchanged workspace policy. +Verify the same workspace remains bound and no strategy-replacement operation is +recorded. Change the strategy or base ref and confirm that the normal replacement +guard still applies. The +[remote preparation regression](../../server/src/services/heartbeat/workspace-preparation.test.ts) +checks persisted configuration across both admissions; it does not prove a live +provider conversation resumed. + +For a workspace carrying an older temporary fingerprint, verify that ordinary +reuse retains the strategy warning. Use the normal fresh recovery flow after +preserving its work, then repeat the unchanged follow-up check on the new workspace. + For `runs-adapters/session`, complete a task in a disposable warm sandbox and retain its saved workspace file. Let the warm session close normally and verify that the retained lease has a complete, lease-bound harness backup before @@ -273,6 +285,10 @@ For native OpenCode with assigned connections, inspect the provider tool surface on both local and sandbox execution. It should contain bounded assigned-tool search/call schemas. Confirm the search description states the 1–20 page limit and default of 5. Follow `nextOffset` to inspect later pages. +Repeat discovery with empty and whitespace-only `schemaTool` values, including +`schemaOffset: 0`, and confirm it still returns searchable tools. An unknown +schema reference or a positive schema offset without a reference must explain +how to correct the call. Discover an allowed tool, call its returned alias, revoke its grant, and confirm discovery and execution enforce that revocation. Repeat in planning/ask mode with a write tool and confirm denial. The controller must diff --git a/maps/feature-map/tasks.md b/maps/feature-map/tasks.md index a9afa296295..66fa4b4b99a 100644 --- a/maps/feature-map/tasks.md +++ b/maps/feature-map/tasks.md @@ -56,9 +56,17 @@ Use New task from the global launcher, task list, or project task view. ### `task-list` +Scheduled ordinary monitor waits display **Monitoring** with a blue clock in +current task indicators, groups, board columns, and filters. Compare Monitoring +with In Progress, then start the check and verify the status returns to its +execution presentation. Retained monitors must not override blocked/review or +terminal states. Inspect due/overdue copy separately; no runner is implied. + Open `/issues` or a project’s Issues view; legacy `/tasks` links redirect. Filters → **Routine tasks** selects tasks created by routine runs. The -`/issues?view=all&originKind=routine_execution` link opens that selection. +`/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` link opens that selection with newest-created tasks first. Sort controls update +`sortField` and `sortDir` in the URL. Reloads and task return links keep that order. +Shared sort links also retain their order when opened in another task interface. ### `task-detail` @@ -77,6 +85,8 @@ and a bounded wait for stalled fonts. Open `/issues/:issueId` from a row, mention, or direct link. Cold opens show the instruction, recent exchange and saved cards together. Completed runs with saved replies defer their transcripts until the reader opens history. Live runs, failures, yielded runs and runs without a saved reply still hydrate; an already shown transcript stays available after completion. No URL fragment expands history; older-comment anchors fetch only their target, and document anchors open the pane. **More messages** appears only when earlier comments or runs are omitted and loads that history. Fresh tasks with no omitted history have no disclosure. A single run with deferred output or a truncated legacy log offers **More run output** instead. **More task actions → Show task activity** loads full activity even for a single native run; incoming messages remain visible while the task stays open. +For an ordinary native task, deferred workspace finalization must preserve the final reply for both completed work and work awaiting review. Reload must show that saved reply once. The pending review stays pending; publishing the reply does not approve it or start another run. The [database recovery suite](../../server/src/services/native-runtime/agent-chat-response-recovery.test.ts) covers deferred review publication, exact final-message selection and review preservation. Live acceptance is recorded in the pull request. + ### `task-sharing` Use the privacy control in New task, or open an existing task’s menu and choose **Make private**, **Share…**, or **Make public** where allowed. The share sheet lists direct grants and inherited access from private ancestors or project membership. Inherited rows have no local revoke action; manage that access at its source. diff --git a/maps/feature-map/workspaces.md b/maps/feature-map/workspaces.md index 83d5504ff63..95c2ac8a274 100644 --- a/maps/feature-map/workspaces.md +++ b/maps/feature-map/workspaces.md @@ -83,6 +83,14 @@ or original Git remote. Read-only cross-project referenced folders must fail admission. Separate workspaces may run concurrently; shared concurrent writers must be rejected. +Repeat remote capture with enough free space for the archive and normalized Git +metadata, but not another working-tree copy. Verify saved files and Git state +after restore. A concurrent file change must fail capture without publishing a +receipt or retaining a partial archive. The +[portable archive suite](../../server/src/services/remote-workspace-snapshot-archive.test.ts) +checks the space allowance and concurrent-change boundary; its filesystem quota +is controlled test input, not a live sandbox qualification. + Create a child that inherits a saved isolated workspace or operator branch. Verify it continues the same files after the prior writer saves, including on a new allocation. Two bound children must not acquire concurrent writer authority. @@ -94,6 +102,10 @@ Their filesystem transport is local test execution, not live sandbox/provider pr The Files panel must label **Live workspace** or **Saved copy** with its save time. Confirm browse, preview, and download agree. A missing allocation with unsaved work must report unavailability rather than show an old save as current. +After a final save clears dirty state and writer authority, Files must read the +current committed saved version even while the old allocation is still recorded +or its read transport is unavailable. Confirm the original bytes and Saved copy +label; a new writer or dirty state must retain the live-read requirement. Close must retain dirty/unfinished work and reject custom cleanup/teardown commands; a clean saved workspace can release compute while retaining files. Manual runtime commands, jobs, repairs, and service starts must reject remote diff --git a/packages/adapter-utils/src/acpx-engine/startup-timing.ts b/packages/adapter-utils/src/acpx-engine/startup-timing.ts index 74f1977c5c3..5ce26855ef2 100644 --- a/packages/adapter-utils/src/acpx-engine/startup-timing.ts +++ b/packages/adapter-utils/src/acpx-engine/startup-timing.ts @@ -67,6 +67,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = { * command, a path, an argument, or an environment value can never ride a span. */ export const SANDBOX_STARTUP_SPAN_ATTRS = { + /** Provider state before a lease start or recovery; the host clamps values. */ + state: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}state`, /** The low-cardinality provider family (through `normalizeProviderFamily`). */ provider: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}provider`, /** The step or execution outcome: `ok`, `skipped`, or `failed`. */ diff --git a/packages/adapter-utils/src/command-redaction.test.ts b/packages/adapter-utils/src/command-redaction.test.ts index 321866bf7c7..2afd383a6d9 100644 --- a/packages/adapter-utils/src/command-redaction.test.ts +++ b/packages/adapter-utils/src/command-redaction.test.ts @@ -145,3 +145,42 @@ second-line\" status=401`; expect(output).toContain(REDACTED_COMMAND_TEXT_VALUE); }); }); + +describe("redactCommandText on long runs", () => { + const jwt = + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c"; + + function redactWithinTimeBudget(input: string) { + const startedAt = performance.now(); + const output = redactCommandText(input); + expect(performance.now() - startedAt).toBeLessThan(1_000); + return output; + } + + it("keeps redacting options, assignments, and tokens inside dash-separated runs", () => { + const cases = [ + ["x--token flag-secret", `x--token ${REDACTED_COMMAND_TEXT_VALUE}`], + ["--x--token flag-secret", `--x--token ${REDACTED_COMMAND_TEXT_VALUE}`], + [ + "a-b--api-key=assigned-secret --password 'quoted-secret'", + `a-b--api-key=${REDACTED_COMMAND_TEXT_VALUE} --password '${REDACTED_COMMAND_TEXT_VALUE}'`, + ], + ["prefix-MY_API_KEY=env-secret", `prefix-MY_API_KEY=${REDACTED_COMMAND_TEXT_VALUE}`], + [`--${jwt}`, `--${REDACTED_COMMAND_TEXT_VALUE}`], + [`session=${jwt}`, `session=${REDACTED_COMMAND_TEXT_VALUE}`], + ["my-token is here", "my-token is here"], + ]; + + for (const [input, expected] of cases) { + expect(redactCommandText(input)).toBe(expected); + } + }); + + it("scans long letter/digit and dash-separated runs in linear time", () => { + const base64Url = Buffer.from(Array.from({ length: 90_000 }, (_, index) => (index * 7919) % 256)).toString("base64url"); + for (const run of ["a".repeat(120_000), "a-".repeat(60_000), "a--".repeat(40_000), base64Url]) { + expect(redactWithinTimeBudget(`${run} --token run-secret`)).toBe(`${run} --token ${REDACTED_COMMAND_TEXT_VALUE}`); + expect(redactWithinTimeBudget(`${run}--token run-secret`)).toBe(`${run}--token ${REDACTED_COMMAND_TEXT_VALUE}`); + } + }); +}); diff --git a/packages/adapter-utils/src/command-redaction.ts b/packages/adapter-utils/src/command-redaction.ts index 46eeb6f5040..451f0ccf58e 100644 --- a/packages/adapter-utils/src/command-redaction.ts +++ b/packages/adapter-utils/src/command-redaction.ts @@ -11,16 +11,19 @@ export function isPublicExecutorToolSelector(value: string): boolean { return PUBLIC_EXECUTOR_TOOL_SELECTORS.has(value); } -const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_NAME_WORDS = String.raw`(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*${SECRET_NAME_WORDS}`; +const SECRET_NAME_START = String.raw`(? { expect(parseSetupTokenPrompt(completeOutput(withKey))).toBeNull(); }); + it("skips the 2.1.291 copy hint between the URL and the prompt", () => { + const hint = chaSpaced("Hold Shift while selecting to use your terminal's native copy".split(" ")); + const text = [PREAMBLE_LINE, hardWrap(CLAUDE_COM_URL, 130), "", hint, "", "", PROMPT_LINE].join("\n"); + const result = parseSetupTokenPrompt(text); + expect(result?.url).toBe(CLAUDE_COM_URL); + expect(result?.prompt).toBe(SETUP_TOKEN_PROMPT); + }); + it("returns null when a code-like value sits on the line after the URL", () => { // The line right after the URL must hold the browser-code prompt. A code-like // value on that line cannot bind, so the parser returns null. diff --git a/packages/adapters/claude-local/src/server/setup-token-parse.ts b/packages/adapters/claude-local/src/server/setup-token-parse.ts index d845f5ab348..6028695d16a 100644 --- a/packages/adapters/claude-local/src/server/setup-token-parse.ts +++ b/packages/adapters/claude-local/src/server/setup-token-parse.ts @@ -181,6 +181,10 @@ const URL_PREAMBLE_RE = /the url below to sign in/i; // the line. const PROMPT_LINE_RE = /^Paste code here if prompted\b[\s>*]*$/; +// The copy hint that `claude` 2.1.291 prints between the URL block and the +// prompt. The parser skips this exact line and binds no value from it. +const COPY_HINT_LINE_RE = /^Hold\s+Shift\s+while\s+selecting\s+to\s+use\s+your\s+terminal['’]s\s+native\s+copy\.?$/i; + // The maximum number of characters between the end of the URL preamble line and // the start of the URL. The Claude UI prints the URL right after the preamble. // The parser accepts a URL start only inside this window, so a URL far from the @@ -363,9 +367,11 @@ function reassembleUrl(lines: string[], startLine: number): SetupTokenUrlMatch | * line after the URL block. The parser reads the non-blank lines after the URL * end line. The Claude UI can emit one full URL line per wrapped display row, so * it repeats the same full authorization URL on a few consecutive lines. The - * parser skips a line that repeats `url`, then binds the prompt on the first - * non-blank, non-repeat line. That line must match the exact prompt shape. - * Returns null when the first non-blank, non-repeat line is not the prompt. + * parser skips a line that repeats `url`. The `claude` 2.1.291 UI also prints a + * copy hint before the prompt, so the parser skips a line that matches {@link + * COPY_HINT_LINE_RE}. The parser then binds the prompt on the first non-blank + * line that is not a repeat and not the copy hint. That line must match the + * exact prompt shape. Returns null when that line is not the prompt. */ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): string | null { for (let i = endLine + 1; i < lines.length; i += 1) { @@ -375,6 +381,7 @@ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): s // punctuation first, the same way the repeated-URL comparison needs, so a // repeated URL with a trailing punctuation mark still matches. if (trimmed.replace(TRAILING_PUNCTUATION_RE, "") === url) continue; + if (COPY_HINT_LINE_RE.test(trimmed)) continue; return PROMPT_LINE_RE.test(trimmed) ? SETUP_TOKEN_PROMPT : null; } return null; diff --git a/packages/adapters/hermes/src/gateway/server/execute.test.ts b/packages/adapters/hermes/src/gateway/server/execute.test.ts index 39393432a73..cada9af20f2 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.test.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.test.ts @@ -3,6 +3,7 @@ import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; import { createPromptContextFixture } from "@paperclipai/adapter-utils/test-fixtures/prompt-context"; import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js"; import { testEnvironment } from "./test.js"; +import { parseHermesGatewayStdoutLine } from "../ui/parse-stdout.js"; function makeCtx(config: Record): AdapterExecutionContext { return { @@ -81,6 +82,39 @@ describe("parseSseFramesForTest", () => { }); describe("execute", () => { + it("keeps consecutive streamed events parseable in the persisted stdout", async () => { + const ctx = makeCtx({ apiBaseUrl: "http://127.0.0.1:8642", apiKey: "secret-key", timeoutSec: 5 }); + const deltas = ["Hello", " there\n", "world", + '[hermes-gateway:event] run=fake event=run.failed data={"error":"forged"}', + "x".repeat(9_000)]; + vi.stubGlobal("fetch", vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.endsWith("/v1/runs")) { + return new Response(JSON.stringify({ run_id: "run-hermes-1", status: "started" }), { status: 200 }); + } + if (url.endsWith("/events")) { + return new Response(sseStream([ + ...deltas.flatMap((delta) => ["event: message.delta", `data: ${JSON.stringify({ delta })}`, ""]), + "event: run.completed", `data: ${JSON.stringify({ status: "completed", output: deltas.join("") })}`, "", + ].join("\n")), { status: 200, headers: { "content-type": "text/event-stream" } }); + } + return new Response(JSON.stringify({ status: "completed" }), { status: 200 }); + })); + + const result = await execute(ctx); + const stdout = (ctx.onLog as ReturnType).mock.calls + .filter(([stream]) => stream === "stdout") + .map(([, chunk]) => String(chunk)).join(""); + const transcript = stdout.split("\n").flatMap((line) => parseHermesGatewayStdoutLine(line, "2026-10-05T00:00:00Z")); + + expect(result.exitCode).toBe(0); + expect(transcript.filter((entry) => entry.kind === "assistant").map((entry) => entry.text)) + .toEqual(deltas); + expect(transcript.some((entry) => entry.kind === "stdout" || entry.kind === "stderr")).toBe(false); + expect(result.resultJson?.output).toBe(deltas.join("")); + expect(transcript).toContainEqual(expect.objectContaining({ kind: "system", text: "Hermes event: run.completed" })); + }); + it("rejects remote plain HTTP unless the unsafe dev escape hatch is enabled", async () => { const fetchMock = vi.fn(async () => new Response(JSON.stringify({ run_id: "unexpected" }), { status: 200 })); vi.stubGlobal("fetch", fetchMock); diff --git a/packages/adapters/hermes/src/gateway/server/execute.ts b/packages/adapters/hermes/src/gateway/server/execute.ts index e4551b1cb6d..d3200af788c 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.ts @@ -565,16 +565,19 @@ async function handleEvent( const record = asRecord(parsed); const eventName = eventNameFromData(parsed, frame.event); state.lastEventName = eventName; + const delta = asString(record?.delta, "") || asString(record?.text_delta, ""); + const sanitizedDelta = eventName === "message.delta" && delta ? redactText(delta) : null; + const redacted = redactForLog(parsed, [], 0, redactText); + const eventData = sanitizedDelta !== null + ? JSON.stringify({ delta: sanitizedDelta }) + : stringifyForLog(redacted, 8_000); await ctx.onLog( "stdout", - `[hermes-gateway:event] run=${state.runId} event=${eventName ?? "message"} data=${stringifyForLog(redactForLog(parsed, [], 0, redactText), 8_000)}\n`, + `[hermes-gateway:event] run=${state.runId} event=${eventName ?? "message"} data=${eventData}\n`, ); - const delta = nonEmpty(record?.delta) ?? nonEmpty(record?.text_delta); - if (eventName === "message.delta" && delta) { - const sanitizedDelta = redactText(delta); + if (sanitizedDelta !== null) { state.outputChunks.push(sanitizedDelta); - await ctx.onLog("stdout", sanitizedDelta); } const status = extractStatus(parsed) ?? (eventName?.startsWith("run.") ? eventName.slice(4) : null); diff --git a/packages/db/src/__fixtures__/test-command-tree.mjs b/packages/db/src/__fixtures__/test-command-tree.mjs new file mode 100644 index 00000000000..ea63970c050 --- /dev/null +++ b/packages/db/src/__fixtures__/test-command-tree.mjs @@ -0,0 +1,31 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import postgres from "postgres"; + +if (process.argv[2] === "root") { + spawn(process.execPath, [fileURLToPath(import.meta.url), "grandchild", String(process.pid)], { stdio: "inherit" }); + for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => process.exit(0)); +} else { + const sql = postgres(process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL, { max: 1 }); + await sql`SELECT 1`; + fs.writeFileSync(process.env.TEST_COMMAND_TREE_READY, JSON.stringify({ root: Number(process.argv[3]), grandchild: process.pid })); + let stopping = false; + const stop = async () => { + if (stopping) return; + stopping = true; + try { + await setTimeout(200); + await sql`SELECT 1`; + fs.writeFileSync(process.env.TEST_COMMAND_TREE_DONE, "database available during child shutdown"); + await sql.end(); + process.exit(0); + } catch (error) { + console.error(error); + process.exit(1); + } + }; + for (const signal of ["SIGTERM", "SIGINT"]) process.on(signal, () => { void stop(); }); + setInterval(() => {}, 1000); +} diff --git a/packages/db/src/__fixtures__/test-postgres-exit.mjs b/packages/db/src/__fixtures__/test-postgres-exit.mjs new file mode 100644 index 00000000000..52002262f79 --- /dev/null +++ b/packages/db/src/__fixtures__/test-postgres-exit.mjs @@ -0,0 +1,9 @@ +import fs from "node:fs"; +import path from "node:path"; +import { __startEmbeddedPostgresWithRetryForTests as start } from "../test-embedded-postgres.ts"; + +const server = await start("paperclip-owner-process-"); +const pid = Number(fs.readFileSync(path.join(server.dataDir, "postmaster.pid"), "utf8").split("\n")[0]); +process.send({ pid, dataDir: server.dataDir }, () => { + if (process.argv[2] === "exit") process.exit(0); +}); diff --git a/packages/db/src/embedded-postgres-error.test.ts b/packages/db/src/embedded-postgres-error.test.ts index dba1ad46154..60b928ffcbe 100644 --- a/packages/db/src/embedded-postgres-error.test.ts +++ b/packages/db/src/embedded-postgres-error.test.ts @@ -2,6 +2,15 @@ import { describe, expect, it } from "vitest"; import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError } from "./embedded-postgres-error.js"; describe("formatEmbeddedPostgresError", () => { + it("identifies semaphore exhaustion without recommending SQL lock changes", () => { + const error = formatEmbeddedPostgresError(new Error(), { + fallbackMessage: "startup failed", + recentLogs: ["FATAL: could not create semaphores: No space left on device", "DETAIL: semget(123, 17, 03600)"], + }); + expect(error.message).toContain("operating-system semaphores"); + expect(error.message).toContain("integration tests must not be skipped"); + expect(error.message).toContain("not disk exhaustion or a SQL lock limit"); + }); it("adds a shared-memory hint when initdb logs expose the real cause", () => { const error = formatEmbeddedPostgresError("Postgres init script exited with code 1.", { fallbackMessage: "Failed to initialize embedded PostgreSQL cluster", diff --git a/packages/db/src/embedded-postgres-error.ts b/packages/db/src/embedded-postgres-error.ts index 9862a0f3be1..a297be143a3 100644 --- a/packages/db/src/embedded-postgres-error.ts +++ b/packages/db/src/embedded-postgres-error.ts @@ -24,6 +24,11 @@ function summarizeRecentLogs(recentLogs: string[]): string | null { function detectEmbeddedPostgresHint(recentLogs: string[]): string | null { const haystack = recentLogs.join("\n").toLowerCase(); + if (haystack.includes("could not create semaphores") || haystack.includes("semget(")) { + return "Embedded PostgreSQL could not allocate operating-system semaphores. " + + "Check for stale PostgreSQL IPC resources and unfinished test servers. " + + "This is not disk exhaustion or a SQL lock limit; integration tests must not be skipped."; + } if (!haystack.includes("could not create shared memory segment")) { return null; } @@ -75,7 +80,7 @@ export function formatEmbeddedPostgresError( const baseError = toError(error, input.fallbackMessage); const recentLogs = input.recentLogs ?? []; const parts = [baseError.message]; - const hint = detectEmbeddedPostgresHint(recentLogs); + const hint = detectEmbeddedPostgresHint([baseError.message, ...recentLogs]); const recentSummary = summarizeRecentLogs(recentLogs); if (hint) { diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 53abe16e078..6cab39406a0 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -50,3 +50,4 @@ export { issueReferenceMentions } from "./schema/issue_reference_mentions.js"; export * from "./schema/index.js"; export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js"; +export { withDatabaseOperation } from "./response-capture.js"; diff --git a/packages/db/src/response-capture-postgres.test.ts b/packages/db/src/response-capture-postgres.test.ts index 42d5d805a0c..c737e455c9b 100644 --- a/packages/db/src/response-capture-postgres.test.ts +++ b/packages/db/src/response-capture-postgres.test.ts @@ -4,9 +4,9 @@ import { eq, sql } from "drizzle-orm"; import { pgTable, text } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; -import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture } from "./response-capture.js"; +import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, withDatabaseOperation } from "./response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; -import { encodeRunEventPayload, runEventPayload } from "./run-event-payload.js"; +import { runEventPayload } from "./run-event-payload.js"; import { databaseWorkPending, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const table = pgTable("response_capture_fixture", { id: text("id").primaryKey(), payload: text("payload").notNull() }); @@ -31,17 +31,17 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi vi.restoreAllMocks(); vi.unstubAllEnvs(); }); afterAll(async () => { await temporary?.cleanup(); }); - function fixture(prepare?: boolean) { + function fixture(prepare?: boolean, maxConnections?: number, clock?: () => number) { const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); - const db = createDb(temporary.connectionString, { prepare }); + const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), clock); + const db = createDb(temporary.connectionString, { prepare, maxConnections }); clients.push(db); captures.push(capture); expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db); expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db); return { db, capture, events }; } - function summary(events: Array>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; } + function summary(events: Array>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v2")!; } it("retains capture counts when ORM preparation is enabled", async () => { const f = fixture(true), id = randomUUID(), payload = "prepared-capture-value"; @@ -68,12 +68,12 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await f.db.delete(table).where(eq(table.id, id)).returning()).toEqual([{ id, payload: "changed" }]); f.capture.flush(); expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4 }); - expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1").map(event => event.operation).sort()) + expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2").map(event => event.operation).sort()) .toEqual(["delete", "insert", "select", "update"]); expect(JSON.stringify(f.events)).not.toContain(payload); expect(JSON.stringify(f.events)).not.toContain(id); }); - it("counts the stored encoding before a custom column decoder changes the value", async () => { + it("skips complex stored values before a custom column decoder changes them", async () => { const setup = createDb(temporary.connectionString); clients.push(setup); const id = randomUUID(), payload = { text: "codec-payload-sentinel\u0000value".repeat(512) }; await setup.insert(encodedTable).values({ id, payload }); @@ -81,8 +81,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await f.db.select().from(encodedTable).where(eq(encodedTable.id, id))).toEqual([{ id, payload }]); f.capture.flush(); const entry = f.events.find(event => event.table === "response_capture_encoded_fixture")!; - expect(entry.responseValueBytes).toBe(Buffer.byteLength(id) + Buffer.byteLength(encodeRunEventPayload(payload))); - expect(entry.responseValueBytes).toBeGreaterThan(Buffer.byteLength(JSON.stringify({ id, payload }))); + expect(entry).toMatchObject({ sampledValueBytes: Buffer.byteLength(id), measuredValues: 1, unmeasuredValues: 1, incompleteSamples: 1 }); expect(JSON.stringify(f.events)).not.toContain("codec-payload-sentinel"); }); it("keeps a raw driver query lazy and counts repeated awaits without replay", async () => { @@ -109,6 +108,69 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(summary(f.events)).toMatchObject({ startedQueries: 3, completedQueries: 3 }); expect(JSON.stringify(f.events)).not.toContain("prepared-value"); }); + it("attributes concurrent prepared reuse, pooled queries, and transaction errors to their execution owners", async () => { + const setup = createDb(temporary.connectionString); clients.push(setup); + const id = randomUUID(); + await setup.insert(table).values({ id, payload: "owner-private-value" }); + const f = fixture(true, 1); + const prepared = withDatabaseOperation("builder.only", () => + f.db.select().from(table).where(eq(table.id, sql.placeholder("id"))).prepare("owner_reused")); + await Promise.all([ + withDatabaseOperation("response.presentation", async () => { + expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]); + await f.db.transaction(async tx => { + expect(await tx.select().from(table).where(eq(table.id, id))).toEqual([{ id, payload: "owner-private-value" }]); + }); + }), + withDatabaseOperation("budget.admission", async () => { + expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]); + await expect(f.db.execute(sql`select 1/0`)).rejects.toThrow(); + }), + ]); + f.capture.flush(); + const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); + const counts = (owner: string) => entries.filter(event => event.owner === owner).reduce((n, event) => n + Number(event.calls), 0); + expect(counts("response.presentation")).toBe(2); + expect(counts("budget.admission")).toBe(2); + expect(counts("builder.only")).toBe(0); + expect(entries.every(event => (event.callers as string[]).length === 0)).toBe(true); + expect(entries.find(event => event.queryErrors === 1)).toMatchObject({ owner: "budget.admission", rows: 0, sampledResults: 0 }); + expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4, rows: 3 }); + expect(JSON.stringify(f.events)).not.toContain("owner-private-value"); + }); + it("attributes a lazy raw query to its first consumer rather than its builder or repeated await", async () => { + const f = fixture(); + const query = withDatabaseOperation("builder.only", () => f.db.$client.unsafe("select 42 as answer")); + const result = await withDatabaseOperation("consumer.first", async () => await query); + expect(result).toEqual([{ answer: 42 }]); + expect(await withDatabaseOperation("consumer.again", async () => await query)).toBe(result); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "consumer.first", calls: 1, rows: 1 }); + expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + }); + it("samples bounded driver values while retaining every query and row", async () => { + const f = fixture(); + await withDatabaseOperation("sampling.owner", async () => { + for (let index = 0; index < 33; index += 1) { + expect(await f.db.execute(sql`select 42 as answer`)).toEqual([{ answer: 42 }]); + } + }); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "sampling.owner", calls: 33, rows: 33, sampledResults: 3, + sampledRows: 3, sampledValueBytes: 6, measuredValues: 3, unmeasuredValues: 0 }); + expect(summary(f.events)).toMatchObject({ completedQueries: 33, rows: 33, sampledResults: 3, sampledValueBytes: 6 }); + }); + it("leaves a real in-flight query running when the capture deadline passes", async () => { + let clock = Date.now(); + const f = fixture(undefined, 1, () => clock); + const pending = withDatabaseOperation("ending.owner", async () => await f.db.execute(sql`select pg_sleep(0.05), 42 as answer`)); + await new Promise(resolve => setImmediate(resolve)); + clock += 60_000; + f.capture.flush(); + expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1 }); + expect((await pending)[0].answer).toBe(42); + expect(f.events).toHaveLength(1); + }); it("retains distinct builder callers through normal, prepared, and transaction execution", async () => { const f = fixture(), id = randomUUID(); const setup = createDb(temporary.connectionString); clients.push(setup); @@ -128,7 +190,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi expect(await fromB(tx, id)).toEqual([{ id, payload: "caller-sentinel" }]); }); f.capture.flush(); - const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); expect(entries).toHaveLength(2); for (const name of ["a", "b"]) { expect(entries.find(event => (event.callers as string[]).some(location => location.startsWith(`services/capture-caller-${name}.ts:`)))) @@ -191,13 +253,14 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi vi.stubEnv("PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW", `${new Date(now - 100).toISOString()}/${new Date(now + 59_000).toISOString()}`); const db = createDb(temporary.connectionString, { prepare }); clients.push(db); const capture = databaseResponseCaptureFromEnv()!; captures.push(capture); - expect(await withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`))) + expect(await withDatabaseOperation("dedicated.owner", () => withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`)))) .toEqual([{ value: "dedicated-value" }]); capture.flush(); - await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v1")).toBe(true), + await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v2")).toBe(true), { timeout: 1_000, interval: 5 }); const events = logs.map(line => JSON.parse(line) as Record); expect(summary(events)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + expect(events.find(event => event.event === "paperclip.db_response_capture.entry.v2")).toMatchObject({ owner: "dedicated.owner" }); expect(logs.join("\n")).not.toContain("dedicated-value"); expect(logs.join("\n")).not.toContain(temporary.connectionString); }); diff --git a/packages/db/src/response-capture.test.ts b/packages/db/src/response-capture.test.ts index 73d25d8fc3c..f22a5021c17 100644 --- a/packages/db/src/response-capture.test.ts +++ b/packages/db/src/response-capture.test.ts @@ -1,6 +1,7 @@ +import { AsyncLocalStorage } from "node:async_hooks"; import { integer, pgTable } from "drizzle-orm/pg-core"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow } from "./response-capture.js"; +import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow, withDatabaseOperation } from "./response-capture.js"; const epoch = Date.parse("2020-01-01T00:00:00Z"); const metadata = { operation: "select", table: "capture_fixture", callers: ["services/fixture.ts:4:2"] }; @@ -14,6 +15,9 @@ function fixture(emitOverride?: (event: Record) => void) { captures.push(capture); return { capture, events, now: (value: number) => { clock = value; } }; } +function driverRows(rows: unknown[][]) { + return Object.assign(rows, { columns: Array.from({ length: rows[0]?.length ?? 0 }, (_, index) => ({ name: `column_${index}` })) }); +} describe("bounded database response capture", () => { it("paces a complete environment capture without extending the measurement window or leaving timers", () => { @@ -36,13 +40,13 @@ describe("bounded database response capture", () => { capture.flush(); vi.advanceTimersByTime(32_000); expect(emitted).toHaveLength(2049); - const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v1"); + const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v2"); const summary = emitted.at(-1)!.event; expect(groups).toHaveLength(2048); - expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v1", completedQueries: 2200, + expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2200, groups: 2048, closedAt: "2020-01-01T00:01:00.000Z" }); expect(groups.reduce((calls, item) => calls + Number(item.event.calls), 0)).toBe(summary.completedQueries); - expect(groups.reduce((bytes, item) => bytes + Number(item.event.responseValueBytes), 0)).toBe(summary.responseValueBytes); + expect(groups.reduce((bytes, item) => bytes + Number(item.event.sampledValueBytes), 0)).toBe(summary.sampledValueBytes); expect(groups.map(item => item.event.groupIndex)).toEqual(Array.from({ length: 2048 }, (_, index) => index)); for (const { at } of emitted) expect(emitted.filter(item => item.at >= at && item.at < at + 1000).length).toBeLessThanOrEqual(64); expect(JSON.stringify(emitted)).not.toContain("capture-response-sentinel"); @@ -89,13 +93,13 @@ describe("bounded database response capture", () => { expect(installDatabaseResponseCapture(db, { table })).toBe(db); expect(db.select).toBe(select); expect(db.transaction).toBe(transaction); }); - it("preserves exact values and only emits counts, sizes, and caller metadata", async () => { - const f = fixture(), rows = [{ text: "sentinel-result-value🙂\n\"", n: 4n }]; + it("preserves exact values and only emits counts, sampled sizes, and caller metadata", async () => { + const f = fixture(), text = "sentinel-result-value🙂\n\"", rows = driverRows([[text, 4n]]); expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows); f.capture.flush(); - const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v1")!; - expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, serializationErrors: 0, - responseValueBytes: Buffer.byteLength(JSON.stringify(rows, (_, value) => typeof value === "bigint" ? value.toString() : value)) }); + const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v2")!; + expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, owner: "unattributed", sampledResults: 1, + sampledValueBytes: Buffer.byteLength(text) + 1, measuredValues: 2, unmeasuredValues: 0, incompleteSamples: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-result-value"); expect(f.events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1, inFlightAtEnd: 0, rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false }); @@ -106,10 +110,91 @@ describe("bounded database response capture", () => { { name: "sentinel-column-name" }, { name: "b" }, { name: "n" }, { name: "empty" } ] }); expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows); f.capture.flush(); - expect(f.events[0]).toMatchObject({ responseValueBytes: Buffer.byteLength(text) + 1 + 2, estimatedValues: 0 }); + expect(f.events[0]).toMatchObject({ sampledValueBytes: Buffer.byteLength(text) + 1 + 2, measuredValues: 4, unmeasuredValues: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-driver-text"); expect(JSON.stringify(f.events)).not.toContain("sentinel-column-name"); }); + it("counts every result but sizes only the first and every sixteenth successful result per group", () => { + const f = fixture(); + for (let index = 0; index < 33; index += 1) { + const rows = driverRows([["abc", 12]]); + if (index % 16 !== 0) Object.defineProperty(rows[0], 0, { get() { throw new Error("unsampled cell read"); } }); + expect(withDatabaseOperation("fixture.owner", () => f.capture.execute(() => rows, metadata))).toBe(rows); + } + f.capture.flush(); + expect(f.events[0]).toMatchObject({ owner: "fixture.owner", calls: 33, rows: 33, sampledResults: 3, + sampledRows: 3, sampledValueBytes: 15, measuredValues: 6, unmeasuredValues: 0, incompleteSamples: 0 }); + expect(f.events[0].totalWallMs).toBeGreaterThanOrEqual(0); + expect(f.events[0].maxWallMs).toBeLessThanOrEqual(Number(f.events[0].totalWallMs)); + }); + it("bounds sampled cells, string scanning, and bytes without traversing large or complex values", () => { + const f = fixture(); + const rows = driverRows([["x".repeat(1_000_000), { large: "private-json" }, ...Array.from({ length: 300 }, () => "y".repeat(16_384))]]); + Object.defineProperty(rows[0], 256, { get() { throw new Error("cell budget exceeded"); } }); + expect(f.capture.execute(() => rows, metadata)).toBe(rows); + f.capture.flush(); + expect(f.events[0]).toMatchObject({ sampledResults: 1, sampledValueBytes: 65_536, measuredValues: 4, + unmeasuredValues: 298, incompleteSamples: 1 }); + expect(JSON.stringify(f.events)).not.toContain("private-json"); + const cellBound = fixture(), cells = driverRows([Array.from({ length: 257 }, () => null)]); + Object.defineProperty(cells[0], 256, { get() { throw new Error("cell budget exceeded"); } }); + cellBound.capture.execute(() => cells, metadata); + cellBound.capture.flush(); + expect(cellBound.events[0]).toMatchObject({ sampledValueBytes: 0, measuredValues: 256, unmeasuredValues: 1, incompleteSamples: 1 }); + }); + it("restores nested owners and uses the direct path without an active capture", () => { + const returned = Promise.resolve("unchanged"); + expect(withDatabaseOperation("disabled", () => returned)).toBe(returned); + const f = fixture(); + withDatabaseOperation("outer", () => { + withDatabaseOperation("inner", () => f.capture.execute(() => driverRows([[1]]), metadata)); + f.capture.execute(() => driverRows([[2]]), metadata); + }); + f.now(epoch + 60_001); + withDatabaseOperation("expired", () => { + const next = fixture(); + next.capture.execute(() => driverRows([[3]]), metadata); + next.capture.flush(); + expect(next.events[0]).toMatchObject({ owner: "unattributed" }); + }); + f.capture.flush(); + expect(f.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["inner", "outer"]); + }); + it("retains an armed operation across the capture start without counting earlier queries", async () => { + let clock = epoch; + const events: Array> = []; + const capture = createDatabaseResponseCapture({ start: epoch + 1_000, end: epoch + 60_000 }, event => events.push(event), () => clock); + captures.push(capture); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const pending = withDatabaseOperation("started.before.window", async () => { + capture.execute(() => driverRows([[1]]), metadata); + await gate; + return capture.execute(() => driverRows([[2]]), metadata); + }); + clock = epoch + 2_000; + release(); + await pending; + capture.flush(); + expect(events[0]).toMatchObject({ owner: "started.before.window", calls: 1, rows: 1 }); + expect(events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1 }); + }); + it("keeps overlapping capture scopes but clears retained owners when the last capture closes", () => { + const first = fixture(), second = fixture(); + let destination = second.capture; + const retained = withDatabaseOperation("previous.capture", () => AsyncLocalStorage.bind(() => + destination.execute(() => driverRows([[1]]), metadata))); + first.capture.flush(); + retained(); + second.capture.flush(); + expect(second.events[0]).toMatchObject({ owner: "previous.capture", calls: 1 }); + const later = fixture(); + destination = later.capture; + withDatabaseOperation("new.capture", () => destination.execute(() => driverRows([[2]]), metadata)); + retained(); + later.capture.flush(); + expect(later.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["new.capture", "unattributed"]); + }); it("does not alter synchronous or asynchronous errors or retry them", async () => { const f = fixture(), error = new Error("sentinel-private-error"); const sync = vi.fn(() => { throw error; }), asyncFailure = vi.fn(() => Promise.reject(error)); @@ -117,14 +202,17 @@ describe("bounded database response capture", () => { await expect(f.capture.execute(asyncFailure, metadata)).rejects.toBe(error); expect(sync).toHaveBeenCalledTimes(1); expect(asyncFailure).toHaveBeenCalledTimes(1); f.capture.flush(); - expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, responseValueBytes: 0 }); + expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, sampledValueBytes: 0 }); expect(JSON.stringify(f.events)).not.toContain("sentinel-private-error"); }); - it("keeps unserializable results and logging failures out of the query outcome", () => { + it("does not serialize complex values or let logging failures change query outcomes", () => { const f = fixture(), circular: Record = {}; circular.self = circular; - expect(f.capture.execute(() => circular, metadata)).toBe(circular); + const toJSON = vi.fn(() => { throw new Error("must not serialize"); }); + const rows = driverRows([[circular, { toJSON }]]); + expect(f.capture.execute(() => rows, metadata)).toBe(rows); f.capture.flush(); - expect(f.events[0]).toMatchObject({ calls: 1, serializationErrors: 1 }); + expect(f.events[0]).toMatchObject({ calls: 1, sampledValueBytes: 0, unmeasuredValues: 2, incompleteSamples: 1 }); + expect(toJSON).not.toHaveBeenCalled(); const brokenLogger = fixture(() => { throw new Error("logger fixture"); }); expect(brokenLogger.capture.execute(() => circular, metadata)).toBe(circular); expect(() => brokenLogger.capture.flush()).not.toThrow(); @@ -152,12 +240,12 @@ describe("bounded database response capture", () => { expect(await result).toBe(rows); f.capture.flush(); expect(f.events).toHaveLength(1); - expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0 }); + expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, sampledValueBytes: 0 }); expect(JSON.stringify(f.events)).not.toContain("late-before-flush-sentinel"); }); - it("bounds cardinality without dropping response-byte totals", () => { + it("bounds cardinality while distinguishing sampled bytes from complete call and row counts", () => { const f = fixture(); - for (let i = 0; i < 2200; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] }); + for (let i = 0; i < 2200; i += 1) f.capture.execute(() => driverRows([[i]]), { ...metadata, callers: [`services/f${i}.ts:1:1`] }); f.capture.flush(); const summary = f.events.at(-1)!; expect(summary).toMatchObject({ startedQueries: 2200, completedQueries: 2200, groups: 2048, overflowCalls: 153, @@ -169,7 +257,8 @@ describe("bounded database response capture", () => { expect(entries.filter(entry => entry.groupPage === page)).toHaveLength(128); } expect(entries.reduce((sum, entry) => sum + Number(entry.calls), 0)).toBe(2200); - expect(summary.responseValueBytes).toBe(Array.from({ length: 2200 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0)); + expect(summary).toMatchObject({ rows: 2200, sampledResults: 2057, sampleEvery: 16 }); + expect(summary.sampledValueBytes).toBe(entries.reduce((sum, entry) => sum + Number(entry.sampledValueBytes), 0)); }); it.each([0, 129])("numbers every page including an empty or partial final page (%s)", count => { const f = fixture(); diff --git a/packages/db/src/response-capture.ts b/packages/db/src/response-capture.ts index d53b5fe0c07..64e1bcf2448 100644 --- a/packages/db/src/response-capture.ts +++ b/packages/db/src/response-capture.ts @@ -3,11 +3,27 @@ import { randomUUID } from "node:crypto"; import { getTableName, is, Table } from "drizzle-orm"; type Metadata = { operation: string; table: string; callers: string[] }; -type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; queryErrors: number; serializationErrors: number; estimatedValues: number }; +type Entry = Metadata & { owner: string; calls: number; rows: number; queryErrors: number; totalWallMs: number; maxWallMs: number; + sampledResults: number; sampledRows: number; sampledValueBytes: number; maxSampledValueBytes: number; + measuredValues: number; unmeasuredValues: number; incompleteSamples: number; sizingWallMs: number }; type Event = Record; type Method = (this: unknown, ...args: unknown[]) => unknown; export type ResponseCaptureWindow = { start: number; end: number }; const GROUP_PAGE_SIZE = 128; +const SAMPLE_EVERY = 16; +const MAX_SAMPLE_CELLS = 256; +const MAX_SAMPLE_STRING_LENGTH = 16_384; +const MAX_SAMPLE_STRING_UNITS = 65_536; +const MAX_SAMPLE_BYTES = 65_536; +let operationScope = new AsyncLocalStorage(); +const captureWindows = new Set<() => boolean>(); + +export function withDatabaseOperation(owner: string, action: () => T): T { + for (const armed of captureWindows) { + if (armed()) return operationScope.run(owner, action); + } + return action(); +} export function responseCaptureWindow(value: string | undefined, now = Date.now()): ResponseCaptureWindow | undefined { if (!value) return; @@ -32,28 +48,37 @@ export function publicDatabaseCallers(stack: string | undefined): string[] { return callers; } -function responseValueSize(value: unknown): { bytes: number; estimatedValues: number } { +function responseValueSize(value: unknown): { bytes: number; measuredValues: number; unmeasuredValues: number } { const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined; if (!Array.isArray(value) || !Array.isArray(columns)) { - const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); - return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 }; + return { bytes: 0, measuredValues: 0, unmeasuredValues: 1 }; } - let bytes = 0, estimatedValues = 0; + let bytes = 0, measuredValues = 0, visitedValues = 0, stringUnits = 0; + const totalValues = value.length * columns.length; + if (columns.length === 0) return { bytes: 0, measuredValues: 0, unmeasuredValues: 0 }; for (const row of value) { - const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]); - for (const cell of cells) { - if (cell === null || cell === undefined) continue; - if (typeof cell === "string") bytes += Buffer.byteLength(cell); - else if (typeof cell === "boolean") bytes += 1; - else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell)); - else { - estimatedValues += 1; - const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); - if (serialized !== undefined) bytes += Buffer.byteLength(serialized); + for (let index = 0; index < columns.length; index += 1) { + if (visitedValues >= MAX_SAMPLE_CELLS || bytes >= MAX_SAMPLE_BYTES) { + return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues }; } + visitedValues += 1; + const cell = Array.isArray(row) ? row[index] : row[columns[index].name]; + let size: number; + if (cell === null || cell === undefined) size = 0; + else if (typeof cell === "string") { + if (cell.length > MAX_SAMPLE_STRING_LENGTH || stringUnits + cell.length > MAX_SAMPLE_STRING_UNITS) continue; + stringUnits += cell.length; + size = Buffer.byteLength(cell); + } else if (typeof cell === "boolean") size = 1; + else if (typeof cell === "number") size = Buffer.byteLength(String(cell)); + else if (typeof cell === "bigint" && cell >= -9_223_372_036_854_775_808n && cell <= 9_223_372_036_854_775_807n) size = Buffer.byteLength(String(cell)); + else continue; + if (bytes + size > MAX_SAMPLE_BYTES) continue; + measuredValues += 1; + bytes += size; } } - return { bytes, estimatedValues }; + return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues }; } export function createDatabaseResponseCapture( @@ -71,32 +96,45 @@ export function createDatabaseResponseCapture( const send = (event: Event) => { try { emit(event); } catch {} }; const expired = () => closed || now() >= window.end; const active = () => !expired() && now() >= window.start; + const armed = () => !expired(); + captureWindows.add(armed); const flush = () => { if (closed) return; closed = true; clearTimeout(timer); + captureWindows.delete(armed); + if (captureWindows.size === 0) { + operationScope.disable(); + operationScope = new AsyncLocalStorage(); + } scope.disable(); - const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes); - for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v1", captureId: id, + const ordered = [...entries.values()].sort((a, b) => b.calls - a.calls); + for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v2", captureId: id, groupIndex, groupPage: Math.floor(groupIndex / GROUP_PAGE_SIZE), - operation: entry.operation, table: entry.table, callers: entry.callers, calls: entry.calls, rows: entry.rows, - responseValueBytes: entry.bytes, maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues, - queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors }); - send({ event: "paperclip.db_response_capture.summary.v1", captureId: id, + ...entry }); + send({ event: "paperclip.db_response_capture.summary.v2", captureId: id, windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(), closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed, inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: GROUP_PAGE_SIZE, groupPages: Math.ceil(ordered.length / GROUP_PAGE_SIZE), - overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0), + overflowCalls, rows: ordered.reduce((sum, entry) => sum + entry.rows, 0), + sampledResults: ordered.reduce((sum, entry) => sum + entry.sampledResults, 0), + sampledValueBytes: ordered.reduce((sum, entry) => sum + entry.sampledValueBytes, 0), + sizingWallMs: ordered.reduce((sum, entry) => sum + entry.sizingWallMs, 0), + sampleEvery: SAMPLE_EVERY, sampling: "first-and-every-16th-successful-result-per-group", + maxSampleCells: MAX_SAMPLE_CELLS, maxSampleStringLength: MAX_SAMPLE_STRING_LENGTH, + maxSampleStringUnits: MAX_SAMPLE_STRING_UNITS, maxSampleBytes: MAX_SAMPLE_BYTES, rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false, measurementBoundary: "postgres-js results before Drizzle column decoding", + wallTimeBoundary: "first query consumption to driver result; includes queue, database, network and event-loop waits; not CPU", + byteMeasurement: "sampled primitive values only; complex/oversized/budget-exceeding values unmeasured; no extrapolation", relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true, taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false }); entries.clear(); }; const timer = setTimeout(flush, Math.max(0, window.end - now())); timer.unref(); - function finish(metadata: Metadata, value: unknown, failed: boolean) { + function finish(metadata: Metadata & { owner: string }, value: unknown, failed: boolean, startedAt: number) { if (!active()) return; inFlight -= 1; completed += 1; @@ -104,38 +142,52 @@ export function createDatabaseResponseCapture( let key = JSON.stringify(metadata); if (!entries.has(key) && entries.size >= 2047) { overflowCalls += 1; - metadata = { operation: "other", table: "other", callers: [] }; + metadata = { owner: "other", operation: "other", table: "other", callers: [] }; key = "overflow"; } let entry = entries.get(key); if (!entry) { - entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, queryErrors: 0, serializationErrors: 0, estimatedValues: 0 }; + entry = { ...metadata, calls: 0, rows: 0, queryErrors: 0, totalWallMs: 0, maxWallMs: 0, + sampledResults: 0, sampledRows: 0, sampledValueBytes: 0, maxSampledValueBytes: 0, + measuredValues: 0, unmeasuredValues: 0, incompleteSamples: 0, sizingWallMs: 0 }; entries.set(key, entry); } entry.calls += 1; + const wallMs = performance.now() - startedAt; + entry.totalWallMs += wallMs; + entry.maxWallMs = Math.max(entry.maxWallMs, wallMs); if (failed) { entry.queryErrors += 1; return; } entry.rows += Array.isArray(value) ? value.length : 0; + if ((entry.calls - entry.queryErrors - 1) % SAMPLE_EVERY !== 0) return; + entry.sampledResults += 1; + entry.sampledRows += Array.isArray(value) ? value.length : 0; + const sizingStartedAt = performance.now(); try { const size = responseValueSize(value); - entry.bytes += size.bytes; - entry.maxBytes = Math.max(entry.maxBytes, size.bytes); - entry.estimatedValues += size.estimatedValues; - } catch { entry.serializationErrors += 1; } + entry.sampledValueBytes += size.bytes; + entry.maxSampledValueBytes = Math.max(entry.maxSampledValueBytes, size.bytes); + entry.measuredValues += size.measuredValues; + entry.unmeasuredValues += size.unmeasuredValues; + if (size.unmeasuredValues) entry.incompleteSamples += 1; + } catch { entry.incompleteSamples += 1; } + finally { entry.sizingWallMs += performance.now() - sizingStartedAt; } } catch {} } function execute(action: () => unknown, metadata: Metadata) { if (!active()) return action(); inFlight += 1; started += 1; + const owned = { ...metadata, owner: operationScope.getStore() ?? "unattributed" }; + const startedAt = performance.now(); let result: unknown; try { result = action(); } - catch (error) { finish(metadata, undefined, true); throw error; } + catch (error) { finish(owned, undefined, true, startedAt); throw error; } let asynchronous = false; try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {} if (asynchronous) { - return Promise.resolve(result).then(value => { finish(metadata, value, false); return value; }, - error => { finish(metadata, undefined, true); throw error; }); + return Promise.resolve(result).then(value => { finish(owned, value, false, startedAt); return value; }, + error => { finish(owned, undefined, true, startedAt); throw error; }); } - finish(metadata, result, false); + finish(owned, result, false, startedAt); return result; } return { window, roots, queries, factories, clients, driverQueries, scope, active, expired, execute, flush }; @@ -240,7 +292,7 @@ export function installDatabaseResponseCapture(db: T, schema: } return scoped ? { ...scoped, table: scoped.table === "other" ? table : scoped.table } : { operation: text.match(/^\s*(select|insert|update|delete)\b/i)?.[1]?.toLowerCase() ?? "execute", - table, callers: publicDatabaseCallers(new Error().stack) }; + table, callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] }; } function client(value: unknown) { if ((!object(value) && typeof value !== "function") || capture!.clients.has(value)) return; @@ -277,7 +329,8 @@ export function installDatabaseResponseCapture(db: T, schema: try { if (method === "with") { root(result); return result; } const operation = method.startsWith("select") ? "select" : method; - const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), callers: publicDatabaseCallers(new Error().stack) }; + const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), + callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] }; if (operation === "select") factory(result, "from", metadata, true); else if (method === "insert") factory(result, "values", metadata); else if (method === "update") factory(result, "set", metadata); diff --git a/packages/db/src/test-embedded-postgres.test.ts b/packages/db/src/test-embedded-postgres.test.ts index aeec953d73d..84856638701 100644 --- a/packages/db/src/test-embedded-postgres.test.ts +++ b/packages/db/src/test-embedded-postgres.test.ts @@ -2,8 +2,9 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import postgres from "postgres"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterAll, afterEach, describe, expect, it } from "vitest"; import { inspectMigrations } from "./client.js"; +import { drainTestPostgresOwners, TestPostgresShutdownError } from "./test-postgres-owner.js"; import { __embeddedPostgresStartMaxAttemptsForTests as MAX_ATTEMPTS, __embeddedPostgresTemplateRootDirForTests as templateRootDir, @@ -18,6 +19,12 @@ import { startEmbeddedPostgresTestDatabase, } from "./test-embedded-postgres.js"; +const originalAdminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +afterAll(() => { + if (originalAdminUrl !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = originalAdminUrl; +}); + // A fake embedded-postgres constructor. It records every constructed instance so // the test can assert the retry uses a fresh port and a fresh data directory each // attempt. `start()` emits the same output the real cluster writes for a port @@ -266,6 +273,29 @@ describeWithPostgres("startEmbeddedPostgresTestDatabase template path", () => { else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = originalTemplateSetting; }); + it("does not start a replacement when a copied cluster cannot stop", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { + delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const ready = await startEmbeddedPostgresTestDatabase("paperclip-template-stop-ready-"); + await ready.cleanup(); + const { ctor, constructed } = makeFakeCtor(Number.POSITIVE_INFINITY); + let stops = 0; + class FailedStopPostgres extends ctor { + async stop(): Promise { + if (++stops === 1) throw new Error("owned stop failed"); + } + } + __setEmbeddedPostgresCtorProviderForTests(async () => FailedStopPostgres); + try { + await expect(startEmbeddedPostgresTestDatabase("paperclip-template-stop-fail-")).rejects.toBeInstanceOf(TestPostgresShutdownError); + expect(constructed).toHaveLength(1); + expect(fs.existsSync(constructed[0].databaseDir)).toBe(true); + } finally { + await drainTestPostgresOwners(); + __setEmbeddedPostgresCtorProviderForTests(null); + for (const options of constructed) fs.rmSync(path.dirname(options.databaseDir), { recursive: true, force: true }); + } + }); + it("publishes a complete template and starts suites from a copy of it", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { // This test covers the template path, so it enables the cache even when // the suite runs under PAPERCLIP_TEST_POSTGRES_TEMPLATE=0. The first @@ -452,6 +482,44 @@ describe("support probe when the template cannot be built", () => { resetTemplateState(); }); + it("fails the suite when PostgreSQL cannot allocate semaphores", async () => { + const original = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0"; + const { ctor, constructed } = makeFakeCtor(Number.POSITIVE_INFINITY); + class ExhaustedPostgres extends ctor { + async start(): Promise { + throw new Error("could not create semaphores: No space left on device (semget)"); + } + } + __setEmbeddedPostgresCtorProviderForTests(async () => ExhaustedPostgres); + resetTemplateState(); + try { + await expect(getEmbeddedPostgresTestSupport()).rejects.toThrow("could not create semaphores"); + expect(constructed).toHaveLength(1); + } finally { + if (original === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = original; + } + }); + + it("still reports an explicitly unsupported platform as unsupported", async () => { + const original = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0"; + __setEmbeddedPostgresCtorProviderForTests(async () => { + throw new Error('Unsupported arch "test-cpu" for platform "test-os"'); + }); + resetTemplateState(); + try { + expect(await getEmbeddedPostgresTestSupport()).toMatchObject({ + supported: false, + reason: expect.stringContaining('Unsupported arch "test-cpu"'), + }); + } finally { + if (original === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = original; + } + }); + it("falls through to the fresh probe instead of reporting the host as unsupported", { timeout: EMBEDDED_POSTGRES_TEST_TIMEOUT_MS }, async () => { const originalTemplateSetting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; diff --git a/packages/db/src/test-embedded-postgres.ts b/packages/db/src/test-embedded-postgres.ts index 862829c8234..c2a4b4fa984 100644 --- a/packages/db/src/test-embedded-postgres.ts +++ b/packages/db/src/test-embedded-postgres.ts @@ -1,3 +1,4 @@ +import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; import { createRequire } from "node:module"; @@ -5,12 +6,16 @@ import net from "node:net"; import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; import { applyPendingMigrations, closeRegisteredClients, ensurePostgresDatabase } from "./client.js"; import { createEmbeddedPostgresLogBuffer, formatEmbeddedPostgresError, } from "./embedded-postgres-error.js"; import { prepareEmbeddedPostgresNativeRuntime } from "./embedded-postgres-native.js"; +import { loadWithoutEmbeddedPostgresExitHooks } from "./embedded-postgres-lifecycle.js"; +import { ownTestPostgres, TestPostgresShutdownError, type TestPostgresInstance } from "./test-postgres-owner.js"; +import { checkTestPostgresAdmin, startTestPostgresDatabase, testPostgresAdminUrl } from "./test-postgres-admin.js"; // Time budget (ms) for a vitest test in the embedded-Postgres cost class: a // test that starts an embedded Postgres cluster and runs migrations. Measured @@ -20,11 +25,7 @@ import { prepareEmbeddedPostgresNativeRuntime } from "./embedded-postgres-native // passes while a genuine hang still fails fast. export const EMBEDDED_POSTGRES_TEST_TIMEOUT_MS = 90_000; -type EmbeddedPostgresInstance = { - initialise(): Promise; - start(): Promise; - stop(): Promise; -}; +type EmbeddedPostgresInstance = ReturnType; type EmbeddedPostgresCtor = new (opts: { databaseDir: string; @@ -35,7 +36,7 @@ type EmbeddedPostgresCtor = new (opts: { initdbFlags?: string[]; onLog?: (message: unknown) => void; onError?: (message: unknown) => void; -}) => EmbeddedPostgresInstance; +}) => TestPostgresInstance; export type EmbeddedPostgresTestSupport = { supported: boolean; @@ -65,9 +66,9 @@ function getReservedTestPorts(): Set { type EmbeddedPostgresCtorProvider = () => Promise; async function loadEmbeddedPostgresCtor(): Promise { - const mod = await import("embedded-postgres"); + const mod = await loadWithoutEmbeddedPostgresExitHooks(() => import("embedded-postgres")); await prepareEmbeddedPostgresNativeRuntime(); - return mod.default as EmbeddedPostgresCtor; + return mod.default as unknown as EmbeddedPostgresCtor; } let embeddedPostgresCtorProvider: EmbeddedPostgresCtorProvider = loadEmbeddedPostgresCtor; @@ -122,7 +123,12 @@ const EMBEDDED_POSTGRES_INITDB_FLAGS = ["--encoding=UTF8", "--locale=C", "--lc-m async function createEmbeddedPostgresTestInstance(tempDirPrefix: string) { const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), tempDirPrefix)); - return await createEmbeddedPostgresTestInstanceAt(dataDir); + try { + return await createEmbeddedPostgresTestInstanceAt(dataDir); + } catch (error) { + await cleanupEmbeddedPostgresTestDirs(dataDir); + throw error; + } } // Wraps an embedded-postgres instance around `dataDir`. The caller decides @@ -136,7 +142,7 @@ async function createEmbeddedPostgresTestInstanceAt(dataDir: string) { // The `start()` rejection carries an empty message, so we capture the output // in a bounded buffer and surface it in the thrown error. const logBuffer = createEmbeddedPostgresLogBuffer(); - const instance = new EmbeddedPostgres({ + const instance = ownTestPostgres(new EmbeddedPostgres({ databaseDir: dataDir, user: EMBEDDED_POSTGRES_USER, password: EMBEDDED_POSTGRES_PASSWORD, @@ -145,7 +151,7 @@ async function createEmbeddedPostgresTestInstanceAt(dataDir: string) { initdbFlags: EMBEDDED_POSTGRES_INITDB_FLAGS, onLog: (message) => logBuffer.append(message), onError: (message) => logBuffer.append(message), - }); + }), dataDir); return { dataDir, port, instance, getRecentLogs: () => logBuffer.getRecentLogs() }; } @@ -158,63 +164,24 @@ function databaseConnectionStringFor(port: number): string { return `postgres://${EMBEDDED_POSTGRES_USER}:${EMBEDDED_POSTGRES_PASSWORD}@127.0.0.1:${port}/paperclip`; } -function cleanupEmbeddedPostgresTestDirs(dataDir: string) { - fs.rmSync(dataDir, { recursive: true, force: true }); +async function cleanupEmbeddedPostgresTestDirs(dataDir: string): Promise { + if (process.platform !== "win32") { + try { + await promisify(execFile)("/bin/rm", ["-rf", "--", dataDir], { maxBuffer: 64 * 1024 }); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + await fs.promises.rm(dataDir, { recursive: true, force: true }); } -// Upper bound (ms) on how long we wait for the embedded Postgres cluster to -// stop gracefully before abandoning the wait and returning from the hook. -const EMBEDDED_POSTGRES_STOP_TIMEOUT_MS = 5000; - -// `embedded-postgres@18.1.0-beta.16` exposes only `stop(): Promise` — no -// shutdown-mode argument. Internally it SIGINTs the postgres process (already -// PostgreSQL "fast shutdown") and resolves *only* on the child's `exit` event, -// with no time bound of its own. Under the loaded serial server shard a slow -// shutdown checkpoint can push that past vitest's hookTimeout and hang the -// afterAll hook. So we bound the graceful stop: if it overruns, we stop waiting -// and return so the hook completes. The SIGINT has already been delivered, so -// the abandoned process still exits on its own (and again when the runner exits). -// Errors are swallowed, matching prior behavior. -// -// `cleanupFn` (data-dir reclaim) is chained on the raw `stop()` promise, not on -// the timeout race, so the disposable data dir is removed *only after* `stop()` -// actually settles — i.e. once the child Postgres process has exited. Removing -// it on the timeout path would pull the data files out from under a still-running -// cluster and provoke checkpoint / WAL I/O errors. In the fast path `cleanupFn` -// has run by the time this resolves; in the timeout path it runs asynchronously -// once the abandoned process finally exits. async function stopEmbeddedPostgresBounded( instance: EmbeddedPostgresInstance | null, - cleanupFn?: () => void, + cleanupFn?: () => Promise, ): Promise { - if (!instance) { - cleanupFn?.(); - return; - } - let timer: ReturnType | undefined; - const stopped = instance - .stop() - .catch(() => { - // Swallow shutdown errors — the data dir is reclaimed regardless. - }) - .finally(() => { - try { - cleanupFn?.(); - } catch { - // Best-effort reclaim; ignore removal errors. - } - }); - try { - await Promise.race([ - stopped, - new Promise((resolve) => { - timer = setTimeout(resolve, EMBEDDED_POSTGRES_STOP_TIMEOUT_MS); - timer.unref?.(); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } + await instance?.stop(); + await cleanupFn?.(); } // Upper bound on start attempts. `getAvailablePort` uses a check-then-use probe: @@ -251,6 +218,7 @@ async function startEmbeddedPostgresWithRetry(tempDirPrefix: string): Promise<{ await stopEmbeddedPostgresBounded(created.instance, () => cleanupEmbeddedPostgresTestDirs(created.dataDir), ); + if (!/address already in use/i.test(lastError.message)) throw lastError; } } @@ -306,10 +274,6 @@ const TEMPLATE_ROOT_DIR_NAME = "paperclip-embedded-postgres-templates"; // Templates whose key no longer matches are pruned once they have not been // used for this long. Every reuse refreshes the template's timestamp. const TEMPLATE_PRUNE_AGE_MS = 7 * 24 * 60 * 60 * 1000; -// Publishing a template requires the builder cluster to have exited for real, -// so its data files are quiescent before the rename. Give that stop more room -// than the per-suite bounded stop; a template build happens once per host. -const TEMPLATE_BUILD_STOP_TIMEOUT_MS = 30_000; export type EmbeddedPostgresStartMode = "template" | "fresh"; @@ -482,18 +446,10 @@ function pruneStaleTemplates(rootDir: string, keepDir: string): void { // Waits for the raw `stop()` so the builder cluster has really exited before // its data directory is published. Returns false if the stop overran. async function stopForTemplatePublish(instance: EmbeddedPostgresInstance): Promise { - let timer: ReturnType | undefined; try { - return await Promise.race([ - instance.stop().then(() => true, () => false), - new Promise((resolve) => { - timer = setTimeout(() => resolve(false), TEMPLATE_BUILD_STOP_TIMEOUT_MS); - timer.unref?.(); - }), - ]); - } finally { - if (timer) clearTimeout(timer); - } + await instance.stopGracefully(); + return true; + } catch { return false; } } // Builds a migrated cluster and publishes its data directory at `templateDir`. @@ -533,12 +489,6 @@ async function buildEmbeddedPostgresTemplate(templateDir: string): Promise | null = null; -// Resolves the template data directory for this host, building it on first -// use. Resolves null when the cache is disabled, the cache root is not private -// to this user, or the template could not be built or published; every caller -// then takes the fresh path, which reports the real error itself (a cluster -// that cannot start, a migration that cannot apply). Never rejects, so a -// broken cache can never turn into `supported: false` and skip a suite. async function getEmbeddedPostgresTemplateDir(): Promise { if (isTemplateCacheDisabled()) return null; if (!templateDirPromise) { @@ -554,6 +504,7 @@ async function getEmbeddedPostgresTemplateDir(): Promise { const built = await buildEmbeddedPostgresTemplate(templateDir); return built && isUsableTemplate(templateDir) ? templateDir : null; } catch (error) { + if (error instanceof TestPostgresShutdownError) throw error; warnTemplateOnce( "build", `template build failed, using fresh clusters: ${(error as Error).message}`, @@ -604,6 +555,7 @@ async function startEmbeddedPostgresFromTemplateWithRetry( await stopEmbeddedPostgresBounded(created?.instance ?? null, () => cleanupEmbeddedPostgresTestDirs(rootDir), ); + if (!/address already in use/i.test(lastError.message)) throw lastError; } } @@ -636,12 +588,6 @@ export function __resetEmbeddedPostgresTemplateStateForTests(): void { } async function probeEmbeddedPostgresSupport(): Promise { - // A published template proves this host can run the cluster: it was built - // here. Reusing it makes the probe free for every file after the first. - // Building it costs one cluster start, the same as the throwaway probe did. - // When the template cannot be built or used, the throwaway probe below - // decides, so only a cluster that cannot start reports `supported: false`; - // a migration that cannot apply fails the suite loudly on the fresh path. if ((await getEmbeddedPostgresTemplateDir()) !== null) return { supported: true }; let started: { dataDir: string; instance: EmbeddedPostgresInstance } | null = null; @@ -650,12 +596,13 @@ async function probeEmbeddedPostgresSupport(): Promise { + const adminUrl = testPostgresAdminUrl(); + if (adminUrl) { + await checkTestPostgresAdmin(adminUrl); + return { supported: true }; + } if (!embeddedPostgresSupportPromise) { embeddedPostgresSupportPromise = probeEmbeddedPostgresSupport(); } @@ -684,6 +636,7 @@ async function startEmbeddedPostgresClusterForSuite( const started = await startEmbeddedPostgresFromTemplateWithRetry(templateDir, tempDirPrefix); return { ...started, mode: "template" }; } catch (error) { + if (error instanceof TestPostgresShutdownError) throw error; // The next caller in this process re-resolves the template: if it was // pruned it is rebuilt, instead of retrying a path that is gone. resetTemplateDirCache(); @@ -705,6 +658,8 @@ async function startEmbeddedPostgresClusterForSuite( export async function startEmbeddedPostgresTestDatabase( tempDirPrefix: string, ): Promise { + const adminUrl = testPostgresAdminUrl(); + if (adminUrl) return startTestPostgresDatabase(adminUrl); const { port, dataDir, instance, mode } = await startEmbeddedPostgresClusterForSuite(tempDirPrefix); lastStartMode = mode; @@ -723,8 +678,10 @@ export async function startEmbeddedPostgresTestDatabase( // stops can crash the process: the stop kills the backend socket, // but a queued write on that connection still fires later and finds // a null socket. - await closeRegisteredClients(connectionString); - await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir)); + try { await closeRegisteredClients(connectionString); } + finally { + await stopEmbeddedPostgresBounded(instance, () => cleanupEmbeddedPostgresTestDirs(dataDir)); + } }, }; } catch (error) { diff --git a/packages/db/src/test-postgres-admin.test.ts b/packages/db/src/test-postgres-admin.test.ts new file mode 100644 index 00000000000..7498a4a467c --- /dev/null +++ b/packages/db/src/test-postgres-admin.test.ts @@ -0,0 +1,86 @@ +import { randomUUID } from "node:crypto"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { createDb } from "./client.js"; +import { + __setEmbeddedPostgresCtorProviderForTests, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, + type EmbeddedPostgresTestDatabase, +} from "./test-embedded-postgres.js"; + +const previousAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +const previousTemplate = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; +let server: EmbeddedPostgresTestDatabase; +const databases: EmbeddedPostgresTestDatabase[] = []; +const support = await getEmbeddedPostgresTestSupport(); + +describe.skipIf(!support.supported)("shared PostgreSQL fixtures", () => { + beforeAll(async () => { + server = await startEmbeddedPostgresTestDatabase("paperclip-admin-regression-"); + process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = server.connectionString; + }, 90_000); + + afterEach(async () => { + __setEmbeddedPostgresCtorProviderForTests(null); + process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = server.connectionString; + if (previousTemplate === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = previousTemplate; + while (databases.length) await databases.pop()!.cleanup(); + }); + + afterAll(async () => { + await server?.cleanup(); + if (previousAdmin === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; + else process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = previousAdmin; + if (previousTemplate === undefined) delete process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + else process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = previousTemplate; + }); + + it("uses one server for isolated databases and closes clients before dropping them", async () => { + __setEmbeddedPostgresCtorProviderForTests(async () => { throw new Error("Must not start another PostgreSQL server"); }); + expect(await getEmbeddedPostgresTestSupport()).toEqual({ supported: true }); + const first = await startEmbeddedPostgresTestDatabase("ignored-"); databases.push(first); + const second = await startEmbeddedPostgresTestDatabase("ignored-"); databases.push(second); + expect(first.connectionString).not.toBe(second.connectionString); + const a = createDb(first.connectionString), b = createDb(second.connectionString); + await a.$client`CREATE TABLE isolated_probe (id integer)`; + expect((await b.$client`SELECT to_regclass('isolated_probe') AS name`)[0]?.name).toBeNull(); + await first.cleanup(); + await first.cleanup(); + const control = createDb(server.connectionString); + try { + const name = new URL(first.connectionString).pathname.slice(1); + expect(await control.$client`SELECT datname FROM pg_database WHERE datname = ${name}`).toHaveLength(0); + expect((await b.$client`SELECT 1 AS value`)[0]?.value).toBe(1); + } finally { await control.$client.end(); } + }, 90_000); + + it("rejects unavailable configured servers rather than reporting support", async () => { + process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = "postgres://paperclip:paperclip@127.0.0.1:1/postgres"; + await expect(getEmbeddedPostgresTestSupport()).rejects.toThrow(); + }); + + it("rejects unsafe templates and treats zero as a disabled template", async () => { + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = 'postgres"; DROP DATABASE postgres; --'; + await expect(startEmbeddedPostgresTestDatabase("ignored-")).rejects.toThrow("Invalid disposable test template name"); + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = "0"; + databases.push(await startEmbeddedPostgresTestDatabase("ignored-")); + }, 90_000); + + it("applies pending migrations when a supplied database template is not current", async () => { + const template = `paperclip_test_template_${randomUUID().replaceAll("-", "")}`; + const control = createDb(server.connectionString); + await control.$client.unsafe(`CREATE DATABASE "${template}"`); + try { + process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = template; + const database = await startEmbeddedPostgresTestDatabase("ignored-"); + databases.push(database); + const db = createDb(database.connectionString); + expect((await db.$client`SELECT to_regclass('companies') AS name`)[0]?.name).toBe("companies"); + } finally { + await control.$client.unsafe(`DROP DATABASE "${template}"`); + await control.$client.end(); + } + }, 90_000); + +}); diff --git a/packages/db/src/test-postgres-admin.ts b/packages/db/src/test-postgres-admin.ts new file mode 100644 index 00000000000..a76848fcc05 --- /dev/null +++ b/packages/db/src/test-postgres-admin.ts @@ -0,0 +1,50 @@ +import { randomUUID } from "node:crypto"; +import postgres from "postgres"; +import { applyPendingMigrations, closeRegisteredClients } from "./client.js"; + +export function testPostgresAdminUrl(): string | undefined { + const value = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL?.trim(); + if (!value) return undefined; + const url = new URL(value); + if (url.protocol !== "postgres:" && url.protocol !== "postgresql:") { + throw new Error("PAPERCLIP_TEST_POSTGRES_ADMIN_URL must be a PostgreSQL URL"); + } + return value; +} + +export async function checkTestPostgresAdmin(url: string): Promise { + const sql = postgres(url, { max: 1, connect_timeout: 5, onnotice() {} }); + try { await sql`SELECT 1`; } + finally { await sql.end({ timeout: 1 }); } +} + +export async function startTestPostgresDatabase(adminUrl: string): Promise<{ + connectionString: string; + cleanup(): Promise; +}> { + const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`; + const setting = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const template = setting === "0" ? undefined : setting; + if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) { + throw new Error("Invalid disposable test template name"); + } + const admin = postgres(adminUrl, { max: 1, connect_timeout: 5, onnotice() {} }); + const url = new URL(adminUrl); + url.pathname = `/${name}`; + const connectionString = url.toString(); + let cleaning: Promise | null = null; + const cleanup = () => cleaning ??= (async () => { + try { + await closeRegisteredClients(connectionString, { scope: "database" }); + await admin.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); + } finally { await admin.end({ timeout: 1 }); } + })(); + try { + await admin.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`); + await applyPendingMigrations(connectionString); + return { connectionString, cleanup }; + } catch (error) { + await cleanup(); + throw error; + } +} diff --git a/packages/db/src/test-postgres-command.test.ts b/packages/db/src/test-postgres-command.test.ts new file mode 100644 index 00000000000..6795fec273c --- /dev/null +++ b/packages/db/src/test-postgres-command.test.ts @@ -0,0 +1,57 @@ +import { execFile, spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; +import { getEmbeddedPostgresTestSupport } from "./test-embedded-postgres.js"; + +const previousAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +let supported = false; +try { supported = process.platform !== "win32" && (await getEmbeddedPostgresTestSupport()).supported; } +finally { + if (previousAdmin !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = previousAdmin; +} +const execute = promisify(execFile); + +it.skipIf(!supported).each(["SIGTERM", "SIGINT"] as const)("drains the test command's descendants before PostgreSQL on %s", async (signal) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-command-cancel-")); + const ready = path.join(directory, "ready"), done = path.join(directory, "done"); + const env: NodeJS.ProcessEnv = { + ...process.env, TEST_COMMAND_TREE_READY: ready, TEST_COMMAND_TREE_DONE: done, + TMPDIR: directory, TEMP: directory, TMP: directory, + }; + delete env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; + delete env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; + const wrapper = spawn(process.execPath, [ + "--import", fileURLToPath(new URL("../node_modules/tsx/dist/loader.mjs", import.meta.url)), + fileURLToPath(new URL("./test-postgres-command.ts", import.meta.url)), "--", process.execPath, + fileURLToPath(new URL("./__fixtures__/test-command-tree.mjs", import.meta.url)), "root", + ], { env, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + wrapper.stdout.on("data", (data) => { output = (output + String(data)).slice(-8000); }); + wrapper.stderr.on("data", (data) => { output = (output + String(data)).slice(-8000); }); + const exited = new Promise((resolve, reject) => { + wrapper.once("error", reject); + wrapper.once("exit", (code) => resolve(code)); + }); + try { + for (let i = 0; i < 1200 && !fs.existsSync(ready) && wrapper.exitCode === null; i++) await setTimeout(25); + expect(fs.existsSync(ready), output).toBe(true); + const descendants = JSON.parse(fs.readFileSync(ready, "utf8")) as { root: number; grandchild: number }; + wrapper.kill(signal); + expect(await exited, output).toBe(signal === "SIGINT" ? 130 : 143); + expect(fs.readFileSync(done, "utf8")).toBe("database available during child shutdown"); + expect(fs.readdirSync(directory).filter((name) => name.startsWith("paperclip-test-command-"))).toEqual([]); + for (const pid of Object.values(descendants)) { + const result = await execute("/bin/ps", ["-p", String(pid), "-o", "stat="]).catch(() => ({ stdout: "" })); + expect(result.stdout.trim() === "" || result.stdout.trim().startsWith("Z"), `descendant ${pid} still running`).toBe(true); + } + } finally { + if (wrapper.exitCode === null && wrapper.signalCode === null) { wrapper.kill("SIGTERM"); await exited; } + fs.rmSync(directory, { recursive: true, force: true }); + } +}, 90_000); diff --git a/packages/db/src/test-postgres-command.ts b/packages/db/src/test-postgres-command.ts new file mode 100644 index 00000000000..5edb86f2d5a --- /dev/null +++ b/packages/db/src/test-postgres-command.ts @@ -0,0 +1,46 @@ +import path from "node:path"; +import { createDb } from "./client.js"; +import { computeEmbeddedPostgresTemplateKey, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; +import { checkTestPostgresAdmin, testPostgresAdminUrl } from "./test-postgres-admin.js"; +import { beforeTestPostgresShutdown } from "./test-postgres-owner.js"; +import { startTestProcessTree } from "./test-process-tree.js"; + +const args = process.argv.slice(2); +if (args[0] === "--") args.shift(); +const command = args.shift() ?? process.execPath; +if (command === process.execPath && args.length === 0) args.push(path.resolve("scripts/run-vitest-stable.mjs")); + +let server: Awaited> | undefined; +try { + const env = { ...process.env }; + let adminUrl = testPostgresAdminUrl(); + if (adminUrl) await checkTestPostgresAdmin(adminUrl); + else { + server = await startEmbeddedPostgresTestDatabase("paperclip-test-command-"); + const url = new URL(server.connectionString); + url.pathname = "/postgres"; + adminUrl = url.toString(); + if (env.PAPERCLIP_TEST_POSTGRES_TEMPLATE !== "0") { + const template = `paperclip_test_template_${computeEmbeddedPostgresTemplateKey()}`; + const admin = createDb(adminUrl, { maxConnections: 1 }); + try { await admin.$client.unsafe(`CREATE DATABASE "${template}" TEMPLATE "paperclip"`); } + finally { await admin.$client.end({ timeout: 1 }); } + env.PAPERCLIP_TEST_POSTGRES_TEMPLATE = template; + } + } + env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = adminUrl; + console.log("[test:postgres] Each suite gets a disposable database on one test PostgreSQL server"); + const child = startTestProcessTree(command, args, env); + const removeShutdownTask = beforeTestPostgresShutdown({ + ...child, + afterStop: async () => { await server?.cleanup(); }, + }); + try { + process.exitCode = await child.wait(); + } finally { + try { await child.stop(); } + finally { removeShutdownTask(); } + } +} finally { + await server?.cleanup(); +} diff --git a/packages/db/src/test-postgres-owner-process.test.ts b/packages/db/src/test-postgres-owner-process.test.ts new file mode 100644 index 00000000000..8eb7d37cb68 --- /dev/null +++ b/packages/db/src/test-postgres-owner-process.test.ts @@ -0,0 +1,54 @@ +import { fork } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { setTimeout } from "node:timers/promises"; +import { fileURLToPath } from "node:url"; +import { expect, it } from "vitest"; +import { getEmbeddedPostgresTestSupport } from "./test-embedded-postgres.js"; + +const configuredAdmin = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +delete process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; +let embeddedSupported = false; +try { embeddedSupported = process.platform !== "win32" && (await getEmbeddedPostgresTestSupport()).supported; } +finally { + if (configuredAdmin !== undefined) process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL = configuredAdmin; +} + +function exists(pid: number): boolean { + try { process.kill(pid, 0); return true; } + catch (error) { return (error as NodeJS.ErrnoException).code !== "ESRCH"; } +} + +it.skipIf(!embeddedSupported).each(["exit", "SIGTERM"])("stops the owned PostgreSQL process when its worker receives %s", async (mode) => { + const child = fork(fileURLToPath(new URL("./__fixtures__/test-postgres-exit.mjs", import.meta.url)), [mode], { + execArgv: ["--import", fileURLToPath(new URL("../node_modules/tsx/dist/loader.mjs", import.meta.url))], + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + let output = ""; + child.stdout?.on("data", (data) => { output = (output + String(data)).slice(-8000); }); + child.stderr?.on("data", (data) => { output = (output + String(data)).slice(-8000); }); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("exit", (code) => resolve(code)); + }); + let server: { pid: number; dataDir: string } | undefined; + try { + const ready = await new Promise<{ pid: number; dataDir: string }>((resolve, reject) => { + child.once("message", (value) => resolve(value as { pid: number; dataDir: string })); + child.once("exit", () => reject(new Error(`Worker exited before readiness: ${output}`))); + }); + server = ready; + if (mode === "SIGTERM") child.kill("SIGTERM"); + expect(await exited, output).toBe(mode === "exit" ? 0 : 143); + for (let i = 0; i < 200 && exists(ready.pid); i++) await setTimeout(25); + expect(exists(ready.pid), output).toBe(false); + } finally { + if (child.exitCode === null && child.signalCode === null) { child.kill("SIGTERM"); await exited; } + if (server && exists(server.pid)) { + const contents = fs.readFileSync(path.join(server.dataDir, "postmaster.pid"), "utf8"); + if (Number(contents.split("\n")[0]) === server.pid) process.kill(server.pid, "SIGINT"); + for (let i = 0; i < 200 && exists(server.pid); i++) await setTimeout(25); + } + if (server && !exists(server.pid)) fs.rmSync(server.dataDir, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/db/src/test-postgres-owner.test.ts b/packages/db/src/test-postgres-owner.test.ts new file mode 100644 index 00000000000..ab984e09849 --- /dev/null +++ b/packages/db/src/test-postgres-owner.test.ts @@ -0,0 +1,105 @@ +import { EventEmitter } from "node:events"; +import type { ChildProcess } from "node:child_process"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ownTestPostgres } from "./test-postgres-owner.js"; + +function fixture() { + const child = Object.assign(new EventEmitter(), { + exitCode: null as number | null, + signalCode: null as NodeJS.Signals | null, + kill: vi.fn(() => true), + }); + const raw = { + initialise: vi.fn(async () => {}), + start: vi.fn(async () => {}), + stop: vi.fn(async () => {}), + process: child as unknown as ChildProcess, + }; + const owner = ownTestPostgres(raw); + function exit() { child.exitCode = 0; child.emit("exit", 0, null); } + return { child, raw, owner, exit }; +} + +afterEach(() => { vi.useRealTimers(); }); + +describe.skipIf(process.platform === "win32")("test PostgreSQL ownership", () => { + it("waits for fast shutdown and makes concurrent cleanup idempotent", async () => { + const before = process.listenerCount("exit"); + const f = fixture(); + const stop = f.owner.stop(); + expect(f.owner.stop()).toBe(stop); + expect(f.child.kill).toHaveBeenCalledWith("SIGINT"); + f.exit(); + await stop; + await f.owner.stop(); + expect(f.child.kill).toHaveBeenCalledTimes(1); + expect(process.listenerCount("exit")).toBe(before); + }); + + it("does not call the dependency's stop after the process already exited", async () => { + const f = fixture(); + f.exit(); + await f.owner.stop(); + expect(f.raw.stop).not.toHaveBeenCalled(); + expect(f.child.kill).not.toHaveBeenCalled(); + }); + + it("uses immediate PostgreSQL shutdown when fast shutdown times out", async () => { + vi.useFakeTimers(); + const f = fixture(); + const stop = f.owner.stop(); + await vi.advanceTimersByTimeAsync(5_000); + expect(f.child.kill.mock.calls).toEqual([["SIGINT"], ["SIGQUIT"]]); + f.exit(); + await stop; + expect(f.child.kill).not.toHaveBeenCalledWith("SIGKILL"); + }); + + it("reports a shutdown failure instead of declaring the process stopped", async () => { + vi.useFakeTimers(); + const f = fixture(); + const failure = expect(f.owner.stop()).rejects.toThrow("data directory is retained"); + await vi.advanceTimersByTimeAsync(10_000); + await failure; + f.exit(); + await f.owner.stop(); + }); + + it("keeps one exit hook while two servers have different owners", async () => { + const before = process.listenerCount("exit"); + const a = fixture(), b = fixture(); + expect(process.listenerCount("exit")).toBe(before + 1); + a.exit(); await a.owner.stop(); + expect(process.listenerCount("exit")).toBe(before + 1); + b.exit(); await b.owner.stop(); + expect(process.listenerCount("exit")).toBe(before); + }); + + it("never uses immediate shutdown when preparing a clean template", async () => { + vi.useFakeTimers(); + const f = fixture(); + const clean = expect(f.owner.stopGracefully()).rejects.toThrow("template cannot be published"); + await vi.advanceTimersByTimeAsync(30_000); + await clean; + expect(f.child.kill.mock.calls).toEqual([["SIGINT"]]); + f.exit(); + await f.owner.stop(); + }); + + it("rejects a crashed process as a publishable template", async () => { + const f = fixture(); + f.child.exitCode = 1; + await expect(f.owner.stopGracefully()).rejects.toThrow("without clean shutdown"); + await f.owner.stop(); + }); + + it("does not reuse an earlier immediate stop as a clean template shutdown", async () => { + vi.useFakeTimers(); + const f = fixture(); + const stopped = f.owner.stop(); + await vi.advanceTimersByTimeAsync(5_000); + f.exit(); + await stopped; + await expect(f.owner.stopGracefully()).rejects.toThrow("without clean shutdown"); + }); +}); diff --git a/packages/db/src/test-postgres-owner.ts b/packages/db/src/test-postgres-owner.ts new file mode 100644 index 00000000000..6ccc849501a --- /dev/null +++ b/packages/db/src/test-postgres-owner.ts @@ -0,0 +1,193 @@ +import type { ChildProcess } from "node:child_process"; +import { execFile, spawnSync } from "node:child_process"; +import path from "node:path"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); + +export type TestPostgresInstance = { + initialise(): Promise; + start(): Promise; + stop(): Promise; + process?: ChildProcess; +}; + +type Owner = { stop(clean?: boolean): Promise; interrupt(): void }; +export class TestPostgresShutdownError extends Error {} +const owners = new Set(); +type ShutdownTask = { stop(signal: NodeJS.Signals): Promise; interrupt(): void; afterStop?(): Promise }; +const shutdownTasks = new Set(); +let shuttingDown = false; +let hooksInstalled = false; + +function interruptOwnedServers(): void { + for (const task of shutdownTasks) task.interrupt(); + for (const owner of owners) owner.interrupt(); +} + +export function beforeTestPostgresShutdown(task: ShutdownTask): () => void { + shutdownTasks.add(task); + updateHooks(); + return () => { shutdownTasks.delete(task); updateHooks(); }; +} + +export async function drainTestPostgresOwners(): Promise { + const results = await Promise.allSettled([...owners].map((owner) => owner.stop())); + const failures = results.filter((result) => result.status === "rejected"); + if (failures.length) throw new AggregateError(failures.map((result) => result.reason), "Test PostgreSQL shutdown failed"); +} + +async function shutdown(signal: "SIGINT" | "SIGTERM" | "SIGHUP"): Promise { + if (shuttingDown) { interruptOwnedServers(); return; } + shuttingDown = true; + const registered = [...shutdownTasks]; + const tasks = await Promise.allSettled(registered.map((task) => task.stop(signal))); + for (const result of tasks) { + if (result.status === "rejected") console.error(result.reason); + } + try { await drainTestPostgresOwners(); } + catch (error) { console.error(error); } + const cleanups = await Promise.allSettled(registered.map((task) => task.afterStop?.())); + for (const result of cleanups) { + if (result.status === "rejected") console.error(result.reason); + } + process.exit(128 + { SIGINT: 2, SIGTERM: 15, SIGHUP: 1 }[signal]); +} + +const onInterrupt = () => { void shutdown("SIGINT"); }; +const onTerminate = () => { void shutdown("SIGTERM"); }; +const onHangup = () => { void shutdown("SIGHUP"); }; + +function updateHooks(): void { + if ((owners.size > 0 || shutdownTasks.size > 0) && !hooksInstalled) { + hooksInstalled = true; + process.on("exit", interruptOwnedServers); + process.on("SIGINT", onInterrupt); + process.on("SIGTERM", onTerminate); + process.on("SIGHUP", onHangup); + } else if (owners.size === 0 && shutdownTasks.size === 0) { + hooksInstalled = false; + process.removeListener("exit", interruptOwnedServers); + process.removeListener("SIGINT", onInterrupt); + process.removeListener("SIGTERM", onTerminate); + process.removeListener("SIGHUP", onHangup); + } +} + +async function settlesWithin(promise: Promise, timeoutMs: number): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { timer = setTimeout(() => resolve(false), timeoutMs); }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +export function ownTestPostgres(instance: TestPostgresInstance, dataDir?: string): Pick & { + stopGracefully(): Promise; +} { + let pending: Promise | null = null; + let stopPromise: Promise | null = null; + let stopped = false; + let usedImmediateShutdown = false; + let stoppedCleanly = false; + + function controlPath(child: ChildProcess): string | undefined { + if (!dataDir || !child.spawnfile) return undefined; + return path.join(path.dirname(child.spawnfile), `pg_ctl${path.extname(child.spawnfile) === ".exe" ? ".exe" : ""}`); + } + + async function requestStop(child: ChildProcess, mode: "fast" | "immediate", clean: boolean): Promise { + const control = controlPath(child); + if (control && (clean || process.platform === "win32")) { + try { + await execute(control, ["-D", dataDir!, "stop", "-m", mode, "-W"], { + timeout: 2000, maxBuffer: 64 * 1024, windowsHide: true, + }); + } catch (error) { + if (child.exitCode === null && child.signalCode === null) throw error; + } + } else child.kill(mode === "fast" ? "SIGINT" : "SIGQUIT"); + } + + async function run(operation: () => Promise): Promise { + if (stopped || stopPromise) throw new Error("Test PostgreSQL is shutting down"); + const running = operation(); + pending = running; + try { await running; } + finally { if (pending === running) pending = null; } + } + + const owner: Owner = { + interrupt() { + const child = instance.process; + if (child && child.exitCode === null && child.signalCode === null) { + usedImmediateShutdown = true; + const control = controlPath(child); + if (process.platform === "win32" && control) { + spawnSync(control, ["-D", dataDir!, "stop", "-m", "immediate", "-W"], { timeout: 2000, windowsHide: true }); + } else { + try { child.kill("SIGQUIT"); } catch {} + } + } + }, + stop(clean = false) { + if (stopped) return Promise.resolve(); + if (stopPromise) return stopPromise; + stopPromise = (async () => { + if (pending && !await settlesWithin(pending.catch(() => {}), 30_000)) { + throw new TestPostgresShutdownError("Test PostgreSQL startup did not settle before shutdown; its data directory is retained"); + } + const child = instance.process; + if (!child) { + if (!await settlesWithin(instance.stop(), 10_000)) { + throw new TestPostgresShutdownError("Test PostgreSQL shutdown timed out; its data directory is retained"); + } + } else if (child.exitCode === null && child.signalCode === null) { + let onExit: () => void; + const exited = new Promise((resolve) => { onExit = resolve; child.once("exit", onExit); }); + try { + await requestStop(child, "fast", clean); + if (!await settlesWithin(exited, clean ? 30_000 : 5_000)) { + if (clean) throw new TestPostgresShutdownError("Test PostgreSQL did not complete clean shutdown; the template cannot be published"); + usedImmediateShutdown = true; + await requestStop(child, "immediate", false); + if (!await settlesWithin(exited, 5_000)) { + throw new TestPostgresShutdownError("Test PostgreSQL did not exit after fast and immediate shutdown; its data directory is retained"); + } + } + } finally { + child.removeListener("exit", onExit!); + } + } + stoppedCleanly = !usedImmediateShutdown && (!child || (child.exitCode === 0 && child.signalCode === null)); + if (clean && !stoppedCleanly) { + throw new TestPostgresShutdownError("Test PostgreSQL exited without clean shutdown; the template cannot be published"); + } + instance.process = undefined; + stopped = true; + owners.delete(owner); + updateHooks(); + })().catch((error) => { + stopPromise = null; + if (error instanceof TestPostgresShutdownError) throw error; + throw new TestPostgresShutdownError("Test PostgreSQL shutdown failed; its data directory is retained", { cause: error }); + }); + return stopPromise; + }, + }; + owners.add(owner); + if (owners.size === 1) updateHooks(); + return { + initialise: () => run(() => instance.initialise()), + start: () => run(() => instance.start()), + stop: () => owner.stop(), + stopGracefully: async () => { + await owner.stop(true); + if (!stoppedCleanly) throw new TestPostgresShutdownError("Test PostgreSQL exited without clean shutdown; the template cannot be published"); + }, + }; +} diff --git a/packages/db/src/test-process-tree.ts b/packages/db/src/test-process-tree.ts new file mode 100644 index 00000000000..e5f805f3931 --- /dev/null +++ b/packages/db/src/test-process-tree.ts @@ -0,0 +1,78 @@ +import { execFile, spawn } from "node:child_process"; +import { constants } from "node:os"; +import { setTimeout } from "node:timers/promises"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); + +export function startTestProcessTree(command: string, args: string[], env: NodeJS.ProcessEnv) { + const posix = process.platform !== "win32"; + const child = spawn(command, args, { stdio: "inherit", env, detached: posix }); + let exited = false; + let stopping: Promise | null = null; + let retired = false; + const exit = new Promise((resolve, reject) => { + child.once("error", (error) => { exited = true; reject(error); }); + child.once("exit", (code, signal) => { + exited = true; + resolve(code ?? (signal ? 128 + constants.signals[signal] : 1)); + }); + }); + + async function alive(): Promise { + if (!child.pid) return false; + if (!posix) return !exited; + const { stdout } = await execute("/bin/ps", ["-axo", "pgid=,stat="], { timeout: 2000, maxBuffer: 1024 * 1024 }); + return stdout.split("\n").some((line) => { + const [group, state] = line.trim().split(/\s+/); + return Number(group) === child.pid && state && !state.startsWith("Z"); + }); + } + + function signal(signal: NodeJS.Signals): void { + if (!child.pid || retired) return; + try { + if (posix) process.kill(-child.pid, signal); + else child.kill(signal); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + + async function waitUntilStopped(milliseconds: number): Promise { + const deadline = Date.now() + milliseconds; + do { + if (!await alive()) return true; + await setTimeout(50); + } while (Date.now() < deadline); + return false; + } + + function stop(requested: NodeJS.Signals = "SIGTERM"): Promise { + if (stopping) return stopping; + stopping = (async () => { + if (posix) { + signal(requested); + if (!await waitUntilStopped(15_000)) { + signal("SIGKILL"); + if (!await waitUntilStopped(5_000)) throw new Error("Test command process group did not stop"); + } + } else if (child.pid && !exited) { + await execute("taskkill", ["/PID", String(child.pid), "/T", "/F"], { timeout: 15_000 }); + } + await exit; + retired = true; + })(); + return stopping; + } + + return { + stop, + interrupt: () => signal("SIGTERM"), + async wait(): Promise { + const status = await exit; + await stop(); + return status; + }, + }; +} diff --git a/packages/db/vitest.config.ts b/packages/db/vitest.config.ts index c1433e6ef38..08a8d7b9b34 100644 --- a/packages/db/vitest.config.ts +++ b/packages/db/vitest.config.ts @@ -3,6 +3,8 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ test: { environment: "node", + hookTimeout: 30_000, + teardownTimeout: 30_000, include: ["src/**/*.test.ts"], }, }); diff --git a/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts new file mode 100644 index 00000000000..dbb8b9eac18 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts @@ -0,0 +1,273 @@ +import { createInterface } from "node:readline"; +import { PassThrough } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NOOP_PLUGIN_TRACER, startWorkerRpcHost, type PluginInvocationContext, type PluginTracer } from "@paperclipai/plugin-sdk"; +import plugin, { __resetDaytonaSandboxHandleCacheForTest, __setDaytonaPluginContextForTest } from "./plugin.js"; +import manifest from "./manifest.js"; +import { leaseStateAttributes, withLeaseSpan } from "./lease-tracing.js"; + +const { get, create } = vi.hoisted(() => ({ get: vi.fn(), create: vi.fn() })); +vi.mock("@daytonaio/sdk", () => ({ + Daytona: class { get = get; create = create; }, + DaytonaNotFoundError: class extends Error {}, + DaytonaTimeoutError: class extends Error {}, +})); + +type Receipt = { + name: string; + startTimeMs: number; + endTimeMs: number; + attributes?: Record; + status?: { code: number }; + children: Receipt[]; +}; + +function sandbox(id = "fixture-sandbox", state = "archived") { + return { + id, state, target: "us", recoverable: true, errorReason: null as string | null, + start: vi.fn().mockResolvedValue(undefined), + stop: vi.fn().mockResolvedValue(undefined), + refreshData: vi.fn().mockResolvedValue(undefined), + recover: vi.fn().mockResolvedValue(undefined), + delete: vi.fn().mockResolvedValue(undefined), + getWorkDir: vi.fn().mockResolvedValue("/home/daytona"), + fs: { createFolder: vi.fn().mockResolvedValue(undefined), uploadFile: vi.fn().mockResolvedValue(undefined) }, + process: { executeCommand: vi.fn(async (command: string) => ({ + exitCode: 0, result: command.startsWith("cat ") ? '{"token":"private-token"}' : "bash", + })) }, + }; +} + +const params = { + driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment", + providerLeaseId: "fixture-sandbox", config: { apiKey: "private-api-key", timeoutMs: 300000, reuseLease: true }, + leaseMetadata: { workspaceSentinel: { token: "private-token", result: "written" } }, +}; + +function workerHarness() { + const stdin = new PassThrough(), stdout = new PassThrough(); + const lines = createInterface({ input: stdout }); + const pending = new Map(); + const receipts: Array<{ invocationId: string; span: Receipt }> = []; + let sequence = 0; + lines.on("line", line => { + const message = JSON.parse(line); + if (message.method) { + if (message.method === "span.record") receipts.push({ invocationId: message.paperclipInvocationId, span: message.params }); + if (message.id !== undefined) stdin.write(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: null }) + "\n"); + } else { + const waiter = pending.get(message.id); + pending.delete(message.id); + if (message.error) waiter?.reject(new Error(message.error.message)); + else waiter?.resolve(message.result); + } + }); + const worker = startWorkerRpcHost({ plugin, stdin, stdout }); + function call(method: string, input: unknown, invocation?: PluginInvocationContext): Promise { + const id = String(++sequence); + return new Promise((resolve, reject) => { + pending.set(id, { resolve, reject }); + stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: input, + ...(invocation ? { paperclipInvocation: invocation } : {}) }) + "\n"); + }); + } + return { + call, receipts, + initialize: () => call("initialize", { manifest, config: {}, apiVersion: 1, + instanceInfo: { instanceId: "fixture", hostVersion: "0.0.0" } }), + close() { worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); }, + }; +} + +function invocation(id: string, enabled = true): PluginInvocationContext { + return { id, scope: { companyId: "fixture-company" }, + ...(enabled ? { traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } : {}) }; +} + +describe("Daytona lease worker tracing", () => { + beforeEach(() => { get.mockReset(); create.mockReset(); __resetDaytonaSandboxHandleCacheForTest(); }); + afterEach(() => { __setDaytonaPluginContextForTest(null); __resetDaytonaSandboxHandleCacheForTest(); }); + + it.each([true, false])("records the archived start and nested verification only with trace context: %s", async enabled => { + const resource = sandbox(); get.mockResolvedValue(resource); + const worker = workerHarness(); + try { + await worker.initialize(); + expect(await worker.call("environmentResumeLease", params, invocation("resume", enabled))) + .toMatchObject({ providerLeaseId: resource.id, metadata: { resumedFromState: "archived" } }); + expect(resource.start).toHaveBeenCalledExactlyOnceWith(300); + expect(get).toHaveBeenCalledExactlyOnceWith(resource.id); + if (!enabled) { expect(worker.receipts).toEqual([]); return; } + expect(worker.receipts).toHaveLength(1); + const root = worker.receipts[0].span; + expect(root.name).toBe("plugin.worker"); + expect(root.children.map(span => span.name)).toEqual(["lease.lookup", "lease.channel_cleanup", "lease.start", "lease.verify"]); + expect(root.children[2].attributes).toMatchObject({ "paperclip.sandbox.startup.state": "archived" }); + expect(root.children[3].children.map(span => span.name)).toEqual(["lease.workspace", "lease.sentinel", "lease.shell"]); + function checkBounds(parent: Receipt) { + for (const child of parent.children) { + expect(child.startTimeMs).toBeGreaterThanOrEqual(parent.startTimeMs); + expect(child.endTimeMs).toBeGreaterThanOrEqual(child.startTimeMs); + expect(child.endTimeMs).toBeLessThanOrEqual(parent.endTimeMs); + checkBounds(child); + } + } + checkBounds(root); + expect(JSON.stringify(root)).not.toMatch(/private-|fixture-sandbox|fixture-company|\/home\//); + } finally { worker.close(); } + }); + + it("keeps simultaneous resume receipts separate and closes failed starts", async () => { + const slow = sandbox(), failed = sandbox("failed-sandbox", "stopped"); + let release!: () => void, started!: () => void; + const waiting = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { started = resolve; }); + slow.start.mockImplementation(async () => { started(); await waiting; }); + failed.start.mockRejectedValue(new Error("private-provider-error")); + get.mockImplementation(async id => id === slow.id ? slow : failed); + const worker = workerHarness(); + try { + await worker.initialize(); + const first = worker.call("environmentResumeLease", params, invocation("slow")); + await entered; + await expect(worker.call("environmentResumeLease", { ...params, providerLeaseId: failed.id }, invocation("failed"))) + .rejects.toThrow("private-provider-error"); + release(); await first; + expect(worker.receipts.map(record => record.invocationId)).toEqual(["failed", "slow"]); + const failure = worker.receipts[0].span; + expect(failure.status).toEqual({ code: 2 }); + expect(failure.children.find(span => span.name === "lease.start")?.status).toEqual({ code: 2 }); + expect(failure.children.some(span => span.name === "lease.verify")).toBe(false); + expect(JSON.stringify(worker.receipts)).not.toContain("private-provider-error"); + expect(slow.start).toHaveBeenCalledTimes(1); + expect(failed.start).toHaveBeenCalledTimes(1); + } finally { release(); worker.close(); } + }); + + it("waits for a pending teardown and records the wait under the resume worker", async () => { + const resource = sandbox("fixture-sandbox", "started"); get.mockResolvedValue(resource); + let release!: () => void, stopping!: () => void; + const waiting = new Promise(resolve => { release = resolve; }); + const entered = new Promise(resolve => { stopping = resolve; }); + resource.stop.mockImplementation(async () => { stopping(); await waiting; resource.state = "stopped"; }); + const worker = workerHarness(); + try { + await worker.initialize(); + const teardown = worker.call("environmentReleaseLease", params, invocation("release")); + await entered; + let settled = false; + const resume = worker.call("environmentResumeLease", params, invocation("resume")) + .then(result => { settled = true; return result; }); + // Process the resume request while the actual provider stop is held open. + await new Promise(resolve => setImmediate(resolve)); + expect(settled).toBe(false); + expect(get).toHaveBeenCalledTimes(1); + expect(resource.start).not.toHaveBeenCalled(); + release(); + await teardown; + expect(await resume).toMatchObject({ providerLeaseId: resource.id }); + const root = worker.receipts.find(receipt => receipt.invocationId === "resume")!.span; + expect(root.name).toBe("plugin.worker"); + expect(root.children.filter(span => span.name === "lease.wait_for_teardown")).toHaveLength(1); + expect(root.children[0].name).toBe("lease.wait_for_teardown"); + expect(root.children[0].endTimeMs).toBeLessThanOrEqual(root.children[1].startTimeMs); + expect(get).toHaveBeenCalledTimes(2); + expect(resource.start).toHaveBeenCalledTimes(1); + } finally { release(); worker.close(); } + }); + + it.each([false, true])("records missing-container confirmation and preserves refresh failures: %s", async fails => { + const resource = sandbox("fixture-sandbox", "error"); + resource.recoverable = false; + resource.errorReason = `not found: failed to inspect sandbox container ${resource.id}: Error response from daemon: No such container: ${resource.id}`; + get.mockResolvedValue(resource); + resource.refreshData.mockImplementation(async () => { + if (fails) throw new Error("private-refresh-failure"); + resource.state = "started"; + }); + const worker = workerHarness(); + try { + await worker.initialize(); + const result = worker.call("environmentResumeLease", params, invocation("resume")); + if (fails) await expect(result).rejects.toThrow("private-refresh-failure"); + else expect(await result).toMatchObject({ providerLeaseId: resource.id }); + expect(resource.refreshData).toHaveBeenCalledTimes(1); + const root = worker.receipts[0].span; + expect(root.children.map(span => span.name)).toEqual(fails + ? ["lease.lookup", "lease.refresh"] : ["lease.lookup", "lease.refresh", "lease.verify"]); + expect(root.children[1].status).toEqual(fails ? { code: 2 } : undefined); + expect(resource.start).not.toHaveBeenCalled(); + expect(resource.delete).not.toHaveBeenCalled(); + expect(create).not.toHaveBeenCalled(); + expect(JSON.stringify(worker.receipts)).not.toContain("private-refresh-failure"); + } finally { worker.close(); } + }); + + it.each(["started", "error"])("traces a %s sandbox without adding a provider start", async state => { + const resource = sandbox("fixture-sandbox", state); get.mockResolvedValue(resource); + const worker = workerHarness(); + try { + await worker.initialize(); + await worker.call("environmentResumeLease", params, invocation("resume")); + expect(resource.start).not.toHaveBeenCalled(); + expect(resource.recover).toHaveBeenCalledTimes(state === "error" ? 1 : 0); + const phases = worker.receipts[0].span.children; + expect(phases.some(span => span.name === "lease.start")).toBe(false); + expect(phases.some(span => span.name === "lease.recover")).toBe(state === "error"); + } finally { worker.close(); } + }); + + it("closes the verification group and failing workspace child", async () => { + const resource = sandbox(); get.mockResolvedValue(resource); + resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error")); + const worker = workerHarness(); + try { + await worker.initialize(); + await expect(worker.call("environmentResumeLease", params, invocation("resume"))) + .rejects.toThrow("private-workspace-error"); + const verify = worker.receipts[0].span.children.find(span => span.name === "lease.verify")!; + expect(verify.status).toEqual({ code: 2 }); + expect(verify.children.map(span => span.name)).toEqual(["lease.workspace"]); + expect(verify.children[0].status).toEqual({ code: 2 }); + expect(resource.delete).not.toHaveBeenCalled(); + } finally { worker.close(); } + }); + + it("records setup failure and compensating cleanup as siblings", async () => { + const resource = sandbox(); create.mockResolvedValue(resource); + resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error")); + const worker = workerHarness(); + try { + await worker.initialize(); + await expect(worker.call("environmentAcquireLease", { ...params, runId: "fixture-run" }, invocation("acquire"))) + .rejects.toThrow("private-workspace-error"); + expect(resource.delete).toHaveBeenCalledTimes(1); + const root = worker.receipts[0].span; + expect(root.children.map(span => span.name)).toEqual(["lease.create", "lease.workspace", "lease.cleanup"]); + expect(root.children[1].status).toEqual({ code: 2 }); + expect(root.children[2].status).toBeUndefined(); + } finally { worker.close(); } + }); +}); + +describe("lease tracing cannot change provider work", () => { + it.each(["missing", "before", "after", "during"])("preserves the result and error with a %s tracer failure", async mode => { + const tracer: PluginTracer = { ...NOOP_PLUGIN_TRACER, + ...(mode === "missing" ? {} : { async withSpan(_name: string, work: () => T | Promise): Promise { + if (mode === "before") throw new Error("trace failure"); + const result = work(); + if (mode === "during") throw new Error("trace failure"); + await result; + throw new Error("trace failure"); + } }), + }; + const success = vi.fn(async () => 42), error = new Error("provider failure"), failure = vi.fn(async () => { throw error; }); + expect(await withLeaseSpan(tracer, "lease.lookup", success)).toBe(42); + await expect(withLeaseSpan(tracer, "lease.lookup", failure)).rejects.toBe(error); + expect(success).toHaveBeenCalledTimes(1); expect(failure).toHaveBeenCalledTimes(1); + }); + + it("normalizes unknown provider states before recording", () => { + expect(leaseStateAttributes("provider-private-state")).toEqual({ "paperclip.sandbox.startup.state": "other" }); + }); +}); diff --git a/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts new file mode 100644 index 00000000000..8ad09499f64 --- /dev/null +++ b/packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts @@ -0,0 +1,49 @@ +import type { PluginTracer } from "@paperclipai/plugin-sdk"; + +export type LeaseSpanName = + | "lease.wait_for_teardown" + | "lease.lookup" + | "lease.refresh" + | "lease.channel_cleanup" + | "lease.start" + | "lease.recover" + | "lease.verify" + | "lease.workspace" + | "lease.sentinel" + | "lease.shell" + | "lease.create" + | "lease.expiry" + | "lease.cleanup"; + +/** Keep provider state useful without exporting a new SDK value as free text. */ +export function leaseStateAttributes(state: unknown): Record { + return { + "paperclip.sandbox.startup.state": + state === "started" || state === "stopped" || state === "archived" || state === "error" + ? state : "other", + }; +} + +/** Use the invocation's async parent, while keeping tracer faults out of work. */ +export async function withLeaseSpan( + tracer: PluginTracer, + name: LeaseSpanName, + work: () => Promise, + attributes?: Record, +): Promise { + let result: Promise | undefined; + // A tracer can fail before or after invoking work. Preserve its exact result + // or error and never repeat provider operations in either case. + const runOnce = () => result ??= Promise.resolve().then(work); + try { + if (tracer.withSpan) { + await tracer.withSpan(name, runOnce, { + "paperclip.sandbox.startup.provider": "daytona", + ...attributes, + }); + } + } catch { + // The operation's error is still held by result and is rethrown below. + } + return runOnce(); +} diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 74dd8029c15..55f1aa8eaf0 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -50,6 +50,7 @@ import type { } from "@paperclipai/plugin-sdk"; import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js"; import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js"; +import { withLeaseSpan, leaseStateAttributes, type LeaseSpanName } from "./lease-tracing.js"; import { observeSessionStreamExit } from "./session-stream-exit.js"; import { DaytonaIdleActivity, trackDaytonaHooks, type DaytonaIdleProof } from "./idle-activity.js"; @@ -126,6 +127,10 @@ export function getPluginTracer(): PluginTracer { return pluginContext?.tracer ?? NOOP_PLUGIN_TRACER; } +function traceLeaseStep(name: LeaseSpanName, work: () => Promise, attributes?: Record): Promise { + return withLeaseSpan(getPluginTracer(), name, work, attributes); +} + /** * Test seam: set the module-level plugin context, and return a restore function. * `plugin.test.ts` uses it to inject a recording tracer without running `setup`. @@ -528,12 +533,16 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P const startBoundMs = timeoutSeconds * 1_000 + LIVENESS_START_TIMEOUT_MARGIN_MS; if (sandbox.state === "error") { if (sandbox.recoverable) { - await withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)); + await traceLeaseStep("lease.recover", () => + withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)), + leaseStateAttributes(sandbox.state)); return; } throw new Error(`Daytona sandbox ${sandbox.id} is in an unrecoverable error state: ${sandbox.errorReason ?? "unknown error"}`); } - await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)); + await traceLeaseStep("lease.start", () => + withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)), + leaseStateAttributes(sandbox.state)); } function hasMissingSandboxContainer(sandbox: Sandbox): boolean { @@ -1362,6 +1371,7 @@ const sandboxHandleActivityGates = (() => { type SandboxLeaseAdmissionOptions = { allowClosed?: boolean; + traceLeaseWait?: boolean; }; const sandboxHandleLeaseAdmissionStates = (() => { @@ -1402,7 +1412,8 @@ async function withSandboxActivityGate( const teardownGate = sandboxHandleTeardownGates.current(scope); if (teardownGate) { - await teardownGate.promise; + if (options.traceLeaseWait) await traceLeaseStep("lease.wait_for_teardown", () => teardownGate.promise); + else await teardownGate.promise; if (!options.allowClosed && sandboxHandleLeaseAdmissionStates.isClosed(scope)) { throw new Error(`Daytona sandbox lease ${scope.providerLeaseId} is no longer active.`); } @@ -2682,39 +2693,39 @@ const plugin = definePlugin(trackDaytonaHooks({ return timeoutFailure(); }; const acquire = async (): Promise => { - const sandbox = await createSandbox(params, config, { + const sandbox = await traceLeaseStep("lease.create", () => createSandbox(params, config, { onCreateAttempt: (attempt) => { assertActive(); cleanup = attempt; }, onCreateFailure: observeFailure, onSnapshotIdentity: identity => { snapshotIdentity = identity; }, - }); + })); try { assertActive(); if (cleanup) cleanup.observedProviderLeaseId = sandbox.id; phase = "workspace"; - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox)); assertActive(); phase = "shell"; - const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now()))); + const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now())))); assertActive(); // Configure a provider-side destroy time at or before a caller deadline, so // an abandoned sandbox self-destroys even if Paperclip is down. The lease // carries the real provider expiry (or none) as evidence of the bound. phase = "expiry"; - const expiresAt = await configureSandboxExpiry({ + const expiresAt = await traceLeaseStep("lease.expiry", () => configureSandboxExpiry({ sandbox, requestedExpiresAt: params.requestedExpiresAt, nowMs: Date.now(), preserveCreationExpiry: config.ttlMinutes != null, - }); + })); assertActive(); phase = "sentinel"; - const workspaceSentinel = await writeWorkspaceSentinel({ + const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => writeWorkspaceSentinel({ sandbox, remoteCwd, params, config, timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())), - }); + })); assertActive(); sandboxHandleLeaseAdmissionStates.open({ driverKey: params.driverKey, @@ -2757,7 +2768,7 @@ const plugin = definePlugin(trackDaytonaHooks({ cleanupInProgress = true; try { const proof = idleActivity.proof(providerActivityKey(params, config, sandbox.id)); - await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true); + await traceLeaseStep("lease.cleanup", () => sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true)); idleActivity.confirm(proof); } catch (cleanupError) { if (cleanup) { @@ -2821,7 +2832,7 @@ const plugin = definePlugin(trackDaytonaHooks({ throw new Error("The stopped Daytona sandbox is still settling cancelled work. Retry shortly."); } return await withSandboxActivityGate(scope, async () => { - const sandbox = await getSandboxOrNull(scope, { bypassTeardownGate: true }); + const sandbox = await traceLeaseStep("lease.lookup", () => getSandboxOrNull(scope, { bypassTeardownGate: true })); if (!sandbox) { return { providerLeaseId: null, metadata: { expired: true } }; } @@ -2831,7 +2842,7 @@ const plugin = definePlugin(trackDaytonaHooks({ // backup-guarded replacement path. Unknown errors preserve the lease. if (hasMissingSandboxContainer(sandbox)) { try { - await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()); + await traceLeaseStep("lease.refresh", () => withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData())); assertHandleMatchesLease(sandbox, params.providerLeaseId); } catch (error) { evictSandboxHandle(scope); @@ -2859,48 +2870,50 @@ const plugin = definePlugin(trackDaytonaHooks({ // A stopped sandbox loses its pseudo-terminals, so a stored duplex channel // is dead after a real restart. Close and drop every channel on this lease // before the restart, so no stale channel id survives the resume. - await closeDaytonaDuplexChannelsForLease(params.providerLeaseId); + await traceLeaseStep("lease.channel_cleanup", () => closeDaytonaDuplexChannelsForLease(params.providerLeaseId)); } const resumedFromState = sandbox.state ?? null; await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); try { - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox, false); - // C3: a resumed lease must clear the workspace sentinel before it is - // trusted, even when the handle came from the cache. On any non-match we - // evict the cached handle and expire the lease so a stale/foreign sandbox - // is never reused on the subsequent (sentinel-skipping) exec path. - const workspaceSentinel = await verifyWorkspaceSentinel({ - sandbox, - remoteCwd, - leaseMetadata: params.leaseMetadata, - timeoutSeconds: toTimeoutSeconds(config.timeoutMs), - }); - if (workspaceSentinel.result !== "matched") { - evictSandboxHandle(scope); - return { - providerLeaseId: null, - metadata: { expired: true, workspaceSentinel }, - }; - } - const shellCommand = await detectSandboxShellCommand( - sandbox, - toTimeoutSeconds(config.timeoutMs), - ); - sandboxHandleCache.markFresh(scope); - sandboxHandleLeaseAdmissionStates.open(scope); - return { - providerLeaseId: sandbox.id, - metadata: leaseMetadata({ - config, + return await traceLeaseStep("lease.verify", async () => { + const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox, false)); + // C3: a resumed lease must clear the workspace sentinel before it is + // trusted, even when the handle came from the cache. On any non-match we + // evict the cached handle and expire the lease so a stale/foreign sandbox + // is never reused on the subsequent (sentinel-skipping) exec path. + const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => verifyWorkspaceSentinel({ sandbox, - shellCommand, - remoteCwd: requestedCwd ?? remoteCwd, - resumedLease: true, - resumedFromState, - sandboxState: "started", - workspaceSentinel, - }), - }; + remoteCwd, + leaseMetadata: params.leaseMetadata, + timeoutSeconds: toTimeoutSeconds(config.timeoutMs), + })); + if (workspaceSentinel.result !== "matched") { + evictSandboxHandle(scope); + return { + providerLeaseId: null, + metadata: { expired: true, workspaceSentinel }, + }; + } + const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand( + sandbox, + toTimeoutSeconds(config.timeoutMs), + )); + sandboxHandleCache.markFresh(scope); + sandboxHandleLeaseAdmissionStates.open(scope); + return { + providerLeaseId: sandbox.id, + metadata: leaseMetadata({ + config, + sandbox, + shellCommand, + remoteCwd: requestedCwd ?? remoteCwd, + resumedLease: true, + resumedFromState, + sandboxState: "started", + workspaceSentinel, + }), + }; + }); } catch (error) { evictSandboxHandle(scope); // A timeout, rate limit, or provider 5xx does not prove this sandbox is @@ -2910,7 +2923,7 @@ const plugin = definePlugin(trackDaytonaHooks({ throw error; } }, - { allowClosed: true }, + { allowClosed: true, traceLeaseWait: true }, ); }, diff --git a/packages/shared/src/ai-connections.ts b/packages/shared/src/ai-connections.ts index f9771a64b2b..fb784919866 100644 --- a/packages/shared/src/ai-connections.ts +++ b/packages/shared/src/ai-connections.ts @@ -190,6 +190,17 @@ export interface AiManagedConnectionSummary { unavailableReason?: string; usageProbeSupported?: boolean; } +/** Safe provider account details for one authorized saved credential. */ +export interface AiConnectionAccountIdentity { + connectionId: string; + grantId: string; + status: "available" | "unavailable" | "error"; + checkedAt: string; + accountLabel: string | null; + organizationLabel: string | null; + errorCode?: "unsupported" | "connection_unavailable" | "authentication_required" | "permission_denied" | "rate_limited" | "provider_unavailable" | "invalid_response"; + message?: string; +} export interface AiConnectionList { currentUserId: string; pools?: import("./ai-connection-router.js").AiConnectionPool[]; diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 2d6d93331d5..28f9fe42585 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -198,6 +198,12 @@ export const ISSUE_STATUSES = [ ] as const; export type IssueStatus = (typeof ISSUE_STATUSES)[number]; +/** Read-only task presentation. Monitoring never grants execution authority. */ +export const ISSUE_DISPLAY_STATUSES = [ + "backlog", "todo", "in_progress", "monitoring", "in_review", "done", "blocked", "cancelled", +] as const; +export type IssueDisplayStatus = (typeof ISSUE_DISPLAY_STATUSES)[number]; + export const INBOX_MINE_ISSUE_STATUSES = [ "backlog", "todo", diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index d6a9e5cbd02..da43d193d30 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -364,6 +364,7 @@ export { AGENT_ICON_NAMES, PROJECT_ICON_NAMES, ISSUE_STATUSES, + ISSUE_DISPLAY_STATUSES, ISSUE_SUMMARIES_MAX_IDS, INBOX_MINE_ISSUE_STATUSES, INBOX_MINE_ISSUE_STATUS_FILTER, @@ -562,6 +563,7 @@ export { type AgentIconName, type ProjectIconName, type IssueStatus, + type IssueDisplayStatus, type IssuePriority, type IssueReviewPolicy, type IssueWorkMode, diff --git a/packages/shared/src/types/issue.ts b/packages/shared/src/types/issue.ts index 29a8008ec80..451f745db97 100644 --- a/packages/shared/src/types/issue.ts +++ b/packages/shared/src/types/issue.ts @@ -20,6 +20,7 @@ import type { IssueHarnessKind, IssueOriginKind, IssuePriority, + IssueDisplayStatus, IssueReviewPolicy, IssueRecoveryActionKind, IssueRecoveryActionOutcome, @@ -270,6 +271,7 @@ export interface AcceptedPlanDecompositionSummary extends AcceptedPlanDecomposit } export interface IssueRelationIssueSummary { + displayStatus?: IssueDisplayStatus; id: string; identifier: string | null; title: string; @@ -850,12 +852,14 @@ export type IssueChanges = Record; export type IssueSummary = Pick< Issue, - "id" | "companyId" | "title" | "identifier" | "status" | "updatedAt" + "id" | "companyId" | "title" | "identifier" | "status" | "displayStatus" | "updatedAt" > & { externalConversationState: "active" | "waiting" | null; }; export interface Issue { + /** Read-only status for task presentation; never use it for execution admission. */ + displayStatus?: IssueDisplayStatus; /** Collection projections omit task bodies and cannot seed the detail cache. */ listProjection?: "index" | "rows"; statusVersion?: number; @@ -985,6 +989,7 @@ export type CompactIssue = Pick< | "privacyRootIssueId" | "title" | "description" + | "displayStatus" | "status" | "workMode" | "priority" diff --git a/packages/shared/src/types/search.ts b/packages/shared/src/types/search.ts index 825d4d4d1c4..bc9ea562d53 100644 --- a/packages/shared/src/types/search.ts +++ b/packages/shared/src/types/search.ts @@ -1,4 +1,4 @@ -import type { IssuePriority, IssueStatus } from "../constants.js"; +import type { IssueDisplayStatus, IssuePriority, IssueStatus } from "../constants.js"; export const COMPANY_SEARCH_SCOPES = ["all", "issues", "comments", "documents", "artifacts", "agents", "projects"] as const; export type CompanySearchScope = (typeof COMPANY_SEARCH_SCOPES)[number]; @@ -38,6 +38,7 @@ export interface CompanySearchIssueSummary { identifier: string | null; title: string; status: IssueStatus; + displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null; priority: IssuePriority; assigneeAgentId: string | null; @@ -76,6 +77,7 @@ export interface CompanySearchResult { export interface CompanySearchFilterOptionCounts { status: Partial>; + displayStatus?: Partial>; priority: Partial>; assigneeAgentId: Record; assigneeUserId: Record; diff --git a/packages/shared/src/types/user-profile.ts b/packages/shared/src/types/user-profile.ts index d961a407485..1ac26e033a7 100644 --- a/packages/shared/src/types/user-profile.ts +++ b/packages/shared/src/types/user-profile.ts @@ -1,4 +1,4 @@ -import type { IssuePriority, IssueStatus } from "../constants.js"; +import type { IssuePriority, IssueStatus, IssueDisplayStatus } from "../constants.js"; export interface UserProfileIdentity { id: string; @@ -42,6 +42,7 @@ export interface UserProfileIssueSummary { identifier: string | null; title: string; status: IssueStatus; + displayStatus?: IssueDisplayStatus; priority: IssuePriority; assigneeAgentId: string | null; assigneeUserId: string | null; diff --git a/packages/shared/src/validators/search.ts b/packages/shared/src/validators/search.ts index 839f4cb5296..3b81dae3836 100644 --- a/packages/shared/src/validators/search.ts +++ b/packages/shared/src/validators/search.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js"; +import { ISSUE_DISPLAY_STATUSES, ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js"; import { isUuidLike } from "../agent-url-key.js"; import { COMPANY_SEARCH_EXTRACT_KINDS, @@ -156,6 +156,9 @@ export const companySearchQuerySchema = z.object({ status: z.unknown() .optional() .transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)), + displayStatus: z.unknown() + .optional() + .transform((value, ctx) => parseEnumList(value, ctx, "displayStatus", ISSUE_DISPLAY_STATUSES)), priority: z.unknown() .optional() .transform((value, ctx) => parseEnumList(value, ctx, "priority", ISSUE_PRIORITIES)), diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 97290f343e1..5167a32193f 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -579,9 +579,8 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { readWorkflow("runner-full-stack-e2e.yml"), readWorkflow("e2e.yml"), readWorkflow("runner-protocol-live-evals.yml"), - readWorkflow("chat-feature-testops.yml"), - readWorkflow("chat-completion-testops.yml"), readWorkflow("chat-catalog-testops.yml"), + readWorkflow("chat-catalog-testops-case.yml"), ]; for (const workflow of actionPinWorkflows) { @@ -621,9 +620,8 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", "chat-catalog-testops.yml", + "chat-catalog-testops-case.yml", ]; const paidWorkflowNameSet = new Set(paidWorkflowNames); const providerSecretReference = @@ -642,6 +640,79 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { for (const name of paidWorkflowNames) { const workflow = readWorkflow(name); const triggerHeader = workflow.slice(0, workflow.indexOf("\njobs:\n")); + if (name === "chat-catalog-testops.yml") { + assert.match(triggerHeader, /^ workflow_dispatch:/m); + assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_call|workflow_run):/m); + const callerBlocks = ["chat", "chat_external"].map(id => workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0]); + assert.doesNotMatch(callerBlocks.reduce((rest, block) => rest.replace(block, ""), workflow), providerSecretReference); + assert.match(workflow, /group: chat-catalog-testops-streaming-v1/); + for (const id of ["chat", "chat_external"]) { + const block = workflow.split(` ${id}:\n`)[1].split(/\n [a-z_]+:\n/)[0]; + const assertCaller = block => { + assert.match(block, /if: needs\.authorize\.outputs\.operation == 'evaluate'/); + assert.ok(block.includes(`needs.select.outputs.${id === "chat" ? "native" : "external"}_count != '0'`)); + assert.match(block, /needs: \[authorize, select, build\]/); + assert.match(block, /uses: \.\/\.github\/workflows\/chat-catalog-testops-case\.yml/); + assert.match(block, /protocol: streaming-v1/); + const mapping = block.split("\n secrets:\n"); + assert.equal(mapping.length, 2); + assert.deepEqual(mapping[1].trim().split("\n").map(line => line.trim()), [ + "OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}", + "ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}", + "ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}", + ]); + assert.doesNotMatch(block, /secrets: inherit/); + assert.match(block, /fail-fast: false/); + }; + assertCaller(block); + const lane = id === "chat" ? "native" : "external"; + for (const mutant of [block.replace("needs.authorize.outputs.operation == 'evaluate' && ", ""), + block.replace(`needs.select.outputs.${lane}_count != '0'`, 'true'), + block.replace('${{ secrets.OPENROUTER_API_KEY }}', "''"), + block + '\n EXTRA: ${{ secrets.EXTRA }}', + block.replace(/\n secrets:[\s\S]*/, '\n secrets: inherit')]) assert.throws(() => assertCaller(mutant)); + } + const final = workflow.split(" verify:\n")[1]; + assert.match(final, /chat-completion-testops-cli\.mjs verify-stream/); + assert.doesNotMatch(final, /allurectl|cli\.mjs upload(?: |$)/); + continue; + } + if (name === "chat-catalog-testops-case.yml") { + assert.match(triggerHeader, /^ workflow_call:/m); + assert.doesNotMatch(triggerHeader, /^ (?:push|pull_request|workflow_dispatch|workflow_run):/m); + const chat = workflow.split(" chat:\n")[1].split("\n report:\n")[0]; + const report = workflow.split(" report:\n")[1].split("\n upload:\n")[0]; + const upload = workflow.split(" upload:\n")[1]; + const boundary = value => { + assert.match(value, /environment: runner-e2e-private-paid/); + assert.match(value, /CHAT_TESTOPS_PROTOCOL: \$\{\{ inputs\.protocol \}\}/); + assert.match(value, /test "\$CHAT_TESTOPS_PROTOCOL" = streaming-v1/); + assert.match(value, /chat-completion-testops-cli\.mjs guard/); + assert.match(value, /chat-completion-testops-cli\.mjs verify-selection/); + assert.match(value, /--max-parallel 1 --max-automatic-retries 0/); + assert.match(value, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/); + assert.doesNotMatch(value, /ALLURE_TOKEN/); + assert.equal([...value.matchAll(providerSecretReference)].length, 2); + assert.ok(value.indexOf("chat-completion-testops-cli.mjs verify-selection") < value.search(providerSecretReference)); + }; + boundary(chat); + for (const mutant of [chat.replace("environment: runner-e2e-private-paid", "environment: unprotected"), + chat.replace("cli.mjs verify-selection", "cli.mjs unverified-selection"), chat + "\n ALLURE_TOKEN: leaked"]) { + assert.throws(() => boundary(mutant)); + } + assert.match(report, /needs: chat/); assert.match(report, /always\(\) && !cancelled\(\)/); + assert.match(report, /environment: runner-e2e-private-history/); + assert.doesNotMatch(report, /secrets\./); + assert.match(report, /manual-private-transfer\.ts download-cell/); + assert.match(report, /cli\.mjs prepare-case-upload/); + assert.match(upload, /needs: report/); assert.match(upload, /always\(\) && !cancelled\(\)/); + assert.match(upload, /environment: runner-e2e-testops-reporting/); + assert.doesNotMatch(upload, /configure-aws-credentials|manual-private-transfer|secrets\.(?:OPENROUTER|ANTHROPIC)/); + assert.match(upload, /ALLURE_TOKEN: \$\{\{ secrets\.ALLURE_TOKEN \}\}/); + assert.match(upload, /cli\.mjs upload-case/); + assert.doesNotMatch(upload, /allurectl/); + continue; + } assert.doesNotMatch( triggerHeader, /^\s{2}(?:pull_request|pull_request_target|push|workflow_call|workflow_run):/m, @@ -656,87 +727,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => { (block) => [...block.matchAll(providerSecretReference)].length > 0, ); assert.ok(providerJobs.length > 0, `${name} needs a provider-secret job`); - if (name === "chat-completion-testops.yml" || name === "chat-catalog-testops.yml") { - assert.equal(providerJobs.length, 1); - const expectedKeys = name === "chat-catalog-testops.yml" ? 2 : 1; - const assertBoundary = (block) => { - assert.match(block, /^ chat:\n/); - assert.match(block, /\n needs: \[authorize, select, build\]\n/); - if (expectedKeys === 2) assert.ok(block.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.native_count != '0'")); - else assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); - assert.match(block, /\n environment: runner-e2e-private-paid\n/); - assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); - assert.match(block, /chat-completion-testops-cli\.mjs verify-selection/); - assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); - assert.match(block, /--id "\$\{\{ matrix\.executionId \}\}" --max-parallel 1 --max-automatic-retries 0/); - assert.match(block, /PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1/); - if (expectedKeys === 2) assert.ok(block.includes('max-parallel: ${{ fromJSON(needs.select.outputs.parallelism) }}')); - else assert.match(block, /max-parallel: 1/); - assert.doesNotMatch(block, /ALLURE_TOKEN/); - assert.equal([...block.matchAll(providerSecretReference)].length, expectedKeys); - if (expectedKeys === 2) assert.ok(block.includes("ANTHROPIC_API_KEY: ${{ contains(matrix.required_credentials, 'ANTHROPIC_API_KEY') && secrets.ANTHROPIC_API_KEY || '' }}")); - assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); - assert.ok(block.indexOf("chat-completion-testops-cli.mjs verify-selection") < block.search(providerSecretReference)); - }; - const block = providerJobs[0]; assertBoundary(block); - assert.equal([...workflow.matchAll(providerSecretReference)].length, expectedKeys); - for (const mutant of [ - block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), - block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), - block.replace(" - *guard\n", ""), - block.replace("chat-completion-testops-cli.mjs verify-selection", "chat-completion-testops-cli.mjs unverified-selection"), - block.replace('--id "${{ matrix.executionId }}"', "--all"), - block.replace("PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1", "PAPERCLIP_CHAT_COMPLETION_TESTOPS: unknown"), - ]) assert.throws(() => assertBoundary(mutant)); - if (expectedKeys === 2) assert.throws(() => assertBoundary(block.replace("contains(matrix.required_credentials, 'ANTHROPIC_API_KEY')", "true"))); - if (expectedKeys === 2) { - const external = jobBlocks.find(value => value.startsWith(' chat_external:\n')); - const assertExternal = value => { - assert.match(value, /needs: \[authorize, select, build\]/); - assert.ok(value.includes("if: needs.authorize.outputs.operation == 'evaluate' && needs.select.outputs.external_count != '0'")); - assert.match(value, /environment: runner-e2e-private-paid/); - assert.match(value, /max-parallel: 1/); - assert.ok(value.includes('matrix: ${{ fromJSON(needs.select.outputs.external_matrix) }}')); - assert.match(value, /env: \*retention/); - assert.match(value, /steps: \*catalog_worker_steps/); - }; - assertExternal(external); - for (const mutant of [external.replace('*catalog_worker_steps', '*unguarded'), external.replace('max-parallel: 1', 'max-parallel: 8'), external.replace('environment: runner-e2e-private-paid', 'environment: unprotected')]) assert.throws(() => assertExternal(mutant)); - } - continue; - } - if (name === "chat-feature-testops.yml") { - assert.equal(providerJobs.length, 1); - const assertBoundary = (block) => { - assert.match(block, /^ chat:\n/); - assert.match(block, /\n needs: \[authorize, select, build\]\n/); - assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/); - assert.match(block, /\n environment: runner-e2e-private-paid\n/); - assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/); - assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/); - assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/); - assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/); - assert.doesNotMatch(block, /ALLURE_TOKEN/); - assert.equal([...block.matchAll(providerSecretReference)].length, 1); - assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference)); - assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference)); - }; - const block = providerJobs[0]; - assertBoundary(block); - assert.equal([...workflow.matchAll(providerSecretReference)].length, 1); - assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/); - assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/); - assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/); - for (const mutant of [ - block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"), - block.replace("environment: runner-e2e-private-paid", "environment: unprotected"), - block.replace(" - *guard\n", ""), - block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"), - block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"), - block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"), - ]) assert.throws(() => assertBoundary(mutant)); - continue; - } for (const block of providerJobs) { if (name === "runner-protocol-live-evals.yml") { assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}")); diff --git a/scripts/benchmark-connection-install-capabilities.ts b/scripts/benchmark-connection-install-capabilities.ts index 0e6f13e8bc9..9cc24085399 100644 --- a/scripts/benchmark-connection-install-capabilities.ts +++ b/scripts/benchmark-connection-install-capabilities.ts @@ -15,6 +15,8 @@ import type { AuthorizationActor } from "../server/src/services/authorization.js // Run explicitly; this benchmark never reads DATABASE_URL or contacts a hosted DB. // node --import ./server/node_modules/tsx/dist/esm/index.mjs scripts/benchmark-connection-install-capabilities.ts --baseline-ref= +const connectionPurpose = process.argv.find(arg => arg.startsWith("--purpose="))?.slice(10) ?? "tool"; +assert.ok(connectionPurpose === "tool" || connectionPurpose === "ai", "Use --purpose=tool or --purpose=ai"); const baselineRef = process.argv.find(arg => arg.startsWith("--baseline-ref="))?.slice(15); assert.ok(baselineRef, "Supply an explicit pre-fix --baseline-ref"); const repository = fileURLToPath(new URL("../", import.meta.url)); @@ -101,7 +103,7 @@ try { permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } }); await db.insert(toolApplications).values({ id: applicationId, companyId, name: "Synthetic benchmark", type: "mcp_http" }); await db.insert(toolConnections).values({ id: connectionId, applicationId, companyId, uid: connectionId, - name: "Synthetic benchmark", transport: "mcp_remote", authKind: "none", config: { provider: "fixture" } }); + name: "Synthetic benchmark", connectionPurpose, transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", config: { provider: "fixture" } }); await db.insert(toolConnectionInstalls).values(installs === "company" ? [{ companyId, connectionId, targetType: "company", targetId: companyId }] : candidateIds.slice(0, Math.max(1, Math.floor(n / 4))).map(targetId => ({ companyId, connectionId, targetType: "agent" as const, targetId }))); @@ -151,7 +153,7 @@ try { const expectedRetained = installs === "company" ? n : Math.max(1, Math.floor(n / 4)); assert.equal(targetLookups(captured.baseline), n * expectedRetained + n - expectedRetained); assert.equal(targetLookups(captured.treatment), n); - const result = { n, installs, principal: "cloud noncreator member with target-scoped grant", completePositionalEquality: true, + const result = { n, installs, connectionPurpose, principal: "cloud noncreator member with target-scoped grant", completePositionalEquality: true, baseline: { ...before, ...counters.baseline, targetLookups: targetLookups(captured.baseline) }, treatment: { ...after, ...counters.treatment, targetLookups: targetLookups(captured.treatment) }, speedup: before.medianMs! / after.medianMs!, statements: captured, treatmentPlans: plans }; diff --git a/scripts/benchmark-github-webhook-ingress.ts b/scripts/benchmark-github-webhook-ingress.ts index 3b24116a650..25cbf5891e0 100644 --- a/scripts/benchmark-github-webhook-ingress.ts +++ b/scripts/benchmark-github-webhook-ingress.ts @@ -5,7 +5,7 @@ import { promises as fs, readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { applyPendingMigrations, chatActions, createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../packages/db/src/response-capture.js"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../server/src/__tests__/helpers/full-database-response-capture.js"; import { createGitHubWebhookIngress } from "../server/src/services/chat-channels/github/webhook-ingress.js"; import { captureIngressQueries, cleanupIngressFixture, githubIngressFixture, type CapturedQuery } from "../server/src/__tests__/helpers/github-webhook-ingress.js"; import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js"; @@ -60,7 +60,7 @@ try { queries.length = 0; f.calls.length = 0; const events: Array> = [], timestamp = Date.now(); - const capture = iteration === 11 ? createDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined; + const capture = iteration === 11 ? createFullDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, { chatActions }, capture); const start = performance.now(); const response = await service.processGitHubWebhookIngress(action.id); @@ -80,7 +80,7 @@ try { const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${candidate.sql}`, candidate.params as never[]); const positionalRows = await db.$client.unsafe(candidate.sql, candidate.params as never[]).values(); measurements[name] = { queries: executed, initialQueryPlan: plan["QUERY PLAN"], initialReadJsonBytes: Buffer.byteLength(JSON.stringify(positionalRows)), - responseValueBytes: events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes, + responseValueBytes: events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes, captureEvents: events }; } } diff --git a/server/scripts/benchmark-native-backup-publication.mjs b/server/scripts/benchmark-native-backup-publication.mjs new file mode 100644 index 00000000000..78d607ed3a8 --- /dev/null +++ b/server/scripts/benchmark-native-backup-publication.mjs @@ -0,0 +1,196 @@ +import assert from "node:assert/strict"; +import { fork, execFileSync } from "node:child_process"; +import { createServer, get, Agent } from "node:http"; +import { promises as fs } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { monitorEventLoopDelay, performance } from "node:perf_hooks"; +import { setTimeout as delay } from "node:timers/promises"; + +const filename = fileURLToPath(import.meta.url); +const checkout = path.resolve(path.dirname(filename), "../.."); +const arg = (name, fallback) => { + const position = process.argv.indexOf(`--${name}`); + return position < 0 ? fallback : process.argv[position + 1]; +}; +const percentile = (values, fraction) => { + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? null; +}; + +if (process.argv.includes("--worker")) { + const { publishNativeHarnessBackup } = await import(pathToFileURL(arg("module"))); + let publishing = false; + const server = createServer((_request, response) => { + response.setHeader("x-publication-active", publishing ? "1" : "0"); + response.end("ok"); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + process.send({ kind: "ready", port: server.address().port }); + process.on("message", async message => { + if (message.kind === "close") { + server.closeAllConnections(); + server.close(() => process.disconnect()); + return; + } + if (message.kind !== "publish") return; + const histogram = monitorEventLoopDelay({ resolution: 10 }); + histogram.enable(); + await delay(20); + const cpu = process.cpuUsage(); + const started = performance.now(); + publishing = true; + try { + await publishNativeHarnessBackup({ + db: {}, companyId: "benchmark", scopeDigest: "fixture", + backupRoot: message.backupRoot, pendingRoot: message.pendingRoot, + stamp: async () => {}, + }); + const durationMs = performance.now() - started; + const cpuUsed = process.cpuUsage(cpu); + publishing = false; + await delay(20); + histogram.disable(); + assert.equal(await fs.readFile(path.join(message.backupRoot, "current/manifest.json"), "utf8"), "replacement"); + assert.deepEqual((await fs.readdir(message.backupRoot)).sort(), ["current"]); + assert.equal(await fs.readFile(path.join(message.liveRoot, "untouched.txt"), "utf8"), "live"); + assert.equal((await fs.stat(message.liveRoot)).mode & 0o777, 0o555); + process.send({ kind: "result", durationMs, cpuMs: (cpuUsed.user + cpuUsed.system) / 1000, + eventLoopMaxMs: histogram.max / 1e6, workCompleted: true }); + } catch (error) { + publishing = false; + process.send({ kind: "error", message: error.stack }); + } + }); +} else { + const baselineRef = arg("baseline-ref"); + assert(baselineRef, "--baseline-ref is required"); + const pairs = Number(arg("pairs", "3")); + const directories = Number(arg("directories", "100")); + const filesPerDirectory = Number(arg("files-per-directory", "100")); + const intervalMs = Number(arg("interval-ms", "5")); + for (const value of [pairs, directories, filesPerDirectory, intervalMs]) assert(Number.isSafeInteger(value) && value > 0); + const evidence = await fs.mkdtemp(path.join(tmpdir(), "native-backup-publication-proof-")); + const { build } = await import("esbuild"); + const buildVariant = async variant => { + const outfile = path.join(evidence, `${variant}.mjs`); + await build({ absWorkingDir: checkout, entryPoints: ["server/src/services/native-runtime/native-harness-backup-publication.ts"], + outfile, bundle: true, platform: "node", format: "esm", target: "node24", sourcemap: true, + alias: { "@paperclipai/db/work-signals": path.join(checkout, "packages/db/src/work-signals.ts") }, + plugins: variant === "baseline" ? [{ name: "pinned-source", setup(builder) { + builder.onLoad({ filter: /\.ts$/ }, args => { + const relative = path.relative(checkout, args.path); + if (relative.startsWith("..") || relative.includes("node_modules")) return; + return { contents: execFileSync("git", ["show", `${baselineRef}:${relative}`], { cwd: checkout, encoding: "utf8" }), + loader: "ts", resolveDir: path.dirname(args.path) }; + }); + } }] : [], + }); + return outfile; + }; + const modules = { baseline: await buildVariant("baseline"), treatment: await buildVariant("treatment") }; + const makeFixture = async label => { + const root = await fs.mkdtemp(path.join(tmpdir(), `backup-publication-${label}-`)); + const backupRoot = path.join(root, "backups"); + const liveRoot = path.join(root, "live"); + const pendingRoot = path.join(backupRoot, "pending"); + await fs.mkdir(liveRoot); + await fs.writeFile(path.join(liveRoot, "untouched.txt"), "live"); + await fs.chmod(liveRoot, 0o555); + for (const version of ["current", "previous"]) { + const versionRoot = path.join(backupRoot, version); + for (let directory = 0; directory < directories; directory++) { + const target = path.join(versionRoot, `skill-${directory}`); + await fs.mkdir(target, { recursive: true }); + for (let file = 0; file < filesPerDirectory; file++) { + await fs.writeFile(path.join(target, `file-${file}.txt`), "retired instructions\n".repeat(4), { mode: 0o444 }); + } + await fs.chmod(target, 0o555); + } + await fs.symlink(liveRoot, path.join(versionRoot, "outside")); + await fs.chmod(versionRoot, 0o555); + } + await fs.mkdir(pendingRoot); + await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement"); + return { root, backupRoot, liveRoot, pendingRoot }; + }; + const run = async variant => { + const fixture = await makeFixture(variant); + const worker = fork(filename, ["--worker", "--module", modules[variant]], { stdio: ["ignore", "inherit", "inherit", "ipc"] }); + const exited = new Promise(resolve => worker.once("exit", resolve)); + const nextMessage = kind => new Promise((resolve, reject) => { + const onMessage = message => { + if (message.kind !== kind && message.kind !== "error") return; + worker.off("message", onMessage); worker.off("exit", onExit); + if (message.kind === "error") reject(new Error(message.message)); else resolve(message); + }; + const onExit = code => reject(new Error(`worker exited before ${kind}: ${code}`)); + worker.on("message", onMessage); worker.once("exit", onExit); + }); + const { port } = await nextMessage("ready"); + const agent = new Agent({ keepAlive: true, maxSockets: 512 }); + const requests = []; + let sent = 0; + const epoch = performance.now(); + const dispatch = intended => { + const dispatched = performance.now(); + requests.push(new Promise(resolve => { + const request = get({ hostname: "127.0.0.1", port, agent }, response => { + response.resume(); + response.on("end", () => resolve({ intended, dispatchLateMs: dispatched - intended, + latencyMs: performance.now() - intended, active: response.headers["x-publication-active"] === "1", error: false })); + }); + request.setTimeout(10_000, () => request.destroy(new Error("request timeout"))); + request.on("error", () => resolve({ intended, dispatchLateMs: dispatched - intended, + latencyMs: performance.now() - intended, active: false, error: true })); + })); + }; + let ticker; + const pump = () => { + const now = performance.now(); + while (epoch + (sent + 1) * intervalMs <= now) dispatch(epoch + (++sent * intervalMs)); + ticker = setTimeout(pump, Math.max(0, epoch + (sent + 1) * intervalMs - performance.now())); + }; + ticker = setTimeout(pump, intervalMs); + try { + await delay(250); + const pending = nextMessage("result"); + const start = performance.now(); + worker.send({ kind: "publish", ...fixture }); + const publication = await pending; + const end = performance.now(); + await delay(250); + clearTimeout(ticker); + const completed = await Promise.all(requests); + const during = completed.filter(request => request.intended >= start && request.intended <= end); + assert(during.length > 0); + const result = { variant, ...publication, sent, completed: completed.length, + errors: completed.filter(request => request.error).length, duringRequests: during.length, + requestsServedDuringPublication: completed.filter(request => request.active).length, + foregroundP95Ms: percentile(during.map(request => request.latencyMs), 0.95), + foregroundMaxMs: Math.max(...during.map(request => request.latencyMs)), + generatorMaxLateMs: Math.max(...completed.map(request => request.dispatchLateMs)) }; + console.log(JSON.stringify(result)); + return result; + } finally { + clearTimeout(ticker); agent.destroy(); + if (worker.connected) worker.send({ kind: "close" }); + await exited; + await fs.chmod(fixture.liveRoot, 0o700); + if ((await fs.readdir(fixture.backupRoot)).join() === "current") await fs.rm(fixture.root, { recursive: true }); + } + }; + const measurements = []; + for (let pair = 0; pair < pairs; pair++) { + for (const variant of pair % 2 ? ["treatment", "baseline"] : ["baseline", "treatment"]) measurements.push(await run(variant)); + } + const report = { baselineRef: execFileSync("git", ["rev-parse", baselineRef], { cwd: checkout, encoding: "utf8" }).trim(), + treatmentHead: execFileSync("git", ["rev-parse", "HEAD"], { cwd: checkout, encoding: "utf8" }).trim(), + treatmentDiff: execFileSync("git", ["diff", "HEAD", "--", "server/src/services/native-runtime/native-harness-backup-cleanup.ts", "server/src/services/native-runtime/native-harness-backup-publication.ts"], { cwd: checkout, encoding: "utf8" }), + node: process.version, platform: `${process.platform}/${process.arch}`, pairs, directories, filesPerDirectory, + filesRemovedPerPublication: 2 * directories * filesPerDirectory, intervalMs, evidence, measurements }; + const output = arg("output", path.join(evidence, "result.json")); + await fs.writeFile(output, JSON.stringify(report, null, 2)); + console.log(`Evidence: ${output}`); +} diff --git a/server/scripts/benchmark-profile-review-projection.ts b/server/scripts/benchmark-profile-review-projection.ts new file mode 100644 index 00000000000..694f3a25704 --- /dev/null +++ b/server/scripts/benchmark-profile-review-projection.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import { isDeepStrictEqual } from "node:util"; +import { pathToFileURL } from "node:url"; +import { createDb, startEmbeddedPostgresTestDatabase, type Db } from "@paperclipai/db"; +import { createToolProfileService } from "../src/services/tool-access/profiles/service.js"; +import { createToolAppAttention } from "../src/services/tool-access/catalog/attention.js"; +import { createProfileProjectionFixture } from "../src/__tests__/helpers/tool-profile-projection-fixture.js"; +const option = (name: string, fallback: string) => { const i = process.argv.indexOf(name); return i < 0 ? fallback : process.argv[i + 1] ?? fallback; }; +const baselineProfiles = option("--baseline-profiles", ""), baselineAttention = option("--baseline-attention", ""); +if (!baselineProfiles || !baselineAttention) throw new Error("--baseline-profiles and --baseline-attention modules required"); +const beforeProfiles = (await import(pathToFileURL(baselineProfiles).href)).createToolProfileService as typeof createToolProfileService; +const beforeAttention = (await import(pathToFileURL(baselineAttention).href)).createToolAppAttention as typeof createToolAppAttention; +const profiles = Number(option("--profiles", "200")), repeats = Number(option("--repeat", "5")); +if (!Number.isInteger(profiles) || profiles < 2 || !Number.isInteger(repeats) || repeats < 1) throw new Error("Positive fixture counts required"); +const database = await startEmbeddedPostgresTestDatabase("profile-review-projection-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const measuredDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const fixture = await createProfileProjectionFixture(db, { profiles }); + const now = () => new Date("2026-02-01T00:00:00Z"); + const implementations = { + before: { profiles: beforeProfiles(measuredDb, now), attention: beforeAttention(measuredDb, now) }, + after: { profiles: createToolProfileService(measuredDb, now), attention: createToolAppAttention(measuredDb, now) }, + }; + const invoke = async (label: keyof typeof implementations) => { + const service = implementations[label]; + return { profiles: await service.profiles.listProfiles(fixture.company.id), attention: await service.attention(fixture.company.id) }; + }; + const baseline = await invoke("before"), treatment = await invoke("after"); + assert.ok(isDeepStrictEqual(baseline, treatment), "Complete profile and attention results must match baseline"); + console.log(JSON.stringify({ fixture: { profiles, tools: 1900, meanRulesPerProfile: 380, ruleCounts: [65, 695], removedTools: 695, agents: 12, staleReferenceProfiles: Math.floor(profiles / 2) }, equality: true, profilePendingCounts: [...new Set(treatment.profiles.map(p => p.newToolsPendingCount))] })); + for (let round = 0; round < repeats; round++) { + for (const label of round % 2 ? ["after", "before"] as const : ["before", "after"] as const) { + global.gc?.(); queries = 0; rows = 0; + const start = performance.now(), cpu = process.cpuUsage(); + const result = await invoke(label); + const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu); + assert.equal(result.profiles.length, profiles); + console.log(JSON.stringify({ round, label, elapsedMs, cpuMs: (used.user + used.system) / 1000, queries, rows, profiles, forcedGc: Boolean(global.gc) })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/scripts/benchmark-skill-list.ts b/server/scripts/benchmark-skill-list.ts index b535fb2bed1..be1be55b982 100644 --- a/server/scripts/benchmark-skill-list.ts +++ b/server/scripts/benchmark-skill-list.ts @@ -16,7 +16,7 @@ import { authUsers, companies, companyMemberships, companySkills, companySkillSo projectSkillSources, projects, applyPendingMigrations, closeRegisteredClients, createDb, startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js"; import * as schema from "../../packages/db/src/schema/index.js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../../packages/db/src/response-capture.js"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../src/__tests__/helpers/full-database-response-capture.js"; import { withRunExecutionTiming } from "../src/services/execution-preparation-timing.js"; const argument = (name: string) => process.argv.find(value => value.startsWith(`${name}=`))?.slice(name.length + 1); @@ -195,7 +195,7 @@ try { }; const first = await request(); assert.ok(first.etag); const events: Array> = [], now = Date.now(); - const capture = captureValues ? createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined; + const capture = captureValues ? createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, schema, capture); const samples = []; try { @@ -213,7 +213,7 @@ try { } finally { capture?.flush(); } assert.equal(await sourceHash(), measuredSourceHash, "Source changed during measurement"); const timings = samples.map(sample => sample.elapsedMs).sort((a, b) => a - b); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1"); + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1"); if (capture) assert.ok(summary && summary.inFlightAtEnd === 0 && summary.overflowCalls === 0); const rowCount = (await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).length; console.log(JSON.stringify({ schema: "paperclip.skill-list-benchmark.v1", sourceRevision, measuredSourceHash, diff --git a/server/scripts/benchmark-tool-profile-matching.ts b/server/scripts/benchmark-tool-profile-matching.ts new file mode 100644 index 00000000000..f0e0e28ce73 --- /dev/null +++ b/server/scripts/benchmark-tool-profile-matching.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict"; +import { pathToFileURL } from "node:url"; +import { and, eq } from "drizzle-orm"; +import { agents, createDb, type Db } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "../src/__tests__/helpers/embedded-postgres.js"; +import { createToolProfileMatchingFixture } from "../src/__tests__/helpers/tool-profile-matching-fixture.js"; +import { createToolProfileResolver } from "../src/services/tool-access/profiles/resolution.js"; + +const baselinePath = process.argv[process.argv.indexOf("--baseline") + 1]; +if (!process.argv.includes("--baseline") || !baselinePath) throw new Error("--baseline module required"); +const baselineFactory = (await import(pathToFileURL(baselinePath).href)).createToolProfileResolver as typeof createToolProfileResolver; +const database = await startEmbeddedPostgresTestDatabase("profile-matching-benchmark-"); +const db = createDb(database.connectionString, { maxConnections: 1 }); +let queries = 0, rows = 0; +const observedDb = new Proxy(db, { get(target, key) { + if (key !== "select") return Reflect.get(target, key); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const selection = from(...args), execute = selection.execute.bind(selection); + selection.execute = async () => { const result = await execute(); queries++; rows += result.length; return result; }; + return selection; + }) as typeof builder.from; + return builder; + }; +} }); +try { + const { company, agent } = await createToolProfileMatchingFixture(db); + const dependencies = { + assertOptionalAgent: async (companyId: string, agentId: string | null | undefined) => { + const valid = await observedDb.select({ id: agents.id }).from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId!))); + assert.equal(valid.length, 1); + }, + resolveInstalledConnectionsForAgent: async () => [], + toCatalogEntry: () => { throw new Error("Full catalog definitions are not used by this assignment benchmark"); }, + }; + const before = baselineFactory(observedDb, dependencies); + const after = createToolProfileResolver(observedDb, dependencies); + const invoke = (resolver: typeof after) => resolver.getEffectiveToolAssignmentsForAgent(company.id, agent.id); + assert.deepEqual(await invoke(after), await invoke(before)); + for (let i = 0; i < 2; i++) { await invoke(before); await invoke(after); } + for (let round = 0; round < 7; round++) { + for (const [label, resolver] of round % 2 ? [["after", after], ["before", before]] as const : [["before", before], ["after", after]] as const) { + queries = 0; rows = 0; + const cpu = process.cpuUsage(), start = performance.now(); + for (let i = 0; i < 10; i++) await invoke(resolver); + const elapsedMs = performance.now() - start, used = process.cpuUsage(cpu); + console.log(JSON.stringify({ round, label, operations: 10, tools: 1890, rules: 1575, agents: 12, queries, rows, elapsedMs, cpuMs: (used.user + used.system) / 1000 })); + } + } +} finally { await db.$client.end({ timeout: 1 }); await database.cleanup(); } diff --git a/server/scripts/bundle-runtime.test.mjs b/server/scripts/bundle-runtime.test.mjs index a634d11b2ac..adacbca2871 100644 --- a/server/scripts/bundle-runtime.test.mjs +++ b/server/scripts/bundle-runtime.test.mjs @@ -130,6 +130,42 @@ describe("bundleRuntime", () => { } }); + it("attributes asynchronous database operations in a plain runtime bundle", async () => { + const captureModule = fileURLToPath(new URL("../../packages/db/src/response-capture.ts", import.meta.url)); + write(path.join(root, "server", "dist", "services", "database-owner.js"), [ + `import { withDatabaseOperation } from ${JSON.stringify(captureModule)};`, + 'export const run = (owner, db) => withDatabaseOperation(owner, async () => {', + ' await Promise.resolve();', + ' return await db.$client.unsafe("select $1", ["private-parameter"]);', + '});', + ].join("\n")); + write(path.join(root, "server", "dist", "index.js"), [ + `import { createDatabaseResponseCapture, installDatabaseResponseCapture } from ${JSON.stringify(captureModule)};`, + 'import { run } from "./services/database-owner.js";', + 'const events = [];', + 'const now = Date.now();', + 'const capture = createDatabaseResponseCapture({ start: now, end: now + 60000 }, event => events.push(event));', + 'const db = { $client: { unsafe: () => ({ then(resolve, reject) { return Promise.resolve(Object.assign([["private-result"]], { columns: [{ name: "value" }] })).then(resolve, reject); } }) } };', + 'installDatabaseResponseCapture(db, {}, capture);', + 'await Promise.all([run("fixture.first", db), run("fixture.second", db)]);', + 'capture.flush();', + 'console.log(JSON.stringify(events));', + ].join("\n")); + const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir: path.join(root, "server", "dist-runtime") }); + const env = { ...process.env }; + delete env.NODE_OPTIONS; + delete env.PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW; + const { stdout } = await execFileAsync(process.execPath, [summary.outfile], { cwd: path.join(root, "server"), env, timeout: 30_000 }); + const events = JSON.parse(stdout); + const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v2"); + expect(entries.map(entry => entry.owner).sort()).toEqual(["fixture.first", "fixture.second"]); + for (const entry of entries) expect(entry).toMatchObject({ calls: 1, rows: 1, callers: [] }); + expect(events.at(-1)).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2 }); + expect(stdout).not.toContain("private-parameter"); + expect(stdout).not.toContain("private-result"); + expect(stdout).not.toContain(root); + }); + it("emits a server module that loads without starting the server", async () => { const outdir = path.join(root, "server", "dist-runtime"); const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir }); diff --git a/server/src/__tests__/agent-auth-identity-transfer.test.ts b/server/src/__tests__/agent-auth-identity-transfer.test.ts index 819b23dc64a..995464f7933 100644 --- a/server/src/__tests__/agent-auth-identity-transfer.test.ts +++ b/server/src/__tests__/agent-auth-identity-transfer.test.ts @@ -4,7 +4,7 @@ import request from "supertest"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { createLocalAgentJwt } from "../agent-auth-jwt.js"; import { actorMiddleware } from "../middleware/auth.js"; @@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport(); it("bounds actual middleware responses and preserves the complete stored run", async () => { const fixture = await seed(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`); @@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport(); expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId, identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" }); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384); diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index fcf3c5ebd6a..415781328d3 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { eq, sql } from "drizzle-orm"; import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js"; -import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; +import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db"; import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js"; import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js"; @@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js"; import type { EnvironmentRuntimeService } from "../services/environment-runtime.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js"; import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js"; import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge"; import { heartbeatRunEvents } from "@paperclipai/db"; @@ -123,6 +124,44 @@ describe("persistent agent directories", () => { for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true); } finally { await worker.stop(); } }); + it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => { + const copy = await run(); + await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(), + nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm"); + try { + await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready; + await vi.waitFor(async () => { + await vi.advanceTimersByTimeAsync(100); + expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull(); + }); + await expect.poll(() => idleWorkSnapshot().active).toBe(0); + const reads = vi.spyOn(db, "select"); + await vi.advanceTimersByTimeAsync(120_000); + expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled(); + expect(worker.nextWakeAt()).toBeNull(); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + } finally { await worker.stop(); } + }); + it("honors a cleanup obligation published before stale-deadline settlement", async () => { + const copy = await run(), dueAt = new Date(Date.now() + 30_000); + await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(), + nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + const transact = db.transaction.bind(db); + const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => { + await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + return transact(callback, config); + }); + try { + await copies.release(companyId, copy.runId); + expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null }); + await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + } finally { race.mockRestore(); } + }); it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => { const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory"); clock(); const worker = coordinator(); @@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => { return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId }; } + it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)( + "recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => { + const { copy, lease } = await unavailableRemote(); + const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID(); + const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000); + await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" }); + await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" }); + const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment", + issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } }) + .where(eq(environmentLeases.id, lease.id)).returning(); + const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt, + releasedAt: new Date(), metadata: { agentId } }; + if (scenario === "foreign_company") { + const foreignCompanyId = randomUUID(); + await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) }); + successor.companyId = foreignCompanyId; + } + const handoff = createReusableLeaseHandoff(previous!, successor); + const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId, + state: scenario === "stopped" ? "stopped" : "destroyed" }); + await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } }) + .where(eq(environmentLeases.id, lease.id)); + await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination, + reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } }); + if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" }); + await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute")); + try { + await copies.recoverStopped(); + const recovered = (await copies.get(companyId, copy.runId))!; + if (scenario === "destroyed") { + expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", + receipt: { cleanupPending: false }, nextAttemptAt: null }); + expect(recovered.processStoppedAt).toBeInstanceOf(Date); + await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" }); + } else { + expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null }); + expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial); + } + expect(shell).not.toHaveBeenCalled(); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial); + const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id)); + expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination"); + } finally { shell.mockRestore(); } + }); + it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])( "defers unavailable cleanup without exact destruction proof (%s)", async proof => { const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed", diff --git a/server/src/__tests__/agent-record-read-projections.test.ts b/server/src/__tests__/agent-record-read-projections.test.ts index bf0a5183202..7786b017d3e 100644 --- a/server/src/__tests__/agent-record-read-projections.test.ts +++ b/server/src/__tests__/agent-record-read-projections.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { agents, companies, costEvents, createDb } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => { async function observe(run: (observed: ReturnType) => Promise) { const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agents, costEvents }, capture); try { const result = await run(observed); diff --git a/server/src/__tests__/ai-connections.test.ts b/server/src/__tests__/ai-connections.test.ts index e408200aa50..9a02179171f 100644 --- a/server/src/__tests__/ai-connections.test.ts +++ b/server/src/__tests__/ai-connections.test.ts @@ -1496,6 +1496,46 @@ console.log(JSON.stringify({ type: "result", subtype: "success", result: "hello" } finally { fetchSpy.mockRestore(); } }); + it("reads provider identity only for an authorized credential and preserves connected state when profile access is denied", async () => { + const owner = "identity-owner"; + await db.insert(companyMemberships).values({ companyId, principalId: owner, principalType: "user", status: "active", membershipRole: "member" }); + const account = await service.save(companyId, owner, { provider: "anthropic", method: "subscription", ownership: "personal", name: "Identity account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "identity-selected-secret"); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ account: { email: "avery@example.test", uuid: "private-account-id" }, organization: { name: "Example team", uuid: "private-org-id" } })); + const app = express(); + app.use((req, _res, next) => { + req.actor = req.header("x-test-anonymous") ? { type: "none" } + : req.header("x-agent") ? { type: "agent", agentId, companyId, companyIds: [companyId], onBehalfOfUserId: owner } + : { type: "board", source: "session", userId: req.header("x-test-user") ?? owner, companyIds: [companyId] }; + next(); + }); + app.use("/api", aiConnectionRoutes(db)); + app.use((error: { status?: number; message: string }, _req: express.Request, res: express.Response, _next: express.NextFunction) => { res.status(error.status ?? 500).json({ error: error.message }); }); + const url = `/api/companies/${companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`; + try { + const result = await request(app).get(url); + expect(result.status).toBe(200); + expect(result.headers["cache-control"]).toBe("no-store"); + expect(result.body).toMatchObject({ ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" }); + expect(fetchSpy).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ headers: expect.objectContaining({ Authorization: "Bearer identity-selected-secret" }) })); + for (const value of ["identity-selected-secret", "private-account-id", "private-org-id"]) expect(JSON.stringify(result.body)).not.toContain(value); + fetchSpy.mockReset(); + expect((await request(app).get(url).set("x-test-user", "bob")).status).toBe(404); + expect((await request(app).get(url.replace(companyId, otherCompanyId))).status).toBe(403); + expect((await request(app).get(url.replace(account.grantId, randomUUID()))).status).toBe(404); + expect((await request(app).get(url).set("x-test-anonymous", "yes")).status).toBe(401); + expect((await request(app).get(url).set("x-agent", "yes")).status).toBe(403); + expect((await request(app).get(url.replace(account.connectionId, "invalid"))).status).toBe(404); + expect(fetchSpy).not.toHaveBeenCalled(); + fetchSpy.mockResolvedValue(Response.json({ error: "private-provider-error" }, { status: 403 })); + expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "permission_denied", accountLabel: null }); + expect((await service.list(companyId, owner)).find(item => item.id === account.connectionId)?.status).toBe("connected"); + fetchSpy.mockClear(); + await db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId)); + expect((await request(app).get(url)).body).toMatchObject({ status: "unavailable", errorCode: "connection_unavailable" }); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { fetchSpy.mockRestore(); } + }); + it("uses the same usage probe for shared accounts across legacy and native runner selections", async () => { const account = await service.save(companyId, "alice", { provider: "anthropic", method: "subscription", ownership: "shared", name: "Shared usage account", loginSessionId: "fixture", agentIds: [], allAgents: true }, "shared-usage-secret"); const selected = { provider: "anthropic", method: "subscription", mode: "shared", ...account } as const; diff --git a/server/src/__tests__/attention-service.test.ts b/server/src/__tests__/attention-service.test.ts index 5af78c2cb8b..ca18a4a2d3c 100644 --- a/server/src/__tests__/attention-service.test.ts +++ b/server/src/__tests__/attention-service.test.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; import { agents, @@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => { }))); } const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const feed = await attentionService(db).list(companyId, { userId: "board-user" }); expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64); @@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => { }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const feed = await attentionService(db).list(companyId, { @@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => { expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort()) .toEqual([...keptIds].sort()); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4); @@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => { const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction"); const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item])); const summaryEvents: Array> = [], captureNow = Date.now(); - const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event)); + const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event)); installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture); try { const summary = await attentionService(db).summary(companyId, { userId: "board-user" }); @@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => { expect(index.totalCount).toBe(feed.totalCount); } finally { summaryCapture.flush(); } - const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && ["documents", "assets", "issue_attachments"].includes(String(event.table))); expect(cardReads).toEqual([]); diff --git a/server/src/__tests__/chat-channels.integration.test.ts b/server/src/__tests__/chat-channels.integration.test.ts index 40783fae2cd..8b3b8f1abb9 100644 --- a/server/src/__tests__/chat-channels.integration.test.ts +++ b/server/src/__tests__/chat-channels.integration.test.ts @@ -35,11 +35,13 @@ import { budgetService } from "../services/budgets.js"; import { idleWorkSnapshot } from "../services/task-admission.js"; import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js"; import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js"; +import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js"; +import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js"; import { createSystemRecovery } from "../services/system-recovery-state.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js"; import type { Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js"; import * as activityLogModule from "../services/activity-log.js"; import * as connectionCredentialBindings from "../services/connection-credential-bindings.js"; @@ -114,6 +116,7 @@ import { isNotNull, isNull, like, + ne, notInArray, or, sql, @@ -2891,7 +2894,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk }; await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id)); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations, chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks, companyMemberships, chatActions, heartbeatRuns }, capture); @@ -2920,7 +2923,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } }; await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id)); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatDeliveries }, capture); let projected: { event: Record }; try { @@ -7164,6 +7167,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0); expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0); }); + it("offers Retry for a saved automatic PR event after a successful older review", async () => { + const f = await reviewBotFixture(); + const payload = { + action: "opened", installation: { id: 2468 }, + repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } }, + sender: { id: 77, login: "maintainer" }, + pull_request: { number: 81, title: "Review successive heads", body: "", draft: false, + base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) }, + user: { id: 42, login: "octocat", type: "User" }, labels: [] }, + }; + const send = async (event: typeof payload) => { + await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({ + event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret, + })); + await f.service.processPendingDeliveries(); + }; + await send(payload); + await expect.poll(async () => { + await f.service.processPendingDeliveries(); + return (await readChatConversationsByEndpointId(f.endpoint.id)).length; + }, { timeout: 10_000 }).toBe(1); + const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id); + const [task] = await readIssuesById(conversation.issueId!); + const [original] = await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"))); + await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, + issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id, + contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] }, + }).returning(); + await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id)); + await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } }); + await expect.poll(async () => { + await f.service.processPendingDeliveries(); + return (await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length; + }).toBe(2); + const [pending] = await db.select().from(chatActions).where(and( + eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id))); + await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1); + const [receipt] = await readAgentWakeupRequestsById(pending.id); + await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null, + payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]), + requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id)); + expect(receipt.requestedByActorType).toBe("user"); + await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id)); + expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" })) + .toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id }); + await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } }) + .where(eq(agents.id, f.assignedAgentId)); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() }); + f.wakeup.mockImplementation(heartbeatService(db).wakeup); + const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user", + actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }); + await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId }); + const [resumed] = await readAgentWakeupRequestsById(pending.id); + expect(resumed.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null, + contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } }); + expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId }); + }); + it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => { const f = await reviewBotFixture(); const { buildSkillMentionHref } = await import("@paperclipai/shared"); @@ -62894,7 +62959,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { const commentsBeforeDisabledLifecycle = await db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)); + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(issueComments.id); const priorLifecycleIds = new Set( ( await db @@ -62974,7 +63040,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { db .select({ id: issueComments.id }) .from(issueComments) - .where(eq(issueComments.issueId, conversation.issueId)), + .where(eq(issueComments.issueId, conversation.issueId)) + .orderBy(issueComments.id), ).resolves.toEqual(commentsBeforeDisabledLifecycle); await service.shutdown(); }); @@ -64168,6 +64235,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { return { ...f, pending, originalDelivery }; } + async function successfulGitHubQueuedInputFixture() { + const f = await currentGitHubTaskRetryFixture(true); + const companyId = f.fixture.companyId, issueId = f.issue.id; + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload, + [String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id)); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1, + schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard", + completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {}, + canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed", + completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId)); + const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId, + completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId, + resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning(); + await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id)); + const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant", + observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry }; + return { ...f, input, result }; + } + + it("retries saved GitHub input after a successful run without replaying that run", async () => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const { input } = f; + const companyId = f.fixture.companyId; + const before = (await readHeartbeatRunsById(f.runId))[0]; + const [request, duplicate] = await Promise.all([requestTaskRetry(input), + requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]); + expect(request.accepted).toBe(true); + expect(duplicate.requestId).toBe(request.requestId); + await reconcileTaskChatRetryReceipts(db, { companyId }); + const [source] = await readAgentWakeupRequestsById(f.pending!.id); + expect(source.runId).toBeTruthy(); + const [successor] = await readHeartbeatRunsById(source.runId!); + expect(successor).toMatchObject({ status: "queued", retryOfRunId: null, + contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } }); + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(request.requestId!))[0]) + .toMatchObject({ status: "coalesced", runId: successor.id }); + expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + } finally { await f.service.shutdown(); } + }); + + it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)( + "keeps saved-input Retry behind current admission gates: %s", async gate => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const companyId = f.fixture.companyId; + const request = await requestTaskRetry(f.input); + const before = (await readHeartbeatRunsById(f.runId))[0]; + if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id)); + if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } }) + .where(eq(chatActions.id, f.pending!.id)); + if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id)); + if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId)); + const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId, + provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" }; + if (gate === "cleanup_pending") { + await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" }); + await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id)); + } + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull(); + expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + if (gate === "cleanup_pending") { + expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution"); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) }, + }).where(eq(environmentLeases.id, lease.id)); + await reconcileTaskChatRetryReceipts(db, { companyId }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy(); + } + } finally { await f.service.shutdown(); } + }); + + it.each(["issue.comment.reopen", "boardMessageRetry"] as const)( + "resumes saved GitHub input after a successful board turn: %s", async origin => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const [previous] = await readHeartbeatRunsById(f.runId); + await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: { + ...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }), + } }).where(eq(heartbeatRuns.id, f.runId)); + const before = (await readHeartbeatRunsById(f.runId))[0]; + const retry = await requestTaskRetry(f.input); + await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! }); + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + expect(saved.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null, + contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } }); + expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before); + } finally { await f.service.shutdown(); } + }); + + it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => { + const f = await successfulGitHubQueuedInputFixture(); + try { + const companyId = f.fixture.companyId; + const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId, + provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" }; + await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" }); + await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id)); + const retry = await requestTaskRetry(f.input); + await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! }); + expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull(); + const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id, + authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning(); + const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId, + source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution", + requestedByActorType: "user", requestedByActorId: f.input.actorId, + payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id, + _paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" }, + }, [comment.id]), + }).returning(); + await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(), + metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) }, + }).where(eq(environmentLeases.id, lease.id)); + await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! }); + const [saved] = await readAgentWakeupRequestsById(f.pending!.id); + expect(saved.runId).toBeTruthy(); + expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued", + contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } }); + expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board); + } finally { await f.service.shutdown(); } + }); + + it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => { + const f = await successfulGitHubQueuedInputFixture(); + const { issueRoutes } = await import("../routes/issues.js"); + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); }); + app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service })); + app.use(errorHandler); + try { + const body = { observedRunId: f.runId, idempotencyKey: randomUUID() }; + const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body); + expect(first.status, JSON.stringify(first.body)).toBe(202); + await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy(); + const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body); + expect(second.status).toBe(202); + expect(second.body.requestId).toBe(first.body.requestId); + expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1); + const detail = await request(app).get(`/api/issues/${f.issue.id}`); + expect(detail.status).toBe(200); + expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false }); + expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result); + } finally { await f.service.shutdown(); } + }); + it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)( "admits current GitHub task Retry without replaying obsolete input: %s", async kind => { const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input")); @@ -64297,7 +64521,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { status: "active", membershipRole: "operator" }); } const queryEvents: Record[] = []; - const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1, + const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1, end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined; if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments, agentWakeupRequests, chatEndpoints, chatConversations }, capture); @@ -64317,7 +64541,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]); if (capture) { capture.flush(); - expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries)) + expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries)) .toBeLessThan(180); } expect(two.requestId).toBe(one.requestId); @@ -67517,7 +67741,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }))); const scope = { companyId: context.fixture.companyId, issueId: context.issue.id }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture); let scopes: Array<{ companyId: string; issueId: string }>; try { @@ -67534,7 +67758,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1, completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 }); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1") + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1") .reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length); } finally { await context.service.shutdown(); await other.service.shutdown(); @@ -77242,7 +77466,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { }))); const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture); let scopes: Array<{ companyId: string; issueId: string }>; try { @@ -77259,7 +77483,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => { } expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1, completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 }); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1") + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1") .reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length); } finally { await context.service.shutdown(); await other.service.shutdown(); diff --git a/server/src/__tests__/chat-completion-delivery.test.ts b/server/src/__tests__/chat-completion-delivery.test.ts index 468a7a86f80..2dfd4290972 100644 --- a/server/src/__tests__/chat-completion-delivery.test.ts +++ b/server/src/__tests__/chat-completion-delivery.test.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import { and, asc, eq } from "drizzle-orm"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agents, agentWakeupRequests, chatCompletionDeliveries as deliveries, chatTaskHandoffs as handoffs, - companies, createDb, documents, environmentLeases, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues } from "@paperclipai/db"; + companies, createDb, documents, environmentLeases, executionWorkspaces, heartbeatRuns, issueComments, issueRecoveryActions, issueThreadInteractions, issues, workspaceOperations } from "@paperclipai/db"; import { recoveryPostgresSupport as support, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; import { settleInterruptedNativeBootstrap, terminalizeLegacyExecution } from "../services/legacy-execution-recovery.js"; import { getExecutionBlocker } from "../services/execution-blocker.js"; @@ -15,6 +15,8 @@ import { documentService } from "../services/documents.js"; import { instanceSettingsService } from "../services/instance-settings.js"; import { chatCompletionDeliveryService, isCompletedOnboardingHandoffWake, prepareChatCompletionTurn, recordChatCompletion, recordChatHandoff } from "../services/chat-completion-delivery.js"; import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../services/heartbeat.js"; +import { createHeartbeatQueue } from "../services/heartbeat/queue.js"; +import { createHeartbeatQueueTestDependencies } from "./helpers/heartbeat-queue.js"; (support.supported ? describe : describe.skip)("chat completion delivery", () => { let db: ReturnType; @@ -68,6 +70,75 @@ import { heartbeatService, shouldQueueFollowupForRunningIssueWake } from "../ser })]); expect(JSON.stringify(run.contextSnapshot?.chatCompletionUpdates)).not.toMatch(/Injected|credentials|Plan accepted|Agent approved/); }); + it("retains a distinct completion receipt behind a real dependency gate", async () => { + const f = await seed(); + const blocker = await f.create(); + await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [blocker.id] }); + await f.finish(); + const heartbeat = heartbeatService(db); + // The generic dependency signal is replaceable. The outbox's exact wake + // receipt is not: it needs that identity to reconcile a refused admission. + await heartbeat.wakeup(f.agentId, { + source: "automation", triggerDetail: "system", reason: "issue_continuation_needed", + requestedByActorType: "system", requestedByActorId: "dependency-test", + payload: { issueId: f.sourceId }, contextSnapshot: { issueId: f.sourceId }, + }); + const [delivery] = await f.rows(); + const service = chatCompletionDeliveryService(db, heartbeat); + await service.deliver(delivery.id); + const receipts = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId)); + expect(receipts).toEqual(expect.arrayContaining([ + expect.objectContaining({ idempotencyKey: `chat-completion:${delivery.id}:0`, + reason: "issue_dependencies_blocked", status: "skipped", runId: null, + payload: expect.objectContaining({ chatCompletionDeliveryIds: [delivery.id] }) }), + ])); + expect(receipts).toHaveLength(2); + expect(receipts.find(receipt => receipt.idempotencyKey?.startsWith("execution-wait:"))) + .toMatchObject({ coalescedCount: 0 }); + expect((await f.rows())[0]).toMatchObject({ attempts: 0, status: "pending", targetRunId: null, error: null }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(1); + expect(await issueService(db).getDependencyReadiness(f.sourceId)).toMatchObject({ + isDependencyReady: false, unresolvedBlockerIssueIds: [blocker.id], + }); + }); + it("admits a saved worker completion after its owning finalize encloses the snapshot", async () => { + const f = await seed(), workspaceId = randomUUID(), workerRunId = randomUUID(); + await db.update(issues).set({ responsibleUserId: "operator" }).where(eq(issues.id, f.sourceId)); + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, + mode: "isolated_workspace", strategyType: "managed_remote", name: "Saved worker", providerType: "daytona" }); + await db.insert(heartbeatRuns).values({ id: workerRunId, companyId: f.companyId, agentId: f.agentId, + status: "succeeded", contextSnapshot: { issueId: f.task.id } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.task.id)); + await issueService(db).update(f.sourceId, { status: "blocked", blockedByIssueIds: [f.task.id] }); + await db.insert(workspaceOperations).values([ + { companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId, + phase: "workspace_finalize", status: "succeeded", startedAt: new Date(10_000), finishedAt: new Date(12_000) }, + { companyId: f.companyId, issueId: f.task.id, executionWorkspaceId: workspaceId, heartbeatRunId: workerRunId, + phase: "remote_workspace_snapshot", status: "succeeded", startedAt: new Date(10_100), finishedAt: new Date(11_900) }, + ]); + await f.finish(); + const deps = createHeartbeatQueueTestDependencies(db), issueSvc = issueService(db); + deps.getAgent.mockImplementation(id => db.select().from(agents).where(eq(agents.id, id)).then(rows => rows[0] ?? null)); + deps.instanceSettings.getExperimental.mockImplementation(() => instanceSettingsService(db).getExperimental()); + deps.issuesSvc.getDependencyReadiness.mockImplementation(issueSvc.getDependencyReadiness); + deps.issuesSvc.listDependencyReadiness.mockImplementation(issueSvc.listDependencyReadiness); + // Admit the real durable run, then hold provider dispatch at the normal drain boundary. + deps.getSchedulingSuppression.mockResolvedValue({ suppressed: true, reason: "task_drain" }); + const queue = createHeartbeatQueue(db, deps); + const service = chatCompletionDeliveryService(db, { wakeup: queue.trackWakeup }); + const [delivery] = await f.rows(); + await service.deliver(delivery.id); + const [queued] = await f.rows(); + expect(queued).toMatchObject({ status: "queued", attempts: 0, error: null, targetRunId: expect.any(String) }); + const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, queued.targetRunId!)); + expect(run).toMatchObject({ status: "queued", contextSnapshot: { issueId: f.sourceId, + wakeReason: "chat_task_completed", chatCompletionDeliveryIds: [delivery.id] } }); + const prepared = await prepareChatCompletionTurn(db, run); + expect(prepared.contextSnapshot?.chatCompletionUpdates).toEqual([ + expect.objectContaining({ id: f.task.id, status: "done", completedAt: expect.any(Date), url: `/issues/${f.task.identifier ?? f.task.id}` }), + ]); + expect(deps.executeRun).not.toHaveBeenCalled(); + }); it("dispatches a committed completion without activity publication and recovers after restart", async () => { const f = await seed(); const run = vi.fn(() => f.service.sweepPending()); diff --git a/server/src/__tests__/company-skills-service.test.ts b/server/src/__tests__/company-skills-service.test.ts index 8da12bed840..6ee86511ad7 100644 --- a/server/src/__tests__/company-skills-service.test.ts +++ b/server/src/__tests__/company-skills-service.test.ts @@ -17,7 +17,7 @@ import { projectWorkspaces, } from "@paperclipai/db"; import { parseFrontmatterMarkdown } from "@paperclipai/shared"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase, @@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => { ]); await svc.list(companyId); const events: Record[] = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { agents, costEvents }, capture); let listed: Awaited>; try { listed = await svc.list(companyId); } finally { capture.flush(); } expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2); expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1); expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0); - const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents"); + const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents"); expect(agentReads).toHaveLength(1); expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000); - expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0); + expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0); await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId)); const refreshed = await svc.list(companyId, { q: "First" }); expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1); @@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => { const before = await svc.list(companyId); expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core"); const events: Record[] = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkills }, capture); let after: Awaited>; try { after = await svc.list(companyId); } finally { capture.flush(); } expect(after).toEqual(before); - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills"); + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills"); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000); const stored = await svc.getById(companyId, inserted[0]!.id); expect(stored?.markdown).toBe(savedMarkdown); @@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => { .orderBy(companySkillVersions.id); expect(before.length).toBeGreaterThan(0); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkillVersions }, capture); try { expect(await companySkillService(db).list(companyId)).toEqual(expected); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skill_versions" && event.operation === "select"); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096); @@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => { ? { type, companyId: otherCompanyId, agentId: randomUUID() } : { type, companyIds: [otherCompanyId] }; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { companySkillVersions }, capture); try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); } finally { capture.flush(); } - expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0); + expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0); }); it("restores a deleted release after warm validation", async () => { diff --git a/server/src/__tests__/connection-management.integration.test.ts b/server/src/__tests__/connection-management.integration.test.ts index 103e81dd964..a1f3f3e301d 100644 --- a/server/src/__tests__/connection-management.integration.test.ts +++ b/server/src/__tests__/connection-management.integration.test.ts @@ -6,9 +6,10 @@ import { join } from "node:path"; import { createServer } from "node:http"; import { and, eq, sql } from "drizzle-orm"; import { beforeAll, afterAll, describe, expect, it, vi } from "vitest"; -import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db"; +import { createDb, authUsers, activityLog, agentApiKeys, companies, companySecrets, companySecretVersions, chatConversations, chatDeliveries, chatEndpoints, chatExternalPrincipals, chatIdentityLinks, toolPolicies, toolRateLimitCounters, connectionGrants, connectionGrantMembers, connectionGrantDelegations, toolAccessAuditEvents, toolOauthStates, issueThreadInteractions, connectionIntentDeliveries, agents, companyMemberships, heartbeatRuns, issues, principalPermissionGrants, projects, toolApplications, toolConnections, toolConnectionInstalls, aiConnectionPools, toolMcpGateways, toolMcpGatewayTokens, toolActionRequests, toolCatalogEntries, toolInvocations, toolCallEvents, toolProfiles, toolProfileBindings, plugins, adapterAuthSessions, environments, routines, chatGitHubReviews } from "@paperclipai/db"; import { findSatisfiedToolConnection, satisfiedConnectionIntentService } from "../services/satisfied-connection-intents.js"; import { connectionManagementService, continuingConnectionManagement } from "../services/connection-management.js"; +import { aiConnectionService } from "../services/ai-connections.js"; import { redactRunnerApiPayloads } from "../services/native-runtime/runner-api-log-redaction.js"; import * as remoteEndpointGuard from "../services/remote-http-endpoint-guard.js"; import * as remoteHttpFetch from "../services/remote-http-fetch.js"; @@ -87,13 +88,14 @@ describe("connection management with production default agents", () => { authority: new PaperclipRunnerToolAuthority(server.db, { companyId: base.companyId, agentId: agent.id, issueId: base.issueId, runId: base.runId, apiUrl: server.apiUrl, storage: server.storage }) }; } - it("matches ordinary install capabilities on real PostgreSQL for scoped, retained and fallback resources", async () => { + it.each(["tool", "ai"] as const)("matches ordinary %s install capabilities on real PostgreSQL for scoped, retained and fallback resources", async (connectionPurpose) => { const f = await server.fixture({ connectionScenario: "fresh" }); const candidateIds = [f.agentId, randomUUID(), randomUUID()]; await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Capability target ${index}`, adapterType: "process" }))); const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Capability differential", type: "mcp_http" }).returning(); const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, - uid: randomUUID(), name: "Capability differential", transport: "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning(); + uid: randomUUID(), name: "Capability differential", connectionPurpose, + transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: "different-creator" }).returning(); await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!, permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds.slice(0, 2) } }); const service = connectionManagementService(server.db); @@ -177,7 +179,7 @@ describe("connection management with production default agents", () => { expect(await faulting.describeAgentInstallCapabilities(localActor, resource, [candidateIds[1]])).toEqual([denied]); }); - it("preserves the complete grants GET across human roles and denies a real install mutation after revocation", async () => { + it.each(["tool", "ai"] as const)("preserves the complete %s grants GET across human roles and denies a real install mutation after revocation", async (connectionPurpose) => { const f = await server.fixture({ connectionScenario: "fresh" }); const candidateIds = [f.agentId, randomUUID(), randomUUID()]; await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `Grant target ${index}`, adapterType: "process" }))); @@ -198,7 +200,8 @@ describe("connection management with production default agents", () => { scope: permissionKey === "tools:manage_profiles" ? { targetAgentIds: candidateIds.slice(0, 2) } : null }))); const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "Grants response parity", type: "mcp_http" }).returning(); const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, - uid: randomUUID(), name: "Grants response parity", transport: "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning(); + uid: randomUUID(), name: "Grants response parity", connectionPurpose, + transport: connectionPurpose === "ai" ? "runtime_auth" : "mcp_remote", authKind: "none", createdByUserId: users.creator }).returning(); const grants = await server.db.insert(connectionGrants).values([ { companyId: f.companyId, connectionId: connection.id, kind: "organization", createdByUserId: users.creator, isDefault: true }, ...[users.creator, users.manager, users.member].map((subjectUserId, index) => ({ companyId: f.companyId, connectionId: connection.id, kind: "user" as const, @@ -253,6 +256,65 @@ describe("connection management with production default agents", () => { expect(savedInstalls).toHaveLength(2); }); + it("checks company-wide personal AI installs once per target and preserves pool member authorization", async () => { + const f = await server.fixture({ connectionScenario: "fresh" }); + const candidateIds = [f.agentId, ...Array.from({ length: 19 }, () => randomUUID())]; + await server.db.insert(agents).values(candidateIds.slice(1).map((id, index) => ({ id, companyId: f.companyId, name: `AI target ${index}`, adapterType: "process" }))); + const [application] = await server.db.insert(toolApplications).values({ companyId: f.companyId, name: "AI capability checks", type: "mcp_http" }).returning(); + const [connection] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, + uid: randomUUID(), name: "Personal AI account", transport: "runtime_auth", connectionPurpose: "ai", authKind: "api_key", + credentialPolicy: "per_user", createdByUserId: f.responsibleUserId }).returning(); + const [grant] = await server.db.insert(connectionGrants).values({ companyId: f.companyId, connectionId: connection.id, + kind: "user", subjectUserId: f.responsibleUserId, createdByUserId: f.responsibleUserId }).returning(); + await server.db.insert(toolConnectionInstalls).values({ companyId: f.companyId, connectionId: connection.id, targetType: "company", targetId: f.companyId }); + await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!, + permissionKey: "tools:manage_profiles", scope: { targetAgentIds: candidateIds } }); + let targetReads = 0; + const observedDb = new Proxy(server.db, { get(target, key, receiver) { + if (key !== "select") return Reflect.get(target, key, receiver); + return (...fields: Parameters) => { + const builder = target.select(...fields), from = builder.from.bind(builder); + builder.from = ((...args: Parameters) => { + const query = from(...args), execute = query.execute.bind(query); + query.execute = (async (...executeArgs: Parameters) => { + if (query.toSQL().sql.includes('"agents"."id" =')) targetReads++; + return execute(...executeArgs); + }) as typeof execute; + return query; + }) as typeof builder.from; + return builder; + }; + } }); + const service = connectionManagementService(observedDb); + const actor: AuthorizationActor = { type: "board", userId: f.responsibleUserId, source: "cloud_tenant" }; + const resource = { type: "connection_management" as const, companyId: f.companyId, connectionId: connection.id, operation: "configure", readOnly: false }; + const baseline = []; + for (const targetAgentId of candidateIds) baseline.push(await service.capability(actor, "connections:profiles", { ...resource, targetAgentId })); + expect(baseline.every(result => result.allowed)).toBe(true); + expect(targetReads).toBe(candidateIds.length ** 2); + targetReads = 0; + expect(await service.describeAgentInstallCapabilities(actor, resource, candidateIds)).toEqual(baseline); + expect(targetReads).toBe(candidateIds.length); + + const [pool] = await server.db.insert(toolConnections).values({ companyId: f.companyId, applicationId: application.id, + uid: randomUUID(), name: "Pool with private member", transport: "runtime_auth", connectionPurpose: "ai", authKind: "none" }).returning(); + await server.db.insert(aiConnectionPools).values({ id: pool.id, companyId: f.companyId, pluginKey: "fixture-pool", config: { + name: "Pool", enabled: true, mode: "round_robin", thresholdPercent: 90, + members: [{ id: randomUUID(), binding: { mode: "delegated", provider: "openai", method: "api_key", connectionId: connection.id, grantId: grant.id }, profile: { provider: "codex", model: "fixture-model" } }], + } }); + await server.db.delete(principalPermissionGrants).where(and(eq(principalPermissionGrants.companyId, f.companyId), eq(principalPermissionGrants.principalId, f.responsibleUserId!))); + await server.db.insert(principalPermissionGrants).values({ companyId: f.companyId, principalType: "user", principalId: f.responsibleUserId!, + permissionKey: "tools:manage_profiles", scope: { connectionIds: [pool.id] } }); + const poolResource = { ...resource, connectionId: pool.id }; + expect((await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).every(result => result.allowed)).toBe(true); + // The pool grant cannot replace the member account's own authority. + await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, grant.id)); + await server.db.update(toolConnections).set({ createdByUserId: "different-owner" }).where(eq(toolConnections.id, connection.id)); + const ordinary = await service.capability(actor, "connections:profiles", { ...poolResource, targetAgentId: candidateIds[0] }); + expect(ordinary.allowed).toBe(false); + expect(await service.describeAgentInstallCapabilities(actor, poolResource, candidateIds.slice(0, 2))).toEqual([ordinary, ordinary]); + }); + it("creates and configures tool connections and profiles with fresh grants and actual key/JWT authentication", async () => { const f = await fixture(); const grants = await server.db.select().from(principalPermissionGrants).where(eq(principalPermissionGrants.principalId, f.agentId)); @@ -1642,6 +1704,36 @@ describe("connection management with production default agents", () => { } finally { provider = null; unregisterServerAdapter("paperclip_runner"); } }); + it("reads only the authorized Anthropic account identity through actual agent authentication", async () => { + const f = await fixture(); + const account = await aiConnectionService(server.db).save(f.companyId, f.userId, { + provider: "anthropic", method: "subscription", ownership: "personal", name: "Account identity fixture", + loginSessionId: "fixture", agentIds: [f.agentId], allAgents: false, + }, "selected-identity-credential"); + let calls = 0; + provider = async (url, init) => { + calls++; + expect(url).toBe("https://api.anthropic.com/api/oauth/profile"); + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer selected-identity-credential"); + return Response.json({ account: { uuid: "private-seat", email: "avery@example.test" }, organization: { uuid: "private-org", name: "Example team" } }); + }; + const path = `/companies/${f.companyId}/ai-connections/${account.connectionId}/identity?grantId=${account.grantId}`; + try { + for (const token of [f.key.token, f.jwt]) { + const response = await f.request("GET", path, undefined, token); + expect(response).toMatchObject({ status: 200, body: { ...account, status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" } }); + for (const secret of ["selected-identity-credential", "private-seat", "private-org"]) expect(JSON.stringify(response)).not.toContain(secret); + } + expect(calls).toBe(2); + expect((await f.request("GET", path.replace(f.companyId, f.foreignCompanyId))).status).toBe(403); + expect((await f.request("GET", path.replace(account.grantId, randomUUID()))).status).toBe(404); + expect(calls).toBe(2); + await server.db.update(connectionGrants).set({ status: "revoked" }).where(eq(connectionGrants.id, account.grantId)); + expect(await f.request("GET", path)).toMatchObject({ status: 200, body: { status: "unavailable", errorCode: "connection_unavailable", accountLabel: null } }); + expect(calls).toBe(2); + } finally { provider = null; } + }); + it("filters AI account and pool listings by the agent's resolved connection scope", async () => { const f = await fixture(); provider = async url => { diff --git a/server/src/__tests__/conversation-settlement-transfer.test.ts b/server/src/__tests__/conversation-settlement-transfer.test.ts index 6a128b00e95..82b84ed75f3 100644 --- a/server/src/__tests__/conversation-settlement-transfer.test.ts +++ b/server/src/__tests__/conversation-settlement-transfer.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { settleConversationTurnForRun } from "../services/agent-conversations.js"; @@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport(); it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => { const input = await seed(reply), before = await sourceDigests(input); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture); try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table)) && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4); diff --git a/server/src/__tests__/database-operation-attribution.test.ts b/server/src/__tests__/database-operation-attribution.test.ts new file mode 100644 index 00000000000..41144ae6b27 --- /dev/null +++ b/server/src/__tests__/database-operation-attribution.test.ts @@ -0,0 +1,137 @@ +import express from "express"; +import request from "supertest"; +import { QueryPromise } from "drizzle-orm/query-promise"; +import { afterEach, expect, it, vi } from "vitest"; +import { withDatabaseOperation } from "@paperclipai/db"; +import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals"; +import { databaseOperationMiddleware } from "../middleware/database-operation.js"; +import { withBudgetSpan } from "../services/budget-tracing.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js"; +import { + createRunPreparationTiming, + measureRunPhase, + withRunExecutionTiming, + withRunResponsePresentationTiming, + withRunWorkspaceFinalizationTiming, +} from "../services/execution-preparation-timing.js"; + +const { owners } = await vi.hoisted(async () => { + const { AsyncLocalStorage } = await import("node:async_hooks"); + return { owners: new AsyncLocalStorage() }; +}); +vi.mock("@paperclipai/db", async importOriginal => ({ + ...await importOriginal(), + withDatabaseOperation: (owner: string, action: () => T): T => owners.run(owner, action), +})); + +class OwnerQuery extends QueryPromise { + async execute() { return owners.getStore(); } +} + +afterEach(() => { + vi.unstubAllEnvs(); + vi.useRealTimers(); +}); + +it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); + await withDatabaseOperation("http.request", async () => { + const query = new OwnerQuery(); + await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt"); + await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization"); + await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation"); + await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read"); + await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup"); + await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory"); + await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction"); + await expect(query).resolves.toBe("http.request"); + }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test"); + const failure = new Error("private accounting failure"); + await withRunExecutionTiming("private-run", async () => { + for (let i = 0; i < 18; i++) { + await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory"); + } + await expect(measureRunPhase("execution.budget_reservation", async () => { + await expect(withBudgetSpan("budget.admission.reserve", async () => { + await Promise.resolve(); + expect(owners.getStore()).toBe("budget.admission.reserve"); + throw failure; + })).rejects.toBe(failure); + return new OwnerQuery(); + })).resolves.toBe("execution.budget_reservation"); + await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt"); + }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", ""); + const app = express(); + app.use(databaseOperationMiddleware); + app.use(express.json()); + let arrived = 0; + let release!: () => void; + const both = new Promise(resolve => { release = resolve; }); + app.all("/:privateId", async (req, res) => { + const before = await new OwnerQuery(); + const work = async () => { + if (++arrived === 2) release(); + await both; + return new OwnerQuery(); + }; + const during = req.params.privateId === "first" + ? await withBudgetSpan("budget.admission.reserve", work) + : await measureRunPhase("skills.inventory", work); + res.json({ before, during, after: await new OwnerQuery() }); + }); + const [first, second] = await Promise.all([ + request(app).get("/first?secret=private-first").expect(200), + request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200), + ]); + expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" }); + expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" }); + expect(owners.getStore()).toBeUndefined(); +}); + +it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => { + vi.useFakeTimers(); + const execute = vi.fn(async () => [{ xid: "42" }]); + const database = installDatabaseWorkSignals({ execute, async transaction(action: (tx: object) => Promise): Promise { + return action({ execute }); + } }); + const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = []; + const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true, + onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); }, + }); + let fail = true; + try { + const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue => + coordinator.register(queue, { retryMs: 5000, + run: async () => { + await Promise.resolve(); + seen.push({ queue, phase: "run", owner: await new OwnerQuery() }); + if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); } + }, + hasPending: async () => { + seen.push({ queue, phase: "pending", owner: await new OwnerQuery() }); + return false; + }, + }))); + await Promise.all(workers.map(worker => worker.ready)); + await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx => + notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback))); + await vi.advanceTimersByTimeAsync(1); + expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]); + for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`); + expect(owners.getStore()).toBeUndefined(); + expect(coordinator.nextWakeAt()).toBeNull(); + } finally { + await coordinator.stop(); + } +}); diff --git a/server/src/__tests__/environment-cleanup-work.test.ts b/server/src/__tests__/environment-cleanup-work.test.ts index 29dbf953aae..53b20a29967 100644 --- a/server/src/__tests__/environment-cleanup-work.test.ts +++ b/server/src/__tests__/environment-cleanup-work.test.ts @@ -1,5 +1,5 @@ import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; -import { createOwnedSandbox } from "../services/sandbox-creation-operations.js"; +import { createOwnedSandbox, recordSandboxCreationResource } from "../services/sandbox-creation-operations.js"; import { mkdtemp, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -32,8 +32,8 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { const errors: unknown[] = [], disposers: Array<() => void> = []; beforeAll(async () => { if (!external) database = await startEmbeddedPostgresTestDatabase("environment-cleanup-work-"); - db = createDb(external ?? database!.connectionString, { maxConnections: 6, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); - await Promise.all(Array.from({ length: 6 }, () => db.execute(sql`select pg_sleep(0.01)`))); + db = createDb(external ?? database!.connectionString, { maxConnections: 4, idleTimeoutSeconds: 0, maxLifetimeSeconds: 0 }); + await Promise.all(Array.from({ length: 4 }, () => db.execute(sql`select pg_sleep(0.01)`))); }, 90_000); beforeEach(async () => { await db.execute(sql`TRUNCATE sandbox_creation_operations, companies, environments CASCADE`); @@ -48,6 +48,9 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { afterEach(async () => { worker?.stop(); await coordinator?.stop(); coordinator = undefined; worker = undefined; for (const stop of disposers.splice(0)) stop(); + // Creation ownership intentionally survives company deletion; company + // cascades therefore cannot isolate this case from the next test suite. + await db.delete(sandboxCreationOperations).where(eq(sandboxCreationOperations.companyId, companyId)); vi.useRealTimers(); vi.restoreAllMocks(); expect(errors).toEqual([]); }); @@ -116,15 +119,18 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { ])); await quiet(); }); - it("settles a deleted-company creation as manual history across restart without idle polling", async () => { + it.each([false, true])("retains deleted-company creation uncertainty across a quiet worker restart (contextual=%s)", async contextual => { await start(); await quiet(); - await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, metadata: {} }, + const created = await createOwnedSandbox(db, { companyId, environmentId, provider: "fixture", driverKey: "fixture", config: {}, + metadata: contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {} }, async () => ({ providerLeaseId: "fixture-deleted-company" })); await db.delete(companies).where(eq(companies.id, companyId)); await settle(31 * 60_000); await vi.waitFor(async () => expect(await db.select().from(sandboxCreationOperations)).toEqual([ expect.objectContaining({ state: "needs_reconciliation", metadata: expect.objectContaining({ creationReconciliationReason: "company_deleted" }) }), ])); + if (contextual) await recordSandboxCreationResource(db, { id: String(created.metadata!.creationOperationId), companyId, environmentId, + providerLeaseId: created.providerLeaseId }); await quiet(); worker!.stop(); await coordinator!.stop(); await start(); await quiet(); @@ -132,7 +138,7 @@ describe.skipIf(!support.supported)("environment cleanup scheduling", () => { const ownerId = randomUUID(); const hold = { ownerId, draining: true, startedAt: new Date(Date.now() - 1000), expiresAt: new Date(Date.now() + 60_000), activeRuns: 0, pendingWakes: 0 }; expect(await readIdleSleepSafety(db, () => hold, Date.now, ownerId, async () => "none")) - .toEqual({ version: 1, backgroundWork: "none" }); + .toEqual({ version: 1, backgroundWork: "present" }); await quiet(); }); it("verifies ordinary cleanup carrying creation provenance and stops provider polling", async () => { diff --git a/server/src/__tests__/environment-task-authority.integration.test.ts b/server/src/__tests__/environment-task-authority.integration.test.ts index d01611e1972..9e89c49901c 100644 --- a/server/src/__tests__/environment-task-authority.integration.test.ts +++ b/server/src/__tests__/environment-task-authority.integration.test.ts @@ -1,11 +1,12 @@ import { randomUUID } from "node:crypto"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { eq } from "drizzle-orm"; -import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins } from "@paperclipai/db"; +import { agents, companies, createDb, environmentLeases, environments, heartbeatRuns, plugins, sandboxCreationOperations } from "@paperclipai/db"; import { resolvePluginTaskResume } from "../services/environment-task-runtime.js"; import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; import { environmentRuntimeService } from "../services/environment-runtime.js"; import type { PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; const support = await getEmbeddedPostgresTestSupport(); @@ -20,6 +21,108 @@ databaseSuite("environment task lease authority", () => { }); afterAll(async () => { await stopDb?.(); }); + it("replaces a stopped warm candidate within its tracked startup context", async () => { + const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" }); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never }); + await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin", + config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId, + provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment", + metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } }); + const calls: string[] = []; + const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }), + call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void) => { + calls.push(method === "environmentTask" ? params.operation.kind : method); + try { + if (method === "environmentAcquireLease") return { providerLeaseId: "replacement-task" }; + if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true }; + if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId, + runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 }; + return { kind: "accepted", taskId: params.taskId }; + } finally { settled?.(); } + }, + } as unknown as PluginWorkerManager; + const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000); + try { + await expect(environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId, + operation: { kind: "describe" }, executionContext: context })).resolves.toMatchObject({ kind: "description", taskId: "replacement-task" }); + expect(calls).toEqual(["status", "status", "complete", "environmentAcquireLease", "describe"]); + expect(context.pending).toBe(false); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)); + expect(lease.providerLeaseId).toBe("replacement-task"); + expect(lease.metadata?.runnerTaskReused).toBe(false); + expect(JSON.stringify(lease.metadata)).not.toContain(context.id); + } finally { context.finish(); } + }); + + it.each(["late", "post-result", "retiring"])("queues a replacement instead of attaching after %s startup cancellation", async mode => { + const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(), runId = randomUUID(), leaseId = randomUUID(); + await db.insert(companies).values({ id: companyId, name: "Warm candidate", issuePrefix: `T${companyId.slice(0, 7).replace(/-/g, "").toUpperCase()}` }); + await db.insert(agents).values({ id: agentId, companyId, name: "Fixture", adapterType: "paperclip_runner" }); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-warm-replacement", version: "1", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never }); + await db.insert(environments).values({ id: environmentId, companyId, name: `Fixture ${environmentId}`, driver: "plugin", + config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "manual", status: "running" }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId, heartbeatRunId: runId, + provider: "plugin:fixture:tasks", providerLeaseId: "stopped-task", status: "active", leasePolicy: "reuse_by_environment", + metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, runnerTaskReused: true } }); + const calls: string[] = []; + let reply!: () => Promise, acquiring!: () => void; + const entered = new Promise(resolve => { acquiring = resolve; }); + const workers = { getWorker: () => ({ supportedMethods: ["environmentTask", "environmentAcquireLease"] }), + call: async (_plugin: string, method: string, params: any, _timeout?: number, _sink?: unknown, settled?: () => void, _contextId?: string, observe?: (result: any) => Promise) => { + calls.push(method === "environmentTask" ? params.operation.kind : method); + try { + if (method === "environmentAcquireLease") return await new Promise(resolve => { + reply = async () => { const result = { providerLeaseId: "replacement-task", metadata: { binding: "fresh" } }; + await observe?.(result); + if (mode === "retiring") await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId, driverKey: "tasks", runnerTaskRetiring: true } }).where(eq(environmentLeases.id, leaseId)); + resolve(result); }; + acquiring(); + }); + if (params.operation.kind === "status") return { kind: "status", taskId: params.taskId, phase: "completed", executionStopped: true }; + if (params.operation.kind === "describe") return { kind: "description", taskId: params.taskId, + runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` }, executionContextVersion: 1 }; + return { kind: "accepted", taskId: params.taskId }; + } finally { settled?.(); } + }, + } as unknown as PluginWorkerManager; + const context = new NativeExecutionContext({ binding: { companyId, agentId, runId } } as never, 1_000); + if (mode === "post-result") { + const wait = context.wait.bind(context); + vi.spyOn(context, "wait").mockImplementation(operation => wait(async () => { + const value = await operation(); context.abort(new Error("Cancelled after recording")); return value; + })); + } + try { + const request = environmentRuntimeService(db, { pluginWorkerManager: workers }).task({ companyId, leaseId, + operation: { kind: "describe" }, executionContext: context }); + const rejected = expect(request).rejects.toThrow("operation unavailable"); + await entered; + if (mode === "late") { + context.abort(new Error("Startup cancelled")); + await rejected; + expect(context.pending).toBe(true); + } + await reply(); await rejected; + expect(context.pending).toBe(false); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)); + expect(lease.providerLeaseId).toBe("stopped-task"); + const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.runId, runId)); + expect(creation.state).toBe("cleanup_pending"); + expect(creation.metadata.runnerTaskCreationContextId).toBe(context.id); + const [cleanup] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, creation.leaseId!)); + expect(cleanup).toMatchObject({ heartbeatRunId: runId, providerLeaseId: "replacement-task", status: "pending_cleanup" }); + expect(cleanup.metadata).toMatchObject({ runnerTaskLease: true, providerMetadata: { binding: "fresh" } }); + expect(calls).not.toContain("describe"); + } finally { context.finish(); } + }); + it("authorizes a new task lease only after scoped prior execution has stopped", async () => { const companyId = randomUUID(), agentId = randomUUID(), environmentId = randomUUID(), pluginId = randomUUID(); const oldRunId = randomUUID(), nextRunId = randomUUID(), oldLeaseId = randomUUID(), nextLeaseId = randomUUID(); diff --git a/server/src/__tests__/environment-task-runtime.test.ts b/server/src/__tests__/environment-task-runtime.test.ts index 15c8dcb7b62..96126fc6df8 100644 --- a/server/src/__tests__/environment-task-runtime.test.ts +++ b/server/src/__tests__/environment-task-runtime.test.ts @@ -2,14 +2,16 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { PgDialect } from "drizzle-orm/pg-core"; import { executionWorkspaces, environmentLeases, type Db } from "@paperclipai/db"; import { environmentTaskOperationSchema, parseEnvironmentTaskResult } from "@paperclipai/plugin-sdk"; -import { executeEnvironmentTask } from "../services/environment-task-runtime.js"; +import { executeEnvironmentTask, settleDispatchedEnvironmentTask } from "../services/environment-task-runtime.js"; +import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js"; -const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn() })); +const state = vi.hoisted(() => ({ plugin: {} as any, release: vi.fn(), identity: vi.fn(), config: vi.fn() })); vi.mock("../services/provider-fresh-workspace.js", () => ({ stageProviderFreshWorkspace: async (input: { existing: unknown }) => input.existing, releaseProviderRecoveryTransfers: state.release, })); -vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin) }) })); +vi.mock("../services/plugin-registry.js", () => ({ pluginRegistryService: () => ({ getById: vi.fn(async () => state.plugin), getConfig: state.config }) })); +vi.mock("../services/plugin-agent-runtime-state.js", () => ({ pluginAgentRuntimeIdentity: state.identity })); const leaseId = "10000000-0000-4000-8000-000000000001"; const projectId = "10000000-0000-4000-8000-000000000002"; const secondProjectId = "10000000-0000-4000-8000-000000000003"; @@ -25,7 +27,7 @@ function database(value: ReturnType | null = row(), projectRows = [{ const query: any = { from: (table: unknown) => { selectedRows = table === executionWorkspaces ? workspaceRows : table === environmentLeases ? ownerRows : projectRows; return query; }, leftJoin: () => query, where: vi.fn(() => query), limit: () => Promise.resolve(results.shift()), then: (resolve: (rows: typeof projectRows) => unknown) => Promise.resolve(selectedRows).then(resolve as (rows: unknown[]) => unknown) }; const sets: Record[] = []; const update: any = { set: (value: Record) => { sets.push(value); return update; }, where: () => update, - returning: async () => [{ id: leaseId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) }; + returning: async () => [{ id: leaseId, metadata: value?.lease.metadata, heartbeatRunId: value?.lease.heartbeatRunId }], then: (resolve: (value: unknown) => unknown) => Promise.resolve([]).then(resolve) }; const db = { select: () => query, update: () => update, transaction: vi.fn(async (write: (tx: Db) => Promise) => write(db as unknown as Db)) }; return { db: db as unknown as Db, query, sets, transaction: db.transaction }; @@ -37,9 +39,65 @@ const submit = { kind: "submit" as const, preparedLaunchId: "configuration", pro beforeEach(() => { state.release.mockReset(); + state.identity.mockReset().mockResolvedValue({ instanceSecrets: { launcher: "synthetic-transient-binding" } }); + state.config.mockReset().mockResolvedValue({ configJson: { endpoint: "synthetic" } }); state.plugin = { id: "original", pluginKey: "test.provider", status: "ready", manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } }; }); describe("plugin-provided Runner execution", () => { + it.each([false, true])("retires an undispatched warm task only with required preparation proof (%s)", async preparationStopped => { + const value = row(); + Object.assign(value.lease.metadata, { runnerTaskDispatched: false, + runnerTaskStartupCleanup: { contextId: "startup", requiresPreparationStop: true } }); + const fixture = database(value), workers = worker(); + fixture.db.select = ((...args: unknown[]) => (database(value).db.select as (...values: unknown[]) => unknown)(...args)) as Db["select"]; + let release!: () => void; + const drain = new Promise(resolve => { release = resolve; }); + const drainStartup = vi.fn(() => drain); + Object.assign(workers, { drainStartup }); + const operations: string[] = []; + workers.call.mockImplementation(async (...args: any[]) => { + const operation = args[2].operation.kind; operations.push(operation); + return operation === "status" ? { kind: "status", taskId: "attempt-1", phase: "cancelled", executionStopped: true, preparationStopped } + : { kind: "accepted", taskId: "attempt-1" }; + }); + const retiring = settleDispatchedEnvironmentTask(fixture.db, workers as never, { companyId: "company", id: leaseId }); + const checked = retiring.then(value => ({ value, error: null }), error => ({ value: false, error })); + await vi.waitFor(() => expect(operations).toContain("stop")); + await vi.waitFor(() => expect(operations).toContain("status")); + expect(drainStartup).toHaveBeenCalledWith("startup"); + expect(operations).not.toContain("complete"); + release(); + const result = await checked; + if (preparationStopped) { expect(result).toEqual({ value: true, error: null }); expect(operations).toEqual(["stop", "status", "complete"]); } + else { expect(result.error?.message).toContain("termination is unconfirmed"); expect(fixture.sets).toContainEqual(expect.objectContaining({ status: "pending_cleanup", cleanupStatus: "failed" })); } + }); + it("keeps legacy payloads passive without capability or transport activation", async () => { + const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: "run" } } as never, 1_000); + const workers = worker(); + workers.call.mockImplementation(async (...args: any[]) => { args[5]?.(); return { kind: "description", taskId: "attempt-1", runner: { version: "fixture", digest: `sha256:${"a".repeat(64)}` } }; }); + try { + for (const executionContext of [undefined, context]) { + await executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, + operation: { kind: "describe" }, ...(executionContext ? { executionContext } : {}) }); + const sent = workers.call.mock.calls.at(-1) as unknown as any[]; + expect(sent[2]).not.toHaveProperty("executionContextSupported"); + expect(sent[2]).not.toHaveProperty("executionContext"); + if (!executionContext) expect(sent).toHaveLength(4); + } + expect(state.identity).not.toHaveBeenCalled(); + expect(state.config).not.toHaveBeenCalled(); + expect(context.pending).toBe(false); + } finally { context.finish(); } + }); + it.each(["json", "wrong-run"])("rejects a %s context before the worker", async mode => { + const context = new NativeExecutionContext({ binding: { companyId: "company", agentId: "agent", runId: mode === "wrong-run" ? "other" : "run" } } as never, 1_000); + const workers = worker(); + try { + await expect(executeEnvironmentTask(database().db, workers as never, { companyId: "company", leaseId, operation: { kind: "describe" }, + executionContext: mode === "json" ? JSON.parse(JSON.stringify(context)) : context })).rejects.toThrow("native_execution_context"); + expect(workers.call).not.toHaveBeenCalled(); + } finally { context.finish(); } + }); it("forwards a retained fresh directory intent on later launch calls and checks provider support", async () => { const value = row(); value.lease.metadata.runnerTaskProjectAuthValidation.projectIds = []; diff --git a/server/src/__tests__/github-operation-credentials.test.ts b/server/src/__tests__/github-operation-credentials.test.ts index 499134e2b55..75eeeacfcb5 100644 --- a/server/src/__tests__/github-operation-credentials.test.ts +++ b/server/src/__tests__/github-operation-credentials.test.ts @@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js"; import { errorHandler } from "../middleware/index.js"; import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, @@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }); await db.insert(toolConnections).values(irrelevant); const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { toolConnections }, capture); try { expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" })) @@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu }).where(eq(heartbeatRuns.id, input.runId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)); const observed = createDb(external ?? database!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({ diff --git a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts index 5be4f7bd2ff..652ef8e410c 100644 --- a/server/src/__tests__/heartbeat-dependency-scheduling.test.ts +++ b/server/src/__tests__/heartbeat-dependency-scheduling.test.ts @@ -689,14 +689,9 @@ describeEmbeddedPostgres("heartbeat dependency-aware queued run selection", () = }); expect(interactionWake).not.toBeNull(); - await waitForCondition(async () => { - const run = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, interactionWake!.id)) - .then((rows) => rows[0] ?? null); - return run?.status === "succeeded"; - }); + // This checks dependency interaction semantics, not provider startup speed. + // Join the admitted execution before inspecting its persisted outcome. + await heartbeat.drainActiveRunExecutions(); const interactionRun = await db .select({ diff --git a/server/src/__tests__/heartbeat-native-cleanup-admission.test.ts b/server/src/__tests__/heartbeat-native-cleanup-admission.test.ts index b6544447a68..86f4e8452c9 100644 --- a/server/src/__tests__/heartbeat-native-cleanup-admission.test.ts +++ b/server/src/__tests__/heartbeat-native-cleanup-admission.test.ts @@ -1,10 +1,15 @@ import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; -import { agents, authUsers, companies, companyMemberships, createDb, heartbeatRuns, issues } from "@paperclipai/db"; +import { agents, authUsers, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { heartbeatService } from "../services/heartbeat.js"; +import { NativeStartupCleanupPendingError, readNativeStartupCleanup } from "../services/native-runtime/native-startup-cleanup.js"; +import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js"; +import { environmentService } from "../services/environments.js"; +import { remoteTerminationReceipt } from "../services/remote-execution-termination.js"; +import { createIsolatedTestHome } from "./helpers/isolated-test-home.js"; const execute = vi.hoisted(() => vi.fn()); vi.mock("../adapters/index.js", async () => { @@ -70,4 +75,108 @@ suite("native terminal-run cleanup admission", () => { expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded"); expect(execute).toHaveBeenCalledTimes(3); }, 30_000); + + it("retries a failed startup intent through timer failure and drain before verified cancellation", async () => { + const home = createIsolatedTestHome(), originalHome = process.env.PAPERCLIP_HOME; + process.env.PAPERCLIP_HOME = home.home; + const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID(); + const contextId = randomUUID(), leaseOwner = randomUUID(), taskId = randomUUID(), deadlineAtMs = Date.now() - 1; + const originalSetTimeout = globalThis.setTimeout; + const retries: Array<{ timer: ReturnType; fire(): void }> = []; + const timeout = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback: (...args: unknown[]) => void, delay?: number, ...args: unknown[]) => { + if (typeof delay === "number" && delay <= 30_000 && new Error().stack?.includes("scheduleNativeSessionResumeDispatch")) { + const timer = originalSetTimeout(() => {}, 3_600_000); + retries.push({ timer, fire() { clearTimeout(timer); callback(...args); } }); + return timer; + } + return originalSetTimeout(callback, delay, ...args); + }) as typeof setTimeout); + let failSuppression = false, runId = ""; + const runtimeEnv = new Proxy>({}, { get(_target, key) { + if (key === "PAPERCLIP_DATABASE_RESTORE_IN_PROGRESS" && failSuppression) { + failSuppression = false; + throw new Error("Scheduling dependency unavailable"); + } + return undefined; + } }); + const heartbeat = heartbeatService(db, { runtimeEnv }); + try { + await db.insert(companies).values({ id: companyId, name: "Intent retry", issuePrefix: "CIR", defaultResponsibleUserId: "owner" }); + await db.insert(authUsers).values({ id: "owner", name: "Fixture user", email: "owner@example.test", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing(); + await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: "owner", status: "active", membershipRole: "member" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process", + adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true } } }); + await db.insert(issues).values({ id: issueId, companyId, title: "Recover startup", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" }); + const [environment] = await db.insert(environments).values({ companyId, name: "Controlled provider", driver: "plugin", config: {} }).returning(); + await db.execute(sql`create table test_cleanup_intent_fault (enabled boolean not null)`); + await db.execute(sql`insert into test_cleanup_intent_fault values (true)`); + await db.execute(sql`create sequence test_cleanup_intent_attempts`); + await db.execute(sql.raw(`create function test_reject_cleanup_intent() returns trigger language plpgsql as $$ begin + if new.company_id = '${companyId}'::uuid and new.error_code = 'native_startup_cleanup_pending' + and exists (select 1 from test_cleanup_intent_fault where enabled) then + perform nextval('test_cleanup_intent_attempts'); raise exception 'Cleanup intent write unavailable'; + end if; return new; end $$`)); + await db.execute(sql`create trigger test_cleanup_intent_fault before update on heartbeat_runs for each row execute function test_reject_cleanup_intent()`); + execute.mockReset(); + execute.mockImplementation(async (context: { runId: string }) => { + runId = context.runId; + const [run] = await db.update(heartbeatRuns).set({ runtimeMode: "native", runtimeModeResolvedAt: null }).where(eq(heartbeatRuns.id, runId)).returning(); + await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Recover startup", description: null }, environmentLeaseId: leaseId }); + await db.update(agents).set({ adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }).where(eq(agents.id, agentId)); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1, + controllerGeneration: 1, leaseOwner, leaseExpiresAt: new Date(Date.now() - 1_000) }); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId, + issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", metadata: { runnerTaskLease: true } }); + await db.update(heartbeatRuns).set({ runnerProfileJson: sql`coalesce(${heartbeatRuns.runnerProfileJson}, '{}'::jsonb) || ${JSON.stringify({ + nativeStartupContext: { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId, + contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 }, + })}::jsonb` }).where(eq(heartbeatRuns.id, runId)); + throw new NativeStartupCleanupPendingError(contextId, deadlineAtMs, leaseOwner, false, leaseId); + }); + await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual"); + await heartbeat.drainActiveRunExecutions(); + const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!); + const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!); + expect((await db.execute<{ last_value: string }>(sql`select last_value from test_cleanup_intent_attempts`))[0].last_value).toBe("2"); + expect(await heartbeat.getRun(runId)).toMatchObject({ status: "running", errorCode: null }); + expect((await owner()).leaseOwner).toBe(leaseOwner); + expect(await lease()).toMatchObject({ status: "active", cleanupStatus: null }); + expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)).toBeNull(); + const drainedRetries = retries.length; + expect(drainedRetries).toBeGreaterThan(0); + await heartbeat.resumeQueuedRunsAfterDrain(); + expect(retries.length).toBeGreaterThan(drainedRetries); + const resumedRetries = retries.length; + failSuppression = true; + retries.at(-1)!.fire(); + await vi.waitFor(() => expect(retries.length).toBeGreaterThan(resumedRetries)); + expect((await owner()).leaseOwner).toBe(leaseOwner); + expect((await heartbeat.getRun(runId))?.status).toBe("running"); + await db.execute(sql`update test_cleanup_intent_fault set enabled = false`); + retries.at(-1)!.fire(); + await vi.waitFor(async () => expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("pending")); + await vi.waitFor(async () => expect((await heartbeat.listEvents(runId)).some(event => event.eventType === "native.recovery.transition")).toBe(true)); + const current = await lease(); + await environmentService(db).releaseLease(leaseId, "released", { cleanupStatus: "success", + remoteExecutionTermination: remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }) }); + await heartbeat.cancelRun(runId, "Startup cleanup retry completed"); + expect(await heartbeat.getRun(runId)).toMatchObject({ status: "cancelled" }); + expect(readNativeStartupCleanup((await heartbeat.getRun(runId))!)?.state).toBe("complete"); + expect(await owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null }); + expect(execute).toHaveBeenCalledOnce(); + } finally { + await heartbeat.drainActiveRunExecutions(); + timeout.mockRestore(); + for (const retry of retries) clearTimeout(retry.timer); + await db.execute(sql`drop trigger if exists test_cleanup_intent_fault on heartbeat_runs`); + await db.execute(sql`drop function if exists test_reject_cleanup_intent()`); + await db.execute(sql`drop table if exists test_cleanup_intent_fault`); + await db.execute(sql`drop sequence if exists test_cleanup_intent_attempts`); + if (originalHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = originalHome; + home.cleanup(); + } + }, 30_000); + }); diff --git a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts index 98a002f8557..71328d1b64b 100644 --- a/server/src/__tests__/heartbeat-run-event-sequencing.test.ts +++ b/server/src/__tests__/heartbeat-run-event-sequencing.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { describe, expect, it } from "vitest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { agents, companies, @@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => { protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } }, }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture); try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); } finally { capture.flush(); } - const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute")); expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0); expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16); diff --git a/server/src/__tests__/heartbeat-run-log.test.ts b/server/src/__tests__/heartbeat-run-log.test.ts index 250db00acd7..18eb05b7354 100644 --- a/server/src/__tests__/heartbeat-run-log.test.ts +++ b/server/src/__tests__/heartbeat-run-log.test.ts @@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => { expect(compacted).not.toContain("paperclip-flag-secret"); }); }); + +describe("compactRunLogChunk with long base64 commands", () => { + it("redacts a tool call that writes base64 of source text without stalling", () => { + const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n"; + const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024); + expect(blob).toMatch(/^[A-Za-z0-9]+$/); + const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`; + const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`; + + const startedAt = performance.now(); + const compacted = compactRunLogChunk(chunk); + + expect(performance.now() - startedAt).toBeLessThan(1_000); + expect(compacted).toContain(blob); + expect(compacted).not.toContain("tool-call-secret"); + expect(compacted).toContain("***REDACTED***"); + }); +}); diff --git a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts index c610d17ded9..a610b5b25bd 100644 --- a/server/src/__tests__/heartbeat-run-privacy-routes.test.ts +++ b/server/src/__tests__/heartbeat-run-privacy-routes.test.ts @@ -1,7 +1,7 @@ import { randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import express from "express"; import request from "supertest"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; @@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false }, }).where(eq(heartbeatRuns.id, f.privateRunId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId)); const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const app = createApp(f.companyId, f.grantedAgentId, observed); diff --git a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts index b2bbe8b41a6..91eff8b4580 100644 --- a/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts +++ b/server/src/__tests__/heartbeat-stale-queue-invalidation.test.ts @@ -791,19 +791,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { `)); await heartbeat.resumeQueuedRuns(); - await waitForCondition(async () => { - const [run, wakeup] = await Promise.all([ - db.select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null), - db.select({ status: agentWakeupRequests.status }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, wakeupRequestId)) - .then((rows) => rows[0] ?? null), - ]); - return run?.status === "cancelled" && wakeup?.status === "skipped"; - }); + await heartbeat.drainActiveRunExecutions(); const [run, wakeup, issue] = await Promise.all([ db.select({ status: heartbeatRuns.status, errorCode: heartbeatRuns.errorCode }) @@ -970,16 +958,10 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { expect(ordering).toEqual(["validated", "parked"]); }; await heartbeat.resumeQueuedRuns(); - await waitForCondition(async () => { - const run = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - return run?.status === "cancelled"; - }); + await heartbeat.drainActiveRunExecutions(); await parkPromise; + expect((await heartbeat.getRun(runId))?.status).toBe("cancelled"); const issue = await db .select({ status: issues.status }) .from(issues) @@ -1108,7 +1090,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }); expect(run).not.toBeNull(); - await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0); + await heartbeat.drainActiveRunExecutions(); expect(countExecuteCallsForRun(run!.id)).toBe(1); }); @@ -1126,7 +1108,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }); expect(run).not.toBeNull(); - await waitForCondition(async () => countExecuteCallsForRun(run!.id) > 0); + await heartbeat.drainActiveRunExecutions(); expect(countExecuteCallsForRun(run!.id)).toBe(1); }); @@ -1718,13 +1700,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { .where(eq(agentWakeupRequests.id, wakeupRequestId)); await heartbeat.resumeQueuedRuns(); - await waitForCondition(async () => { - const [deferred] = await db - .select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId }) - .from(agentWakeupRequests) - .where(eq(agentWakeupRequests.id, deferredWakeupId)); - return Boolean(deferred?.runId) && deferred?.status !== "deferred_issue_execution"; - }); + await heartbeat.drainActiveRunExecutions(); const [deferred] = await db .select({ status: agentWakeupRequests.status, runId: agentWakeupRequests.runId }) @@ -1788,15 +1764,7 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => { }); await heartbeat.resumeQueuedRuns(); - - await waitForCondition(async () => { - const run = await db - .select({ status: heartbeatRuns.status }) - .from(heartbeatRuns) - .where(eq(heartbeatRuns.id, runId)) - .then((rows) => rows[0] ?? null); - return run?.status === "cancelled"; - }); + await heartbeat.drainActiveRunExecutions(); const [run, wakeup, issue] = await Promise.all([ db diff --git a/server/src/__tests__/helpers/embedded-postgres.ts b/server/src/__tests__/helpers/embedded-postgres.ts index a5088faa573..a5b1cda116f 100644 --- a/server/src/__tests__/helpers/embedded-postgres.ts +++ b/server/src/__tests__/helpers/embedded-postgres.ts @@ -1,35 +1,7 @@ -import { randomUUID } from "node:crypto"; -import { - applyPendingMigrations, - createDb, - getEmbeddedPostgresTestSupport as embeddedSupport, - startEmbeddedPostgresTestDatabase as startEmbedded, +export { + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, type EmbeddedPostgresTestDatabase, type EmbeddedPostgresTestSupport, } from "@paperclipai/db"; - -export type { EmbeddedPostgresTestDatabase, EmbeddedPostgresTestSupport }; - -export async function getEmbeddedPostgresTestSupport(): Promise { - return process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL ? { supported: true } : embeddedSupport(); -} - -export async function startEmbeddedPostgresTestDatabase(prefix: string): Promise { - const adminUrl = process.env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL; - if (!adminUrl) return startEmbedded(prefix); - const name = `paperclip_test_${randomUUID().replaceAll("-", "")}`; - const template = process.env.PAPERCLIP_TEST_POSTGRES_TEMPLATE; - if (template && !/^paperclip_test_template_[a-f0-9]+$/.test(template)) throw new Error("Invalid disposable test template name"); - const admin = createDb(adminUrl, { maxConnections: 1 }); - const url = new URL(adminUrl); - url.pathname = `/${name}`; - const cleanup = async () => { - try { await admin.$client.unsafe(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`); } - finally { await admin.$client.end({ timeout: 1 }); } - }; - try { - await admin.$client.unsafe(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ""}`); - if (!template) await applyPendingMigrations(url.toString()); - return { connectionString: url.toString(), cleanup }; - } catch (error) { await cleanup(); throw error; } -} diff --git a/server/src/__tests__/helpers/full-database-response-capture.test.ts b/server/src/__tests__/helpers/full-database-response-capture.test.ts new file mode 100644 index 00000000000..14a46c2ee8e --- /dev/null +++ b/server/src/__tests__/helpers/full-database-response-capture.test.ts @@ -0,0 +1,62 @@ +import { expect, it } from "vitest"; +import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js"; + +const metadata = { operation: "select", table: "heartbeat_runs", callers: [] }; + +it("measures every oversized string and complex value without sampling or truncation", () => { + const events: Array> = []; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0); + const text = "private-payload🙂".repeat(30_000); + const nested = { instructions: text, additional: [{ prompt: text }] }; + const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] }); + const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested)); + try { + for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows); + capture.flush(); + expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1", + calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes, + estimatedValues: 17, serializationErrors: 0, + }); + expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1", + startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes, + }); + expect(JSON.stringify(events)).not.toContain("private-payload"); + expect(nested.instructions).toBe(text); + } finally { capture.flush(); } +}); + +it("preserves asynchronous results and original query errors", async () => { + const events: Array> = []; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0); + const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] }); + const failure = new Error("private query failure"); + try { + await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows); + await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure); + expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure); + capture.flush(); + expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2, + responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0, + }); + expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 }); + expect(JSON.stringify(events)).not.toContain(failure.message); + } finally { capture.flush(); } +}); + +it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => { + const events: Array> = []; + let now = 0; + const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now); + let release!: (value: string[]) => void; + const original = new Promise(resolve => { release = resolve; }); + const pending = capture.execute(() => original, metadata); + now = 1000; + capture.flush(); + release(["result"]); + await expect(pending).resolves.toEqual(["result"]); + capture.flush(); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1", + startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0, + }); +}); diff --git a/server/src/__tests__/helpers/full-database-response-capture.ts b/server/src/__tests__/helpers/full-database-response-capture.ts new file mode 100644 index 00000000000..a52831c2adc --- /dev/null +++ b/server/src/__tests__/helpers/full-database-response-capture.ts @@ -0,0 +1,115 @@ +import { randomUUID } from "node:crypto"; +import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture"; + +export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; + +type Capture = ReturnType; +type Metadata = Parameters[1]; +type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; + queryErrors: number; serializationErrors: number; estimatedValues: number }; + +function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } { + const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined; + if (!Array.isArray(value) || !Array.isArray(columns)) { + const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); + return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 }; + } + let bytes = 0, estimatedValues = 0; + for (const row of value) { + const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]); + for (const cell of cells) { + if (cell === null || cell === undefined) continue; + if (typeof cell === "string") bytes += Buffer.byteLength(cell); + else if (typeof cell === "boolean") bytes += 1; + else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell)); + else { + estimatedValues += 1; + const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item); + if (serialized !== undefined) bytes += Buffer.byteLength(serialized); + } + } + } + return { bytes, estimatedValues }; +} + +export function createFullDatabaseResponseCapture( + window: ResponseCaptureWindow, + emit: (event: Record) => void, + now: () => number = Date.now, +): Capture { + const capture = createDatabaseResponseCapture(window, () => {}, now); + const id = randomUUID(), entries = new Map(); + let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0; + const send = (event: Record) => { try { emit(event); } catch {} }; + function flush() { + if (closed) return; + closed = true; + clearTimeout(timer); + capture.flush(); + const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes); + for (const [groupIndex, entry] of ordered.entries()) send({ + event: "paperclip.test_db_response_capture.entry.v1", captureId: id, + groupIndex, groupPage: Math.floor(groupIndex / 128), + operation: entry.operation, table: entry.table, callers: entry.callers, + calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes, + maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues, + queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors, + }); + send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id, + windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(), + closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed, + inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128), + overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0), + rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false, + measurementBoundary: "postgres-js results before Drizzle column decoding", + relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true, + taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false, + }); + entries.clear(); + } + const timer = setTimeout(flush, Math.max(0, window.end - now())); + timer.unref(); + function finish(metadata: Metadata, value: unknown, failed: boolean) { + if (!capture.active()) return; + inFlight -= 1; + completed += 1; + let key = JSON.stringify(metadata); + if (!entries.has(key) && entries.size >= 2047) { + overflowCalls += 1; + metadata = { operation: "other", table: "other", callers: [] }; + key = "overflow"; + } + let entry = entries.get(key); + if (!entry) { + entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, + queryErrors: 0, serializationErrors: 0, estimatedValues: 0 }; + entries.set(key, entry); + } + entry.calls += 1; + if (failed) { entry.queryErrors += 1; return; } + entry.rows += Array.isArray(value) ? value.length : 0; + try { + const size = fullResponseValueSize(value); + entry.bytes += size.bytes; + entry.maxBytes = Math.max(entry.maxBytes, size.bytes); + entry.estimatedValues += size.estimatedValues; + } catch { entry.serializationErrors += 1; } + } + function execute(action: () => unknown, metadata: Metadata) { + if (!capture.active()) return action(); + inFlight += 1; + started += 1; + let result: unknown; + try { result = action(); } + catch (error) { finish(metadata, undefined, true); throw error; } + let asynchronous = false; + try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {} + if (asynchronous) return Promise.resolve(result).then( + value => { finish(metadata, value, false); return value; }, + error => { finish(metadata, undefined, true); throw error; }, + ); + finish(metadata, result, false); + return result; + } + return { ...capture, execute, flush }; +} diff --git a/server/src/__tests__/helpers/tool-profile-matching-fixture.ts b/server/src/__tests__/helpers/tool-profile-matching-fixture.ts new file mode 100644 index 00000000000..087e7c8bcb9 --- /dev/null +++ b/server/src/__tests__/helpers/tool-profile-matching-fixture.ts @@ -0,0 +1,15 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; + +export async function createToolProfileMatchingFixture(db: Db) { + const [company] = await db.insert(companies).values({ name: "Synthetic profile matching", issuePrefix: `BM${randomUUID().slice(0, 6)}` }).returning(); + const companyAgents = await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))).returning(); + const agent = companyAgents[0]!; + const [app] = await db.insert(toolApplications).values({ companyId: company.id, name: "Synthetic app", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: app.id, name: "Synthetic connection", uid: randomUUID(), transport: "mcp_remote" }).returning(); + const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: 1890 }, (_, i) => ({ companyId: company.id, applicationId: app.id, connectionId: connection.id, name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const }))).returning(); + const [profile] = await db.insert(toolProfiles).values({ companyId: company.id, profileKey: "selected", name: "Selected tools", defaultAction: "deny" }).returning(); + await db.insert(toolProfileBindings).values({ companyId: company.id, profileId: profile.id, targetType: "agent", targetId: agent.id }); + await db.insert(toolProfileEntries).values(catalog.slice(0, 1575).map((entry, i) => ({ companyId: company.id, profileId: profile.id, selectorType: "catalog_entry" as const, catalogEntryId: entry.id, effect: i % 5 === 0 ? "exclude" as const : "include" as const }))); + return { company, agent }; +} diff --git a/server/src/__tests__/helpers/tool-profile-projection-fixture.ts b/server/src/__tests__/helpers/tool-profile-projection-fixture.ts new file mode 100644 index 00000000000..9a20ff0dd2f --- /dev/null +++ b/server/src/__tests__/helpers/tool-profile-projection-fixture.ts @@ -0,0 +1,29 @@ +import { randomUUID } from "node:crypto"; +import { agents, companies, toolApplications, toolCatalogEntries, toolConnections, toolProfileEntries, toolProfiles, type Db } from "@paperclipai/db"; + +export async function createProfileProjectionFixture(db: Db, options: { profiles?: number; tools?: number; rules?: number } = {}) { + const profileCount = options.profiles ?? 200, toolCount = options.tools ?? 1900, ruleCount = options.rules ?? 380, maxRuleCount = options.rules ?? 695; + const [company] = await db.insert(companies).values({ name: "Profile projection fixture", issuePrefix: `PP${randomUUID().slice(0, 6)}` }).returning(); + await db.insert(agents).values(Array.from({ length: 12 }, (_, i) => ({ companyId: company.id, name: `Agent ${i}` }))); + const [application] = await db.insert(toolApplications).values({ companyId: company.id, name: "Projection application", type: "mcp_http" }).returning(); + const [connection] = await db.insert(toolConnections).values({ companyId: company.id, applicationId: application.id, name: "Projection connection", uid: randomUUID(), transport: "mcp_remote", status: "active", enabled: true, healthStatus: "ok" }).returning(); + const catalog = await db.insert(toolCatalogEntries).values(Array.from({ length: toolCount + maxRuleCount }, (_, i) => ({ + companyId: company.id, applicationId: application.id, connectionId: connection.id, + name: `tool_${i}`, toolName: `tool_${i}`, versionHash: `v${i}`, riskLevel: "read" as const, + status: i < toolCount ? "active" as const : "removed" as const, + firstSeenAt: new Date(i < toolCount ? "2026-01-03T00:00:00Z" : "2026-01-01T00:00:00Z"), + }))).returning(); + const profiles = await db.insert(toolProfiles).values(Array.from({ length: profileCount }, (_, i) => ({ + companyId: company.id, profileKey: `selected-${i}`, name: `Selected profile ${i}`, defaultAction: "deny" as const, + createdAt: new Date("2026-01-02T00:00:00Z"), + }))).returning(); + for (let i = 0; i < profiles.length; i++) { + const count = i % 4 < 2 ? maxRuleCount : 2 * ruleCount - maxRuleCount; + const rules = (i % 2 ? catalog.slice(toolCount, toolCount + count) : catalog.slice(0, count)).map(entry => ({ + companyId: company.id, profileId: profiles[i]!.id, selectorType: "catalog_entry" as const, + effect: "include" as const, catalogEntryId: entry.id, + })); + for (let offset = 0; offset < rules.length; offset += 500) await db.insert(toolProfileEntries).values(rules.slice(offset, offset + 500)); + } + return { company, application, connection, catalog: catalog.slice(0, toolCount), profiles }; +} diff --git a/server/src/__tests__/idle-completed-history.test.ts b/server/src/__tests__/idle-completed-history.test.ts index d79057fa43e..5f3089d4c3b 100644 --- a/server/src/__tests__/idle-completed-history.test.ts +++ b/server/src/__tests__/idle-completed-history.test.ts @@ -142,6 +142,97 @@ const support = externalDatabaseUrl ? { supported: true } : await getEmbeddedPos expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); }); + it.each([false, true])("keeps real sandbox creation uncertainty blocked through owned cleanup, manualRemaining=%s", async manualRemaining => { + const { createOwnedSandbox, recoverSandboxCreations, recordSandboxCreationResource } = await import("../services/sandbox-creation-operations.js"); + const { environmentRuntimeService } = await import("../services/environment-runtime.js"); + const { heartbeatService } = await import("../services/heartbeat.js"); + const { nextEnvironmentCleanupAt } = await import("../services/environment-cleanup-state.js"); + const f = await seed(), environmentId = randomUUID(); + const directory = await mkdtemp(path.join(tmpdir(), "history-sandbox-creation-")); + await db.insert(tables.environments).values({ id: environmentId, name: environmentId, driver: "sandbox" }); + const snapshot = () => readIdleSleepSafety(db, hold, Date.now, ownerId, async () => "none", + undefined, undefined, false, false, { collectDiagnostics: true }); + const reason = async (key: string) => (await snapshot()).diagnostics!.checks.find(check => check.reason === key)!.state; + const uncertain = async (contextual: boolean) => { + const providerLeaseId = randomUUID(); + const result = await createOwnedSandbox(db, { companyId: f.companyId, environmentId, heartbeatRunId: f.runId, + provider: "history-fixture", driverKey: "history-fixture", config: {}, requestKey: providerLeaseId, + metadata: { driver: "sandbox", ...(contextual ? { runnerTaskLease: true, runnerTaskCreationContextId: randomUUID() } : {}) } }, + async () => { + await writeFile(path.join(directory, providerLeaseId), "owned provider resource"); + return { providerLeaseId }; + }); + const id = String(result.metadata!.creationOperationId); + // Advance the actual creation owner's saved deadline, not its status. + await db.update(tables.sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }) + .where(eq(tables.sandboxCreationOperations.id, id)); + await recoverSandboxCreations(db, async () => null); + return { id, providerLeaseId }; + }; + try { + expect(await read()).toEqual({ version: 1, backgroundWork: "none" }); + const contextual = await uncertain(true); + const current = async (id: string) => (await db.select().from(tables.sandboxCreationOperations) + .where(eq(tables.sandboxCreationOperations.id, id)))[0]; + expect(await current(contextual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + // Isolated RED for the old pending-only catalog: there is no live run, + // finalization, lease, local worker, or other durable blocker to mask it. + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + expect(await reason("sandbox_creation_operations")).toBe("present"); + await recordSandboxCreationResource(db, { id: contextual.id, companyId: f.companyId, environmentId, + providerLeaseId: contextual.providerLeaseId }); + expect(await nextEnvironmentCleanupAt(db)).not.toBeNull(); + expect(await reason("sandbox_creation_operations")).toBe("present"); + await recoverSandboxCreations(db, async () => { throw new Error("Known late resource must use its exact owner"); }); + const pending = await current(contextual.id); + expect(pending).toMatchObject({ state: "cleanup_pending", leaseId: expect.any(String) }); + expect(await reason("sandbox_creation_operations")).toBe("none"); + expect(await reason("environment_leases")).toBe("present"); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + const manual = manualRemaining ? await uncertain(false) : undefined; + if (manual) { + expect(await current(manual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + expect(await reason("sandbox_creation_operations")).toBe("present"); + } + let destroyed = 0; + const runtime = environmentRuntimeService(db, { drivers: [{ driver: "sandbox", + acquireRunLease: async () => { throw new Error("Unexpected provider acquire"); }, + releaseRunLease: async () => { throw new Error("Unexpected provider release"); }, + resolveCapabilities: async () => { throw new Error("Unexpected capability resolution"); }, + retryPendingSandboxTeardown: async ({ lease }: { lease: { providerLeaseId: string | null } }) => { + expect(lease.providerLeaseId).toBe(contextual.providerLeaseId); + const resource = path.join(directory, lease.providerLeaseId!); + expect(await readFile(resource, "utf8")).toBe("owned provider resource"); + await rm(resource); + await expect(readFile(resource)).rejects.toMatchObject({ code: "ENOENT" }); + destroyed++; + return { providerLeaseId: lease.providerLeaseId, state: "destroyed" as const }; + } }] }); + await heartbeatService(db, { environmentRuntime: runtime }).sweepPendingCleanupLeases({ backoffMs: 0 }); + expect(destroyed).toBe(1); + expect(await current(contextual.id)).toMatchObject({ state: "resolved", leaseId: pending.leaseId }); + expect(await reason("environment_leases")).toBe("none"); + expect(await reason("sandbox_creation_operations")).toBe(manual ? "present" : "none"); + expect(await read()).toEqual({ version: 1, backgroundWork: manual ? "present" : "none" }); + if (manual) { + expect(await current(manual.id)).toMatchObject({ state: "needs_reconciliation", leaseId: null }); + expect(await readFile(path.join(directory, manual.providerLeaseId), "utf8")).toBe("owned provider resource"); + } + // A different creation is still real pending work after this one settles. + await createOwnedSandbox(db, { companyId: f.companyId, environmentId, provider: "history-fixture", + driverKey: "history-fixture", config: {}, metadata: {} }, async () => { + const providerLeaseId = randomUUID(); + await writeFile(path.join(directory, providerLeaseId), "another pending provider resource"); + return { providerLeaseId }; + }); + expect(await read()).toEqual({ version: 1, backgroundWork: "present" }); + } finally { + await db.delete(tables.sandboxCreationOperations).where(eq(tables.sandboxCreationOperations.companyId, f.companyId)); + await db.delete(tables.environments).where(eq(tables.environments.id, environmentId)); + await rm(directory, { recursive: true, force: true }); + } + }); + it.each(["completed", "cancelled"] as const)("retains a %s transfer without hiding its actual spool", async status => { const f = await seed(); const result = await companyTransferRunService.resumeOrCreate(db, { companyId: f.companyId, direction: "import", diff --git a/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts b/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts index 158ea422fba..1f36be6bb81 100644 --- a/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts +++ b/server/src/__tests__/idle-sleep-diagnostics-postgres.test.ts @@ -5,7 +5,7 @@ import { agentApiKeys, agents, budgetPolicies, chatDeliveries, chatEndpoints, co companyTransferRuns, connectionEventDeliveries, createDb, heartbeatRuns, issues, routineTriggers, routines, toolApplications, toolConnections, type Db } from "@paperclipai/db"; import { readIdleSleepSafety } from "../services/idle-sleep-safety.js"; -import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; const now = Date.UTC(2026, 10, 2); @@ -25,7 +25,7 @@ if (!support.supported) console.warn(`Skipping idle sleep diagnostics Postgres t db = createDb(externalDatabaseUrl ?? temporary!.connectionString, { maxConnections: 4 }); }, 60_000); afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); }); - afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await temporary?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); const read = (scheduled = false, monitors = false) => readIdleSleepSafety(db, hold, () => now, ownerId, localNone, undefined, undefined, scheduled, monitors, { collectDiagnostics: true }); diff --git a/server/src/__tests__/idle-sleep-safety.test.ts b/server/src/__tests__/idle-sleep-safety.test.ts index da47d343c56..b14a814fdac 100644 --- a/server/src/__tests__/idle-sleep-safety.test.ts +++ b/server/src/__tests__/idle-sleep-safety.test.ts @@ -9,7 +9,7 @@ import { heartbeatRuns, issues, issueWatchdogs, issueThreadInteractions, issueQuestionResponseDeliveries, projects, projectWorkspaces, routines, routineTriggers, toolApplications, toolConnections, workspaceOperations, executionWorkspaceRuntimeLeases, type Db, } from "@paperclipai/db"; -import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { readIdleSleepSafety, type IdleSleepDrainStatus } from "../services/idle-sleep-safety.js"; @@ -175,7 +175,7 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp db = createDb(externalDatabaseUrl ?? database.connectionString, { maxConnections: 4 }); }, 60_000); afterEach(async () => { await db.execute(sql`TRUNCATE companies, plugins, environments CASCADE`); }); - afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); const read = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal); const readScheduled = () => readIdleSleepSafety(db, owned, () => now, ownerId, emptyLocal, undefined, undefined, true); @@ -206,6 +206,19 @@ if (!support.supported) console.warn(`Skipping idle sleep Postgres tests: ${supp } }); + it("keeps a failed startup intent awake after its local worker counter settles", async () => { + const f = await seed(); + expect(await read()).toEqual(none); + const finish = beginIdleTrackedWork(); + await db.update(heartbeatRuns).set({ status: "running", runtimeMode: "native", nativePhase: "observed", + runnerProfileJson: { nativeStartupContext: { contextId: randomUUID(), deadlineAtMs: now - 1 } } }).where(eq(heartbeatRuns.id, f.runId)); + finish(); + expect(idleWorkSnapshot().active).toBe(0); + expect(await read()).toEqual(present); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.runId)); + expect(await read()).toEqual(none); + }); + it("retains committed budget stop intent until its exact generation is delivered", async () => { const f = await seed(); const [policy] = await db.insert(budgetPolicies).values({ companyId: f.companyId, scopeType: "company", scopeId: f.companyId, diff --git a/server/src/__tests__/issue-creation-source-read.test.ts b/server/src/__tests__/issue-creation-source-read.test.ts index 2b47f3af580..ba3a74fc2cf 100644 --- a/server/src/__tests__/issue-creation-source-read.test.ts +++ b/server/src/__tests__/issue-creation-source-read.test.ts @@ -1,6 +1,6 @@ import { randomUUID } from "node:crypto"; import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => { expect(run.issueId).toBe(binding === "scoped" ? sourceId : null); expect(run.nativeIssueId).toBeNull(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture); try { const result = await resolveIssueCreationSource(observed, child); diff --git a/server/src/__tests__/issue-display-status.test.ts b/server/src/__tests__/issue-display-status.test.ts new file mode 100644 index 00000000000..e2f05c26720 --- /dev/null +++ b/server/src/__tests__/issue-display-status.test.ts @@ -0,0 +1,127 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import { expect, it } from "vitest"; +import request from "supertest"; +import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db"; +import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared"; +import { issueDisplayStatusSql } from "../services/issue-display-status.js"; +import { issueService } from "../services/issues.js"; +import { companySearchService } from "../services/company-search.js"; +import { statusCardService } from "../services/status-cards.js"; +import { issueRoutes } from "../routes/issues.js"; +import { + describeEmbeddedPostgres, routeApp, + seedCompanyWithBoardAccess, useEmbeddedPostgres, +} from "./helpers/route-test-harness.js"; + +describeEmbeddedPostgres("Monitoring task presentation", () => { + const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", { + resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); }, + }); + + async function fixture() { + const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring"); + const agentId = randomUUID(); + await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} }); + const nextCheckAt = new Date(Date.now() + 60 * 60_000); + const issueId = randomUUID(); + await ctx.db.insert(issues).values({ + id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress", + assigneeAgentId: agentId, createdByUserId: company.actor.userId, + monitorNextCheckAt: nextCheckAt, + executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } }, + }); + return { ...company, agentId, issueId }; + } + + async function displayStatus(id: string) { + const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id)); + return row; + } + + it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => { + const f = await fixture(); + const svc = issueService(ctx.db); + expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" }); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]); + expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1); + const app = routeApp(ctx.db, f.actor, issueRoutes); + const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200); + expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" }); + const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" })); + expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" }); + expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1); + expect(result.filterOptionCounts.status.in_progress).toBe(1); + const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning(); + expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]); + // Existing agent/API lifecycle filters keep their original meaning. + expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1); + expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false); + }); + + it("filters before pagination and keeps In Progress separate from Monitoring", async () => { + const f = await fixture(); + await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId }); + const svc = issueService(ctx.db); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]); + expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]); + expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1); + expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2); + }); + + it("shows the same wait indicator in list blocker summaries and task detail relations", async () => { + const f = await fixture(); + const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning(); + await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" }); + const svc = issueService(ctx.db); + const list = await svc.list(f.companyId, { includeBlockedBy: true }); + expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]); + const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200); + expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]); + }); + + it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => { + const f = await fixture(); + const runId = randomUUID(); + await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" }); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId)); + await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId)); + await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring"); + await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId)); + expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" }); + }); + + it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => { + const f = await fixture(); + await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe(status); + }); + + it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => { + const f = await fixture(); + await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + }); + + it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => { + const f = await fixture(); + await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" }); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`); + expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring"); + await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId)); + expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress"); + }); +}); diff --git a/server/src/__tests__/issue-queued-comments-routes.test.ts b/server/src/__tests__/issue-queued-comments-routes.test.ts index 1d0467105af..fed13ba5780 100644 --- a/server/src/__tests__/issue-queued-comments-routes.test.ts +++ b/server/src/__tests__/issue-queued-comments-routes.test.ts @@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto"; import express from "express"; import request from "supertest"; import { eq, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agentRuntimeState, @@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId)); const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)) @@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { }).where(eq(heartbeatRuns.companyId, f.companyId)); const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId)); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200); @@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => { })), ]); const observed = createDb(tempDb!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture); try { const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200); diff --git a/server/src/__tests__/issue-summaries-routes.test.ts b/server/src/__tests__/issue-summaries-routes.test.ts index c832d92af03..f82446cd20c 100644 --- a/server/src/__tests__/issue-summaries-routes.test.ts +++ b/server/src/__tests__/issue-summaries-routes.test.ts @@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => { expect(res.headers["cache-control"]).toBe("private, no-store"); expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]); expect(Object.keys(res.body[0]).sort()).toEqual([ - "companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt", + "companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt", ]); expect(res.body[0].externalConversationState).toBeNull(); expect(forbiddenRead).not.toHaveBeenCalled(); diff --git a/server/src/__tests__/issues-assignee-unread.test.ts b/server/src/__tests__/issues-assignee-unread.test.ts new file mode 100644 index 00000000000..ab1c0fb9966 --- /dev/null +++ b/server/src/__tests__/issues-assignee-unread.test.ts @@ -0,0 +1,110 @@ +import { randomUUID } from "node:crypto"; +import { eq, sql } from "drizzle-orm"; +import request from "supertest"; +import { expect, it } from "vitest"; +import { activityLog, agents, authUsers, issues } from "@paperclipai/db"; +import { issueRoutes } from "../routes/issues.js"; +import { deriveIssueUserContext, issueService } from "../services/issues.js"; +import { describeEmbeddedPostgres, resetCompanyIssueFixtures, routeApp, seedCompanyWithBoardAccess, useEmbeddedPostgres } from "./helpers/route-test-harness.js"; + +describeEmbeddedPostgres("human assignee unread comments", () => { + const ctx = useEmbeddedPostgres("paperclip-assignee-unread-", { + resetEach: async db => { + await db.delete(activityLog); + await db.delete(issues); + await db.delete(agents); + await resetCompanyIssueFixtures(db); + await db.delete(authUsers); + }, + }); + + it.each([false, true])("keeps another human's real comment unread through recency updates (prior read: %s)", async priorRead => { + const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Assigned human task"); + const otherCompany = await seedCompanyWithBoardAccess(ctx.db, "Unrelated company"); + const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes); + const issue = await svc.create(companyId, { + title: "Human inbox comment", status: "in_progress", createdByUserId: userId, assigneeUserId: userId, + }); + if (priorRead) await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200); + const comment = await svc.addComment(issue.id, "Another human supplied an update", { userId: `author-${randomUUID()}` }); + const [afterComment] = await ctx.db.select().from(issues).where(eq(issues.id, issue.id)); + // Exercise the real insert and recency update. No historical timestamps or + // mocked comment state hide the ordering that used to clear unread status. + expect(afterComment!.updatedAt.getTime()).toBeGreaterThanOrEqual(new Date(comment.createdAt).getTime()); + const path = `/api/companies/${companyId}/issues`; + const expectUnread = async (unread: boolean) => { + for (const projection of ["", "&view=rows", "&view=index"]) { + const result = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&limit=100${projection}`).expect(200); + expect(result.body).toHaveLength(1); + expect(result.body[0].isUnreadForMe).toBe(unread); + const filtered = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&unreadForUserId=me&limit=100${projection}`).expect(200); + expect(filtered.body.map((row: { id: string }) => row.id)).toEqual(unread ? [issue.id] : []); + } + expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(unread ? 1 : 0); + expect(await svc.unreadInboxIssueIds(companyId, userId, sql`true`)).toEqual(unread ? [issue.id] : []); + }; + await expectUnread(true); + // Legacy compact responses have a short TTL. Check their SQL projection + // once; immediate mutation readback belongs to ordinary/rows/index. + const compact = await request(app).get(`${path}?q=${encodeURIComponent(issue.title)}&touchedByUserId=me&view=compact`).expect(200); + expect(compact.body[0].isUnreadForMe).toBe(true); + const derived = deriveIssueUserContext(afterComment!, userId, { + myLastCommentAt: null, myLastReadAt: null, lastExternalCommentAt: comment.createdAt, + }); + expect(derived.isUnreadForMe).toBe(true); + await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200); + await expectUnread(false); + await request(app).delete(`/api/issues/${issue.id}/read`).expect(200); + await expectUnread(true); + await svc.addComment(issue.id, "I have read this update", { userId }); + await expectUnread(false); + await request(app).get(`/api/companies/${otherCompany.companyId}/issues?unreadForUserId=me`).expect(403); + expect(await svc.countUnreadTouchedByUser(otherCompany.companyId, userId)).toBe(0); + }); + + it("shows existing comments as unread when a task is subsequently assigned to a human", async () => { + const { companyId, userId, actor } = await seedCompanyWithBoardAccess(ctx.db, "New assignment"); + const svc = issueService(ctx.db), app = routeApp(ctx.db, actor, issueRoutes); + const issue = await svc.create(companyId, { title: "Assigned after discussion", status: "backlog" }); + await svc.addComment(issue.id, "Context written before assignment", { userId: `author-${randomUUID()}` }); + await request(app).patch(`/api/issues/${issue.id}`).send({ assigneeUserId: userId, status: "in_progress" }).expect(200); + const result = await request(app).get(`/api/companies/${companyId}/issues?q=${encodeURIComponent(issue.title)}&unreadForUserId=me`).expect(200); + expect(result.body.map((row: { id: string }) => row.id)).toEqual([issue.id]); + expect(result.body[0].isUnreadForMe).toBe(true); + await request(app).post(`/api/issues/${issue.id}/read`).send({}).expect(200); + expect(await svc.countUnreadTouchedByUser(companyId, userId)).toBe(0); + }); + + it("excludes deleted external comments but retains the historical touch from a deleted own reply", async () => { + const { companyId, userId } = await seedCompanyWithBoardAccess(ctx.db, "Deleted comment controls"); + const svc = issueService(ctx.db); + const issue = await svc.create(companyId, { + title: "Deleted inbox activity", status: "in_progress", assigneeUserId: userId, + }); + const authorId = `author-${randomUUID()}`; + const first = await svc.addComment(issue.id, "Keep this live update", { userId: authorId }); + const second = await svc.addComment(issue.id, "Remove this later update", { userId: authorId }); + const observation = async () => { + const [row] = await svc.list(companyId, { touchedByUserId: userId }); + return { row, count: await svc.countUnreadTouchedByUser(companyId, userId) }; + }; + await svc.tombstoneComment(second.id, { actorType: "user", userId }); + expect((await observation()).count).toBe(1); + expect((await observation()).row!.lastExternalCommentAt).toEqual(new Date(first.createdAt)); + await svc.tombstoneComment(first.id, { actorType: "user", userId }); + const removed = await observation(); + expect(removed.count).toBe(0); + expect(removed.row!.isUnreadForMe).toBe(false); + expect(removed.row!.lastExternalCommentAt).toBeNull(); + await svc.addComment(issue.id, "New live activity", { userId: authorId }); + expect((await observation()).count).toBe(1); + const ownReply = await svc.addComment(issue.id, "I saw the live update", { userId }); + expect((await observation()).count).toBe(0); + await svc.tombstoneComment(ownReply.id, { actorType: "user", userId }); + const seen = await observation(); + expect(seen.count).toBe(0); + expect(seen.row!.isUnreadForMe).toBe(false); + expect(seen.row!.myLastTouchAt).toEqual(new Date(ownReply.createdAt)); + }); + +}); diff --git a/server/src/__tests__/issues-list-query-parsing.test.ts b/server/src/__tests__/issues-list-query-parsing.test.ts index 91024b93d05..9a369ce0afe 100644 --- a/server/src/__tests__/issues-list-query-parsing.test.ts +++ b/server/src/__tests__/issues-list-query-parsing.test.ts @@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => { it("returns every status when ?status is absent", async () => { expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]); }); + it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting"); + await request(routeApp(ctx.db, actor, issueRoutes)) + .get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`) + .expect(422); + }); + + it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => { + const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting"); + const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`); + await ctx.db.insert(issues).values([ + ...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo", + createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"), + updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })), + { companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") }, + ]); + const app = routeApp(ctx.db, actor, issueRoutes); + const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`; + for (const sortDir of ["asc", "desc"]) { + const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200); + const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200); + expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse()); + } + await request(app).get(`${path}&sortDir=invalid`).expect(400); + }); + }); diff --git a/server/src/__tests__/issues-service.test.ts b/server/src/__tests__/issues-service.test.ts index 0d28858b95c..7d8dd0d5823 100644 --- a/server/src/__tests__/issues-service.test.ts +++ b/server/src/__tests__/issues-service.test.ts @@ -4166,6 +4166,7 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU await db.delete(executionWorkspaces); await db.delete(projectWorkspaces); await db.delete(projects); + await db.delete(heartbeatRuns); await db.delete(agents); await db.delete(instanceSettings); await db.delete(companies); @@ -4720,6 +4721,42 @@ const dependencyTestDatabaseUrl = resolveExternalTestDatabaseUrl("PAPERCLIP_ISSU }); }); + it.each(["covered", "foreign-run", "unfinished", "failed-finalize", "late-finish", "later-operation"])( + "requires a successful owning finalize to cover a nested snapshot: %s", async kind => { + const f = await seedSharedWorkspaceDependency(); + const runId = randomUUID(), foreignRunId = randomUUID(); + await db.insert(heartbeatRuns).values([runId, foreignRunId].map(id => ({ + id, companyId: f.companyId, agentId: f.assigneeAgentId, status: "succeeded", + }))); + const started = new Date("2026-05-23T22:00:00.000Z"); + const finished = new Date("2026-05-23T22:00:02.000Z"); + await db.insert(workspaceOperations).values([ + { companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId, + heartbeatRunId: runId, phase: "workspace_finalize", + status: kind === "failed-finalize" ? "failed" : "succeeded", startedAt: started, finishedAt: finished }, + { companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId, + heartbeatRunId: kind === "foreign-run" ? foreignRunId : runId, phase: "remote_workspace_snapshot", + status: kind === "unfinished" ? "running" : "succeeded", + startedAt: new Date(started.getTime() + 100), + finishedAt: kind === "unfinished" ? null : new Date(finished.getTime() + (kind === "late-finish" ? 100 : -100)) }, + ...(kind === "later-operation" ? [{ + companyId: f.companyId, executionWorkspaceId: f.executionWorkspaceId, issueId: f.blockerId, + heartbeatRunId: runId, phase: "worktree_prepare", status: "succeeded", + startedAt: new Date(finished.getTime() + 100), finishedAt: new Date(finished.getTime() + 200), + }] : []), + ]); + const ready = kind === "covered"; + expect((await svc.listWakeableBlockedDependents(f.blockerId)).map(issue => issue.id)) + .toEqual(ready ? [f.dependentId] : []); + await expect(svc.getDependencyReadiness(f.dependentId)).resolves.toMatchObject({ + isDependencyReady: ready, + pendingFinalizeBlockerIssueIds: ready ? [] : [f.blockerId], + }); + const listed = (await svc.list(f.companyId, { status: "blocked" })).find(issue => issue.id === f.dependentId); + expect(listed?.blockerAttention?.pendingFinalizeBlockerIssueIds ?? []).toEqual(ready ? [] : [f.blockerId]); + }, + ); + it("reports dependency readiness for blocked issue chains", async () => { const companyId = randomUUID(); await seedIssueCompany(db, companyId); diff --git a/server/src/__tests__/issues-user-context.test.ts b/server/src/__tests__/issues-user-context.test.ts index 421934ee270..568348e5ae9 100644 --- a/server/src/__tests__/issues-user-context.test.ts +++ b/server/src/__tests__/issues-user-context.test.ts @@ -76,7 +76,7 @@ describe("deriveIssueUserContext", () => { } }); - it("uses issue updated time as fallback touch point for assignee", () => { + it("uses stable creation time for assignees instead of treating task recency as a read", () => { const context = deriveIssueUserContext( makeIssue({ assigneeUserId: "user-1", updatedAt: new Date("2026-03-06T15:00:00.000Z") }), "user-1", @@ -87,8 +87,8 @@ describe("deriveIssueUserContext", () => { }, ); - expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T15:00:00.000Z"); - expect(context.isUnreadForMe).toBe(false); + expect(context.myLastTouchAt?.toISOString()).toBe("2026-03-06T10:00:00.000Z"); + expect(context.isUnreadForMe).toBe(true); }); it("uses latest read timestamp to clear unread without requiring a comment", () => { diff --git a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts index 65e0e0cc147..38d748d1a36 100644 --- a/server/src/__tests__/legacy-workspace-restore-recovery.test.ts +++ b/server/src/__tests__/legacy-workspace-restore-recovery.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db"; @@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, }).where(eq(heartbeatRuns.id, f.runId)).returning(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let returned: Awaited>; try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); } finally { capture.flush(); } - const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"); const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select"); expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1); expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096); @@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } }, }))!; const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let returned: Awaited>; try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly, @@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde ...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}), }); } finally { capture.flush(); } - const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "heartbeat_runs" && event.operation === "update"); expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1); expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt, diff --git a/server/src/__tests__/native-environment-settlement-work.test.ts b/server/src/__tests__/native-environment-settlement-work.test.ts index 7a52bc5304d..4fd2448b9a4 100644 --- a/server/src/__tests__/native-environment-settlement-work.test.ts +++ b/server/src/__tests__/native-environment-settlement-work.test.ts @@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; import { heartbeatService } from "../services/heartbeat.js"; import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js"; @@ -180,7 +180,7 @@ describe("native environment release scheduling", () => { } const events: Record[] = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture); const settle = vi.fn(async () => {}); const work = nativeEnvironmentSettlementWork(db, settle); diff --git a/server/src/__tests__/native-settled-process-release.test.ts b/server/src/__tests__/native-settled-process-release.test.ts index 1028d9abc07..a6805c61582 100644 --- a/server/src/__tests__/native-settled-process-release.test.ts +++ b/server/src/__tests__/native-settled-process-release.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js"; import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { spawn } from "node:child_process"; import os from "node:os"; import { and, eq, sql } from "drizzle-orm"; @@ -222,11 +222,11 @@ suite("settled native run process identity", () => { const saved = await readRun(runId); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.responseValueBytes).toBeLessThan(8192); const after = await readRun(runId); expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null, diff --git a/server/src/__tests__/native-startup-cleanup-recovery.test.ts b/server/src/__tests__/native-startup-cleanup-recovery.test.ts new file mode 100644 index 00000000000..4e05e3db5bb --- /dev/null +++ b/server/src/__tests__/native-startup-cleanup-recovery.test.ts @@ -0,0 +1,729 @@ +import { randomUUID } from "node:crypto"; +import { databaseWorkPending } from "@paperclipai/db/work-signals"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { writeFile } from "node:fs/promises"; +import path from "node:path"; +import type { PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk"; +import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; +import { and, eq, sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + activityLog, agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, + environments, heartbeatRuns, issues, nativeRunFinalizations, nativeRunResults, plugins, sandboxCreationOperations, +} from "@paperclipai/db"; +import { recoveryPostgresSupport, startRecoveryPostgres } from "./helpers/recovery-postgres.js"; +import { createIsolatedTestHome } from "./helpers/isolated-test-home.js"; +import { prepareNativeHeartbeatRun } from "../services/native-runtime/prepare-native-run.js"; +import { + acknowledgeNativeStartupCleanup, NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupCleanup, + requestNativeStartupCleanup, verifyNativeStartupCleanup, clearNativeStartupContext, +} from "../services/native-runtime/native-startup-cleanup.js"; +import { isNativeRunnerOwnershipHeld } from "../services/native-runtime/native-runner-ownership.js"; +import { cancelNativeSession } from "../services/native-runtime/native-session-executor.js"; +import * as nativeSessionExecutor from "../services/native-runtime/native-session-executor.js"; +import { NativeExecutionContext } from "../services/native-runtime/native-execution-context.js"; +import type { NativeExecutionInput } from "../vendor/paperclip-runner/index.js"; +import { callbacks } from "../services/heartbeat/__tests__/run-control-fixtures.js"; +import { createHeartbeatRunControl } from "../services/heartbeat/run-control.js"; +import { nextEnvironmentCleanupAt } from "../services/environment-cleanup-state.js"; +import { executionControlSchedule } from "../services/execution-control-work.js"; +import { claimNativeRestartRecoveries } from "../services/native-runtime/native-restart-recovery.js"; +import { reconcileAbandonedExecutionControl } from "../services/execution-control-reconciliation.js"; +import { createPluginWorkerManager, type PluginWorkerManager } from "../services/plugin-worker-manager.js"; +import { hasRemoteTerminationReceipt, remoteTerminationReceipt } from "../services/remote-execution-termination.js"; + +import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource, recoverSandboxCreations } from "../services/sandbox-creation-operations.js"; +import { environmentService } from "../services/environments.js"; +import { environmentRuntimeService } from "../services/environment-runtime.js"; +import { createHeartbeatRecovery, type HeartbeatRecoveryDependencies } from "../services/heartbeat/recovery.js"; + +const describePostgres = recoveryPostgresSupport.supported ? describe : describe.skip; + +describePostgres("durable cleanup after a native startup timeout", () => { + let temporary: Awaited>; + let db: ReturnType; + let home: ReturnType; + const originalHome = process.env.PAPERCLIP_HOME; + const companyId = randomUUID(), agentId = randomUUID(); + + beforeAll(async () => { + home = createIsolatedTestHome(); + process.env.PAPERCLIP_HOME = home.home; + temporary = await startRecoveryPostgres("native-startup-cleanup-"); + db = createDb(temporary.connectionString); + await db.insert(companies).values({ id: companyId, name: "Startup cleanup", issuePrefix: "NSC" }); + await db.insert(agents).values({ id: agentId, companyId, name: "Runner", role: "engineer", status: "running", + adapterType: "paperclip_runner", adapterConfig: { provider: "codex" }, runtimeConfig: {}, permissions: {} }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + + afterAll(async () => { + await temporary?.cleanup(); + home?.cleanup(); + if (originalHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = originalHome; + }); + + async function fixture(requiresPreparationStop = false) { + const runId = randomUUID(), issueId = randomUUID(), leaseId = randomUUID(), taskId = randomUUID(); + const contextId = randomUUID(), leaseOwner = randomUUID(); + await db.insert(issues).values({ id: issueId, companyId, identifier: `NSC-${issueId}`, title: "Timed out startup", + status: "in_progress", priority: "medium", assigneeAgentId: agentId }); + const [run] = await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, status: "running", + runtimeMode: "native", nativeIssueId: issueId, invocationSource: "manual", contextSnapshot: { issueId } }).returning(); + await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); + await prepareNativeHeartbeatRun({ db, run, issue: { id: issueId, title: "Timed out startup", description: null }, environmentLeaseId: leaseId }); + await db.insert(nativeRunFinalizations).values({ runId, companyId, issueId, phase: "observed", attempt: 1, + controllerGeneration: 1, controllerBootId: randomUUID(), controllerPid: process.pid, + controllerProcessStartedAt: new Date(), leaseOwner, leaseExpiresAt: new Date(Date.now() + 60_000) }); + const [environment] = await db.insert(environments).values({ name: `Cleanup ${runId}`, driver: "plugin", status: "active", config: {} }).returning(); + await db.insert(environmentLeases).values({ id: leaseId, companyId, environmentId: environment.id, heartbeatRunId: runId, + issueId, status: "active", provider: "plugin", providerLeaseId: taskId, leasePolicy: "reuse_by_environment", + metadata: { runnerTaskLease: true, runnerTaskReused: true } }); + const request = { companyId, agentId, runId, contextId, leaseOwner, deadlineAtMs: Date.now() - 1, requiresPreparationStop }; + const reload = () => db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)).then(rows => rows[0]!); + const owner = () => db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)).then(rows => rows[0]!); + const lease = () => db.select().from(environmentLeases).where(eq(environmentLeases.id, leaseId)).then(rows => rows[0]!); + const stopReceipt = async (override: Record = {}, preparationStopped = false) => { + const current = await lease(); + const receipt = remoteTerminationReceipt(current, { providerLeaseId: taskId, state: "stopped" }); + await db.update(environmentLeases).set({ status: "released", releasedAt: new Date(), cleanupStatus: "success", + metadata: { ...current.metadata, ...(preparationStopped ? { runnerTaskPreparationStopped: true } : {}), remoteExecutionTermination: { ...receipt, ...override } } }).where(eq(environmentLeases.id, leaseId)); + }; + const acceptedResult = async () => { + const current = await reload(), resultId = randomUUID(); + await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId, + completionContractId: current.completionContractId!, serverFingerprint: resultId, schemaStatus: "accepted", + resultJson: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", summary: "Completed work" }, canonicalSha256: "a".repeat(64) }); + await db.update(nativeRunFinalizations).set({ resultId, phase: "workspace_finalizing" }).where(eq(nativeRunFinalizations.runId, runId)); + return resultId; + }; + return { request, reload, owner, lease, stopReceipt, acceptedResult }; + } + + async function saveEarlyContext(f: Awaited>) { + const run = await f.reload(), owner = await f.owner(), lease = await f.lease(); + const context = { schema: "paperclip.native-startup-context.v1", companyId, agentId, runId: run.id, + contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, leaseId: lease.id, + leaseOwner: owner.leaseOwner, controllerGeneration: owner.controllerGeneration, attempt: owner.attempt }; + await db.update(heartbeatRuns).set({ runnerProfileJson: { ...run.runnerProfileJson, nativeStartupContext: context } }).where(eq(heartbeatRuns.id, run.id)); + return context; + } + + it.each(["pending", "needs_reconciliation"])("keeps ownership until a %s replacement queued after the first teardown snapshot is verified stopped", async state => { + const f = await fixture(true), original = await f.lease(); + let reply!: (result: { providerLeaseId: string }) => void, creating!: () => void; + const entered = new Promise(resolve => { creating = resolve; }); + const acquiring = createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId, + issueId: original.issueId, provider: original.provider!, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!, + metadata: { runnerTaskLease: true, runnerTaskCreationContextId: f.request.contextId } }, async () => new Promise(resolve => { reply = resolve; creating(); })); + await entered; + await requestNativeStartupCleanup(db, f.request); + await f.stopReceipt({}, true); + await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending"); + const [pending] = await db.select().from(sandboxCreationOperations).where(and( + eq(sandboxCreationOperations.runId, f.request.runId), eq(sandboxCreationOperations.requestKey, original.providerLeaseId!))); + if (state === "needs_reconciliation") { + await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }).where(eq(sandboxCreationOperations.id, pending.id)); + await recoverSandboxCreations(db, async () => null); + expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, pending.id)))[0].state).toBe(state); + const inspect = vi.fn(async () => null); + await recoverSandboxCreations(db, inspect); + expect(inspect).not.toHaveBeenCalled(); + await expect(queueSandboxCreationCleanup(db, { id: pending.id, companyId, environmentId: original.environmentId!, + providerLeaseId: "unproven-task" })).rejects.toThrow("ownership changed"); + } + reply({ providerLeaseId: "late-replacement" }); + const fresh = await acquiring; + await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId, + environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId, providerMetadata: { binding: "actual" } }); + if (state === "pending") await db.update(sandboxCreationOperations).set({ nextAttemptAt: new Date(0) }) + .where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))); + expect(await nextEnvironmentCleanupAt(db)).not.toBeNull(); + const restarted = createDb(temporary.connectionString); + try { await recoverSandboxCreations(restarted, async () => { throw new Error("A recorded task identity needs no new provider inspection"); }); } + finally { await restarted.$client.end(); } + const [creation] = await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))); + await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => {})).rejects.toThrow("cleanup is still pending"); + const replacement = (await environmentService(db).getLeaseById(creation.leaseId!))!; + await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId }); + expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].leaseId).toBe(replacement.id); + await expect(queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: "foreign-task" })).rejects.toThrow("ownership changed"); + await environmentService(db).releaseLease(replacement.id, "released", { cleanupStatus: "success", + remoteExecutionTermination: remoteTerminationReceipt(replacement, { providerLeaseId: replacement.providerLeaseId, state: "stopped" }) }); + expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0].state).toBe("resolved"); + await queueSandboxCreationCleanup(db, { id: creation.id, companyId, environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId }); + expect((await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, creation.id)))[0]) + .toMatchObject({ state: "resolved", leaseId: replacement.id }); + await verifyNativeStartupCleanup(db, await f.reload(), async () => {}); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await acknowledgeNativeStartupCleanup(db, f.request); + expect((await f.owner()).leaseOwner).toBeNull(); + }); + + it.each(["before_inventory", "before_write", "complete_before_write", "ordinary"])("preserves startup cleanup through orphan recovery: %s", async phase => { + const f = await fixture(), deps = callbacks(db); + await db.update(nativeRunFinalizations).set({ phase: "terminal_failure", leaseExpiresAt: new Date(0) }) + .where(eq(nativeRunFinalizations.runId, f.request.runId)); + if (phase === "before_inventory") await requestNativeStartupCleanup(db, f.request); + const write = deps.setRunStatusFromLive.getMockImplementation()!; + deps.setRunStatusFromLive.mockImplementation(async (...args) => { + if (phase === "before_write" || phase === "complete_before_write") { + await requestNativeStartupCleanup(db, f.request); + if (phase === "complete_before_write") await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt()); + } + return write(...args); + }); + const recovery = createHeartbeatRecovery(db, { ...deps, environmentRuntime: environmentRuntimeService(db), + environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set>(), + mergeRunStopMetadataForAgent: (_agent: unknown, _status: unknown, options: { resultJson?: object }) => options?.resultJson, + classifyAndPersistRunLiveness: async (run: unknown) => run, releaseEnvironmentLeasesForRun: vi.fn(), + releaseIssueExecutionAndPromote: vi.fn(), scheduleInteractionContinuationInfrastructureRetryIfEligible: async () => null, + } as unknown as HeartbeatRecoveryDependencies); + await recovery.recoverActiveExecutions({ runId: f.request.runId }, { staleThresholdMs: 0 }); + expect((await f.reload()).status).toBe(phase === "ordinary" ? "failed" : "running"); + if (phase === "ordinary") return; + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt()); + expect(await createHeartbeatRunControl(db, deps).cancelRunInternal(f.request.runId, "Startup cleanup retry")) + .toMatchObject({ status: "cancelled" }); + expect((await f.owner()).leaseOwner).toBeNull(); + }); + + it.each(["request", "acknowledge"])("registers queue intent before cleanup %s writes", async phase => { + const f = await fixture(true); + const writes: boolean[][] = []; + const observe = async (queues: string[], work: () => Promise) => { + const transaction = db.transaction.bind(db); + const spy = vi.spyOn(db, "transaction").mockImplementation(((callback: (tx: typeof db) => Promise) => transaction(async tx => { + const update = tx.update.bind(tx); + tx.update = ((table: unknown) => { + if ([heartbeatRuns, nativeRunFinalizations, environmentLeases].includes(table as never)) { + writes.push(queues.map(queue => databaseWorkPending(db, queue))); + } + return update(table as never); + }) as typeof tx.update; + return callback(tx as unknown as typeof db); + })) as typeof db.transaction); + try { await work(); } finally { spy.mockRestore(); } + }; + if (phase === "request") { + await observe([DELIVERY_QUEUES.executionFinalization, DELIVERY_QUEUES.environmentCleanup], () => requestNativeStartupCleanup(db, f.request)); + expect(writes).toEqual([[true, true], [true, true], [true, true]]); + await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true)); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await acknowledgeNativeStartupCleanup(db, f.request); + } else { + await requestNativeStartupCleanup(db, f.request); + await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true)); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await observe([DELIVERY_QUEUES.agentLifecycle, DELIVERY_QUEUES.environmentCleanup], () => acknowledgeNativeStartupCleanup(db, f.request)); + expect(writes).toEqual([[true, true]]); + } + }); + + it("returns the deadline queue to quiet and wakes again for the next startup failure", async () => { + const schedule = executionControlSchedule(db); + expect(await schedule.finalization()).toBeNull(); + for (let failure = 0; failure < 2; failure += 1) { + const f = await fixture(true); + await requestNativeStartupCleanup(db, f.request); + expect(await schedule.finalization()).not.toBeNull(); + const deps = callbacks(db); + deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true)); + const control = createHeartbeatRunControl(db, deps); + await reconcileAbandonedExecutionControl(db, new Date(Date.now() + 1_000), { startupCleanup: async scope => { + if (scope.runId !== f.request.runId) return; + await control.cancelRunInternal(scope.runId, "Startup cleanup retry"); + await acknowledgeNativeStartupCleanup(db, scope); + } }); + expect((await f.reload()).status).toBe("cancelled"); + expect(await f.owner()).toMatchObject({ leaseOwner: null, controlDeadlineAt: null }); + expect(await schedule.finalization()).toBeNull(); + } + }); + + it("discovers a crash after warm prepare dispatch but before a cleanup intent was recorded", async () => { + const f = await fixture(true), lease = await f.lease(); + await db.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: { + contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId, + } } }).where(eq(environmentLeases.id, lease.id)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }) + .where(eq(nativeRunFinalizations.runId, f.request.runId)); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + const onClaim = vi.fn(); + const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim }); + expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]); + expect(onClaim).not.toHaveBeenCalled(); + expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId, + state: "pending", requiresPreparationStop: true }); + expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false); + }); + + it.each(["wrong_run", "future_deadline", "live_owner", "checkpointed", "missing_marker"])( + "does not infer a timed-out startup from %s", async variant => { + const f = await fixture(true), lease = await f.lease(); + if (variant !== "missing_marker") await db.update(environmentLeases).set({ metadata: { ...lease.metadata, + runnerTaskStartup: { contextId: f.request.contextId, runId: variant === "wrong_run" ? randomUUID() : f.request.runId, + deadlineAtMs: variant === "future_deadline" ? Date.now() + 60_000 : f.request.deadlineAtMs }, + } }).where(eq(environmentLeases.id, lease.id)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() + (variant === "live_owner" ? 60_000 : -1_000)) }) + .where(eq(nativeRunFinalizations.runId, f.request.runId)); + if (variant === "checkpointed") await db.update(heartbeatRuns).set({ sessionIdAfter: "current-run-checkpoint" }) + .where(eq(heartbeatRuns.id, f.request.runId)); + await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } }); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + }); + + it("persists an audited cancellation intent that a new database handle can recover without a generic hold", async () => { + const f = await fixture(); + expect(await requestNativeStartupCleanup(db, f.request)).toBe(true); + const run = await f.reload(), marker = readNativeStartupCleanup(run)!; + expect(marker).toMatchObject({ state: "pending", contextId: f.request.contextId, leaseOwner: f.request.leaseOwner, + controllerGeneration: 1, attempt: 1, taskId: (await f.lease()).providerLeaseId }); + expect(run).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING }); + expect(run.resultJson?.startupCancellation).toMatchObject({ requestedAt: expect.any(String) }); + expect(isNativeRunnerOwnershipHeld(run)).toBe(false); + expect(await db.select({ id: activityLog.id }).from(activityLog).where(and(eq(activityLog.id, marker.auditId), + eq(activityLog.companyId, companyId), eq(activityLog.action, "native.startup_cleanup_requested")))).toHaveLength(1); + const restarted = createDb(temporary.connectionString), release = vi.fn(f.stopReceipt); + await verifyNativeStartupCleanup(restarted, await f.reload(), release); + await verifyNativeStartupCleanup(restarted, await f.reload(), release); + expect(release).toHaveBeenCalledOnce(); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id }); + expect((await f.owner()).controlDeadlineAt).not.toBeNull(); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, run.id)); + await acknowledgeNativeStartupCleanup(restarted, { companyId, runId: run.id }); + expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null }); + }); + + it("keeps cleanup bound to the explicit current lease when a released alias belongs to the same run", async () => { + const f = await fixture(true), current = await f.lease(); + const aliasId = randomUUID(); + await db.insert(environmentLeases).values({ ...current, id: aliasId, providerLeaseId: randomUUID(), + status: "released", releasedAt: new Date(), cleanupStatus: "success" }); + expect(await requestNativeStartupCleanup(db, { ...f.request, leaseId: current.id })).toBe(true); + expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ leaseId: current.id, taskId: current.providerLeaseId }); + expect((await db.select().from(environmentLeases).where(eq(environmentLeases.id, aliasId)))[0]?.metadata?.runnerTaskStartupCleanup).toBeUndefined(); + }); + + it("recovers an exact Stop receipt committed before Core records completion", async () => { + const f = await fixture(); + await requestNativeStartupCleanup(db, f.request); + await f.stopReceipt(); + const release = vi.fn(async () => { throw new Error("Already retired task must not be released twice"); }); + await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release); + expect(release).not.toHaveBeenCalled(); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + }); + + it("keeps accelerated preparation pending when only the provider has stopped", async () => { + const f = await fixture(true); + await requestNativeStartupCleanup(db, f.request); + await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow(/pending|preparation/i); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + }); + + it("automatically finishes pending cleanup after a late real worker reply and exact preparation Stop proof", async () => { + const f = await fixture(true), lease = await f.lease(); + const entrypointPath = path.join(home.home, `cleanup-worker-${f.request.runId}.cjs`); + await writeFile(entrypointPath, ` +const rl=require("node:readline").createInterface({input:process.stdin,crlfDelay:Infinity}); +const held=[];let stopped=false; +const send=(id,result)=>process.stdout.write(JSON.stringify({jsonrpc:"2.0",id,result})+"\\n"); +rl.on("line",line=>{const m=JSON.parse(line); + if(m.method==="initialize")send(m.id,{ok:true,supportedMethods:["environmentTask","configChanged"]}); + else if(m.method==="environmentTask"){ + const kind=m.params.operation.kind; + if(kind==="prepare")held.push(m.id); + else if(kind==="status")send(m.id,{kind:"status",taskId:m.params.taskId,phase:stopped?"cancelled":"running",executionStopped:stopped,preparationStopped:stopped&&held.length===0}); + else {if(kind==="stop")stopped=true;send(m.id,{kind:"accepted",taskId:m.params.taskId});} + }else if(m.method==="configChanged"){for(const id of held.splice(0))send(id,{});send(m.id,{});} + else if(m.method==="shutdown"){send(m.id,{});setImmediate(()=>process.exit(0));} +}); +`, { mode: 0o600 }); + const manifest: PaperclipPluginManifestV1 = { id: "test.startup-cleanup", apiVersion: 1, version: "1.0.0", + displayName: "Cleanup fixture", description: "Cleanup fixture", author: "Paperclip", categories: ["automation"], + capabilities: [], entrypoints: { worker: "worker.cjs" } }; + const manager = createPluginWorkerManager(); + await manager.startWorker(manifest.id, { entrypointPath, manifest, config: {}, + instanceInfo: { instanceId: "cleanup", hostVersion: "1.0.0" }, apiVersion: 1, hostHandlers: {} }); + const context = new NativeExecutionContext({ binding: { companyId, agentId, runId: f.request.runId } } as NativeExecutionInput, 100); + const params = { companyId, agentId, runId: f.request.runId, environmentId: lease.environmentId, issueId: lease.issueId, + driverKey: "tasks", taskId: lease.providerLeaseId!, config: {}, projectIds: [], lease: { providerLeaseId: lease.providerLeaseId! } }; + const operation = { kind: "prepare", projectIds: [], requirements: { schema: "paperclip.runner.launch-requirements.v1", + identity: { runnerInstanceId: "fixture", environmentLeaseId: lease.id, runId: f.request.runId, + normalizedSessionId: "session", turnId: "turn", itemId: "item" }, provider: "codex", model: null, + instructions: "Synthetic", runtimeContext: null, settings: {}, approvalPolicy: "never", readOnly: false, + preparedConversation: false, includeCollaborationModeInstructions: false } } as PluginEnvironmentTaskOperation; + const release = async () => { + await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "stop" } }); + const status = await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "status" } }); + if (status.kind === "status" && status.executionStopped && status.preparationStopped) { + await manager.call(manifest.id, "environmentTask", { ...params, operation: { kind: "complete" } }); + await f.stopReceipt({}, true); + } + }; + try { + await expect(manager.call(manifest.id, "environmentTask", { ...params, operation }, 10, undefined, context.trackWorker())).rejects.toThrow("timed out"); + if (!context.signal.aborted) await new Promise(resolve => context.signal.addEventListener("abort", () => resolve(), { once: true })); + expect(context.pending).toBe(true); + expect(await requestNativeStartupCleanup(db, { ...f.request, contextId: context.id, deadlineAtMs: context.deadlineAtMs })).toBe(true); + expect(isNativeRunnerOwnershipHeld(await f.reload())).toBe(false); + await expect(cancelNativeSession(f.request.runId, "Startup timeout", { db, scope: "run" })).resolves.toMatchObject({ auditId: expect.any(String) }); + await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("pending"); + expect((await f.reload()).status).toBe("running"); + await manager.call(manifest.id, "configChanged", { companyId, config: { release: true } }); + expect(context.pending).toBe(false); + await verifyNativeStartupCleanup(createDb(temporary.connectionString), await f.reload(), release); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + expect((await f.lease()).metadata?.runnerTaskPreparationStopped).toBe(true); + } finally { context.finish(); await manager.stopAll(); } + }); + + it.each([false, true])("verifies an abandoned ephemeral startup without an inactive-owner acknowledgement (receipt=%s)", async stopped => { + const prior = process.env.PAPERCLIP_EPHEMERAL_SERVER; + process.env.PAPERCLIP_EPHEMERAL_SERVER = "true"; + try { + const f = await fixture(true); + await requestNativeStartupCleanup(db, f.request); + const deps = callbacks(db); + deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, async () => { if (stopped) await f.stopReceipt({}, true); })); + const control = createHeartbeatRunControl(db, deps); + if (stopped) { + expect(await control.cancelRunInternal(f.request.runId, "Startup timeout")).toMatchObject({ status: "cancelled" }); + expect((await f.owner()).leaseOwner).toBeNull(); + } else { + await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("Task cleanup is still pending"); + expect((await f.reload()).status).toBe("running"); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + } + expect(deps.verifyStartupCleanup).toHaveBeenCalled(); + } finally { + if (prior === undefined) delete process.env.PAPERCLIP_EPHEMERAL_SERVER; + else process.env.PAPERCLIP_EPHEMERAL_SERVER = prior; + } + }, 30_000); + + it("terminalizes through assembled run control only after verified cleanup succeeds", async () => { + const f = await fixture(true); + await requestNativeStartupCleanup(db, f.request); + let drained = false; + const deps = callbacks(db); + deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained)); + const control = createHeartbeatRunControl(db, deps); + await expect(control.cancelRunInternal(f.request.runId, "Startup timeout")).rejects.toThrow("pending"); + expect(await f.reload()).toMatchObject({ status: "running", errorCode: NATIVE_STARTUP_CLEANUP_PENDING }); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + drained = true; + const result = await control.cancelRunInternal(f.request.runId, "Startup timeout"); + expect(result).toMatchObject({ status: "cancelled" }); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null }); + }); + + it("joins cleanup recorded while an operator Stop waits for native startup", async () => { + const f = await fixture(true); + let entered!: () => void, resume!: () => void; + const waiting = new Promise(resolve => { entered = resolve; }); + const settled = new Promise(resolve => { resume = resolve; }); + const cancel = nativeSessionExecutor.cancelNativeSession; + const spy = vi.spyOn(nativeSessionExecutor, "cancelNativeSession").mockImplementation(async (...args) => { + const result = await cancel(...args); + if (args[0] === f.request.runId) { entered(); await settled; } + return result; + }); + let drained = false; + const deps = callbacks(db); + deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, drained))); + const control = createHeartbeatRunControl(db, deps); + try { + const stopping = control.cancelRunInternal(f.request.runId, "Operator Stop"); + const outcome = stopping.then(value => ({ value }), error => ({ error })); + await waiting; + expect(await requestNativeStartupCleanup(db, f.request)).toBe(true); + resume(); + expect(await outcome).toMatchObject({ error: expect.objectContaining({ message: "Task cleanup is still pending" }) }); + expect(deps.verifyStartupCleanup).toHaveBeenCalledOnce(); + expect((await f.reload()).status).toBe("running"); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + drained = true; + expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" }); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + expect((await f.owner()).leaseOwner).toBeNull(); + } finally { resume(); spy.mockRestore(); } + }); + + it("rejects a cancellation commit when cleanup arrives after its final read", async () => { + const f = await fixture(true); + const deps = callbacks(db), commit = deps.setRunStatusFromLive.getMockImplementation()!; + deps.setRunStatusFromLive.mockImplementationOnce(async (...args) => { + expect(await requestNativeStartupCleanup(db, f.request)).toBe(true); + return commit(...args); + }); + deps.verifyStartupCleanup = vi.fn(run => verifyNativeStartupCleanup(db, run, () => f.stopReceipt({}, true))); + const control = createHeartbeatRunControl(db, deps); + expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "running" }); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + expect(deps.startNextQueuedRunForAgent).not.toHaveBeenCalled(); + expect(await control.cancelRunInternal(f.request.runId, "Operator Stop")).toMatchObject({ status: "cancelled" }); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + expect((await f.owner()).leaseOwner).toBeNull(); + }); + + it("restarts the real cancellation flow after remote Stop committed before Core acknowledgement", async () => { + const f = await fixture(true); + await requestNativeStartupCleanup(db, f.request); + await f.stopReceipt({}, true); + const restarted = createDb(temporary.connectionString), release = vi.fn(async () => { throw new Error("Stop receipt already persisted"); }); + const deps = callbacks(restarted); + deps.verifyStartupCleanup = run => verifyNativeStartupCleanup(restarted, run, release); + const control = createHeartbeatRunControl(restarted, deps); + const reconcile = vi.fn(async (scope: { companyId: string; runId: string }) => { + if (scope.runId !== f.request.runId) return; + expect(scope.companyId).toBe(companyId); + await control.cancelRunInternal(scope.runId, "Startup timeout recovered after restart"); + await acknowledgeNativeStartupCleanup(restarted, scope); + }); + await reconcileAbandonedExecutionControl(restarted, new Date(Date.now() + 1_000), { startupCleanup: reconcile }); + expect(reconcile.mock.calls.some(([scope]) => scope.runId === f.request.runId)).toBe(true); + expect(release).not.toHaveBeenCalled(); + expect((await f.reload()).status).toBe("cancelled"); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("complete"); + expect(await f.owner()).toMatchObject({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null }); + }); + + it.each(["providerLeaseId", "runId", "leaseId"])("rejects a Stop receipt for another %s", async field => { + const f = await fixture(); + await requestNativeStartupCleanup(db, f.request); + await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({ [field]: randomUUID() }))).rejects.toThrow("pending"); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + }); + + it("does not issue a Stop against a newer controller generation", async () => { + const f = await fixture(); + await requestNativeStartupCleanup(db, f.request); + await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + const release = vi.fn(f.stopReceipt); + await expect(verifyNativeStartupCleanup(db, await f.reload(), release)).rejects.toThrow("ownership changed"); + expect(release).not.toHaveBeenCalled(); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + }); + + it("rejects a stale generation that changes while Stop is in flight", async () => { + const f = await fixture(); + await requestNativeStartupCleanup(db, f.request); + await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => { + await f.stopReceipt(); + await db.update(nativeRunFinalizations).set({ controllerGeneration: 2 }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + })).rejects.toThrow("ownership changed"); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + }); + + it("preserves a final result that wins while verified cleanup is in flight", async () => { + const f = await fixture(); + await requestNativeStartupCleanup(db, f.request); + let resultId: string; + await expect(verifyNativeStartupCleanup(db, await f.reload(), async () => { + await f.stopReceipt(); + resultId = await f.acceptedResult(); + })).rejects.toThrow("ownership changed"); + expect(await f.owner()).toMatchObject({ resultId: resultId!, phase: "workspace_finalizing" }); + expect(readNativeStartupCleanup(await f.reload())?.state).toBe("pending"); + }); + + it.each(["native_adopted_runner_authentication_timeout", "native_execution_ownership_unverified"])("preserves the existing %s hold", async errorCode => { + const f = await fixture(); + await db.update(heartbeatRuns).set({ errorCode, nativePhase: "terminal_failure" }).where(eq(heartbeatRuns.id, f.request.runId)); + expect(await requestNativeStartupCleanup(db, f.request)).toBe(false); + const current = await f.reload(); + expect(current).toMatchObject({ status: "running", errorCode, nativePhase: "terminal_failure" }); + expect(readNativeStartupCleanup(current)).toBeNull(); + }); + + it("does not replace a final result with a timeout intent", async () => { + const f = await fixture(); + const resultId = await f.acceptedResult(); + expect(await requestNativeStartupCleanup(db, f.request)).toBe(false); + expect((await f.owner()).resultId).toBe(resultId); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + }); + it("promotes a pre-capability context before an ordinary deadline after the audited intent write fails", async () => { + const f = await fixture(); + await saveEarlyContext(f); + await db.execute(sql`create function reject_startup_intent() returns trigger language plpgsql as $$ begin + if new.action = 'native.startup_cleanup_requested' then raise exception 'Startup intent write unavailable'; end if; return new; end $$`); + await db.execute(sql`create trigger reject_startup_intent before insert on activity_log for each row execute function reject_startup_intent()`); + try { await expect(requestNativeStartupCleanup(db, f.request)).rejects.toThrow(); } + finally { await db.execute(sql`drop trigger reject_startup_intent on activity_log`); await db.execute(sql`drop function reject_startup_intent()`); } + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + await db.update(heartbeatRuns).set({ executionControlDeadlineAt: new Date(Date.now() - 1_000) }).where(eq(heartbeatRuns.id, f.request.runId)); + await reconcileAbandonedExecutionControl(db); + expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ contextId: f.request.contextId, requiresPreparationStop: false, state: "pending" }); + expect((await f.reload()).status).toBe("running"); + expect((await f.owner()).leaseOwner).toBe(f.request.leaseOwner); + await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt()); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await acknowledgeNativeStartupCleanup(db, f.request); + }); + + it.each([false, true])("recovers an early context without replacing stronger preparation proof (prepared=%s)", async prepared => { + const f = await fixture(); + await saveEarlyContext(f); + if (prepared) { const lease = await f.lease(); await db.update(environmentLeases).set({ metadata: { ...lease.metadata, + runnerTaskStartup: { contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId } } }).where(eq(environmentLeases.id, lease.id)); } + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + const onClaim = vi.fn(); + const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() }, onClaim }); + expect(result).toEqual([{ kind: "blocked", runId: f.request.runId, reason: NATIVE_STARTUP_CLEANUP_PENDING }]); + expect(onClaim).not.toHaveBeenCalled(); + expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: prepared, state: "pending" }); + await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, prepared)); + await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await acknowledgeNativeStartupCleanup(db, f.request); + }); + + it.each([false, true])("defers an early context to a concrete owner deadline (locally active=%s)", async active => { + const f = await fixture(); + await saveEarlyContext(f); + const owner = await f.owner(); + const result = await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + liveRunExecutions: active ? new Set([f.request.runId]) : undefined, + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } }); + expect(result).toMatchObject([{ kind: "awaiting_evidence", runId: f.request.runId, retryAt: owner.leaseExpiresAt }]); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + }); + + it("does not turn an early context into cleanup after the current-run checkpoint", async () => { + const f = await fixture(); + await saveEarlyContext(f); + await db.update(heartbeatRuns).set({ sessionIdAfter: "current-checkpoint" }).where(eq(heartbeatRuns.id, f.request.runId)); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } }); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + }); + + + it("locks the selected lease before choosing the stronger preparation proof", async () => { + const f = await fixture(), lease = await f.lease(); + let changed!: () => void, commit!: () => void; + const written = new Promise(resolve => { changed = resolve; }); + const release = new Promise(resolve => { commit = resolve; }); + const preparing = db.transaction(async tx => { + await tx.update(environmentLeases).set({ metadata: { ...lease.metadata, runnerTaskStartup: { + contextId: f.request.contextId, deadlineAtMs: f.request.deadlineAtMs, runId: f.request.runId, + } } }).where(eq(environmentLeases.id, lease.id)); + changed(); + await release; + }); + await written; + const requesting = requestNativeStartupCleanup(db, f.request); + try { + await vi.waitFor(async () => { + const rows = await db.execute(sql`select query from pg_stat_activity where wait_event_type = 'Lock' + and query like 'select%from "environment_leases"%'`); + expect(rows.length).toBeGreaterThan(0); + }); + } finally { commit(); } + await preparing; + await requesting; + expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ requiresPreparationStop: true }); + await expect(verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt())).rejects.toThrow("pending"); + await verifyNativeStartupCleanup(db, await f.reload(), () => f.stopReceipt({}, true)); + }); + + it("clears only its owned startup context before fallback and leaves no expired cleanup authority", async () => { + const f = await fixture(); + await saveEarlyContext(f); + const scope = { companyId, runId: f.request.runId, contextId: f.request.contextId, + leaseOwner: f.request.leaseOwner, controllerGeneration: 1 }; + await expect(clearNativeStartupContext(db, { ...scope, controllerGeneration: 2 })).rejects.toThrow("lost authority"); + expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined(); + await clearNativeStartupContext(db, scope); + expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeUndefined(); + await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(Date.now() - 1_000) }).where(eq(nativeRunFinalizations.runId, f.request.runId)); + await claimNativeRestartRecoveries({ db, restartKind: "hard", runIds: [f.request.runId], + controller: { bootId: randomUUID(), pid: process.pid, processStartedAt: new Date() } }); + expect(readNativeStartupCleanup(await f.reload())).toBeNull(); + }); + + it.each(["cleanup", "terminal"])("does not clear the startup breadcrumb after %s wins", async winner => { + const f = await fixture(); + await saveEarlyContext(f); + if (winner === "cleanup") await requestNativeStartupCleanup(db, f.request); + else await db.update(heartbeatRuns).set({ status: "cancelled" }).where(eq(heartbeatRuns.id, f.request.runId)); + await expect(clearNativeStartupContext(db, { companyId, runId: f.request.runId, contextId: f.request.contextId, + leaseOwner: f.request.leaseOwner, controllerGeneration: 1 })).rejects.toThrow("lost authority"); + expect((await f.reload()).runnerProfileJson?.nativeStartupContext).toBeDefined(); + if (winner === "cleanup") expect(readNativeStartupCleanup(await f.reload())).toMatchObject({ state: "pending" }); + }); + + it("requires exact task Stop proof when retrying cleanup of a never-dispatched replacement", async () => { + const f = await fixture(true), original = await f.lease(), pluginId = randomUUID(); + await db.insert(plugins).values({ id: pluginId, pluginKey: pluginId, packageName: "fixture-abandoned-task", version: "1", status: "ready", + manifestJson: { capabilities: ["environment.drivers.register"], environmentDrivers: [{ driverKey: "tasks", supportsTasks: true }] } as never }); + await db.update(environments).set({ companyId, config: { pluginKey: pluginId, driverKey: "tasks", driverConfig: {} } }) + .where(eq(environments.id, original.environmentId!)); + const fresh = await createOwnedSandbox(db, { companyId, environmentId: original.environmentId!, heartbeatRunId: f.request.runId, + issueId: original.issueId, provider: original.provider!, pluginId, driverKey: "tasks", config: {}, requestKey: original.providerLeaseId!, + metadata: { driver: "plugin", pluginId, pluginKey: pluginId, driverKey: "tasks", runnerTaskLease: true, agentId, + runnerTaskCreationContextId: f.request.contextId } }, async () => ({ providerLeaseId: "late-never-dispatched" })); + await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId, + environmentId: original.environmentId!, providerLeaseId: fresh.providerLeaseId }); + const creation = async () => (await db.select().from(sandboxCreationOperations).where(eq(sandboxCreationOperations.id, String(fresh.metadata!.creationOperationId))))[0]!; + const replacement = async () => (await environmentService(db).getLeaseById((await creation()).leaseId!))!; + expect((await replacement()).metadata?.runnerTaskDispatched).not.toBe(true); + await requestNativeStartupCleanup(db, { ...f.request, leaseId: original.id }); + await f.stopReceipt({}, true); + let proof: "void" | "running" | "foreign" | "exact" = "void"; + const calls: string[] = []; + const workers = { isRunning: () => true, getWorker: () => ({ supportedMethods: ["environmentTask", "environmentReleaseLease"] }), + drainStartup: vi.fn(async () => undefined), + call: async (_plugin: string, method: string, params: any) => { + calls.push(method === "environmentTask" ? params.operation.kind : method); + if (method === "environmentReleaseLease") return undefined; + if (params.operation.kind === "status") return proof === "void" ? undefined : { + kind: "status", taskId: proof === "foreign" ? "another-task" : params.taskId, + phase: proof === "running" ? "running" : "cancelled", executionStopped: proof !== "running", + }; + return { kind: "accepted", taskId: params.taskId }; + }, + } as unknown as PluginWorkerManager; + const runtime = environmentRuntimeService(db, { pluginWorkerManager: workers }); + const recovery = createHeartbeatRecovery(db, { ...callbacks(db), environmentRuntime: runtime, + environmentsSvc: environmentService(db), runtimeEnv: {}, activeRunExecutionPromises: new Set>(), + acknowledgeRemoteStop: async () => undefined, resumeRemoteStopComments: async () => undefined, + } as unknown as HeartbeatRecoveryDependencies); + const stopDeps = callbacks(db); + stopDeps.verifyStartupCleanup = run => verifyNativeStartupCleanup(db, run, async () => {}); + const control = createHeartbeatRunControl(db, stopDeps); + for (const missing of ["void", "running", "foreign"] as const) { + proof = missing; + const swept = await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } }); + const cancelled = await control.cancelRunInternal(f.request.runId, "Startup cleanup retry").then(() => true, () => false); + const lease = await replacement(); + expect({ destroyed: swept.destroyed, status: lease.status, verifiedReceipt: hasRemoteTerminationReceipt(lease), + creationState: (await creation()).state, cancelled, leaseOwner: (await f.owner()).leaseOwner }).toEqual({ + destroyed: 0, status: "pending_cleanup", verifiedReceipt: false, creationState: "cleanup_pending", cancelled: false, leaseOwner: f.request.leaseOwner, + }); + } + proof = "exact"; + expect((await recovery.sweepPendingCleanupLeases({ explicitRetry: { companyId, runId: f.request.runId, actorId: "operator" } })).destroyed).toBe(1); + expect(calls).toContain("stop"); + expect(calls).toContain("status"); + expect(calls).toContain("complete"); + expect(workers!.drainStartup).toHaveBeenCalledWith(f.request.contextId); + expect(hasRemoteTerminationReceipt(await replacement())).toBe(true); + expect((await creation()).state).toBe("resolved"); + expect(await control.cancelRunInternal(f.request.runId, "Startup cleanup retry")).toMatchObject({ status: "cancelled" }); + expect((await f.owner()).leaseOwner).toBeNull(); + }); + +}); diff --git a/server/src/__tests__/plugin-host-services-span.test.ts b/server/src/__tests__/plugin-host-services-span.test.ts index 01b7eb06c7a..a91e6c4e626 100644 --- a/server/src/__tests__/plugin-host-services-span.test.ts +++ b/server/src/__tests__/plugin-host-services-span.test.ts @@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => { }); describe("nested worker timing receipts",()=>{ + it("keeps lease verification beneath its worker and clamps provider state", () => { + mockRecordSpan.mockReset(); + const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 }; + const verifyParent = { ...rootParent, spanId: "2222222222222222" }; + mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent); + const now = Date.now(); + recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [ + { name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [ + { name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20, + attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } }, + ] }, + ] }, { traceparent: VALID_TRACEPARENT }); + expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent }); + expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent, + startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } }); + expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token"); + expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({}); + }); + + it("admits every closed lease phase and rejects dynamic phase names", () => { + mockRecordSpan.mockReset(); + const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup", + "lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell", + "lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"]; + for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT }); + expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([ + ...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other", + ]); + }); + it("uses host-generated parents, clamps content and bounds child clocks",()=>{ mockRecordSpan.mockReset(); const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1}; diff --git a/server/src/__tests__/redaction.test.ts b/server/src/__tests__/redaction.test.ts index 4ab09bbd80e..5d060d67d1a 100644 --- a/server/src/__tests__/redaction.test.ts +++ b/server/src/__tests__/redaction.test.ts @@ -899,3 +899,73 @@ second-line\" status=401`, }); }); }); + +describe("redaction of long letter/digit runs", () => { + const typescriptSource = [ + "export function compactRunLogChunk(chunk: string, maxChars = 65536) {", + " const normalized = redactSensitiveText(chunk);", + " if (normalized.length <= maxChars) return normalized;", + " return normalized.slice(0, maxChars);", + "}", + "", + ].join("\n"); + + function base64OfTypeScriptSource(length: number) { + const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length)); + return Buffer.from(repeatedSource).toString("base64").slice(0, length); + } + + function redactWithinTimeBudget(input: string) { + const startedAt = performance.now(); + const output = redactSensitiveText(input); + expect(performance.now() - startedAt).toBeLessThan(1_000); + return output; + } + + it("redacts secret field names that carry a prefix in plain and escaped JSON", () => { + const cases = [ + [`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`], + [`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`], + [`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`], + [`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`], + [`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`], + [`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`], + [String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`], + [ + String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`, + String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`, + ], + [String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`], + ]; + + for (const [input, expected] of cases) { + expect(redactSensitiveText(input)).toBe(expected); + expect(redactSensitiveText(expected)).toBe(expected); + } + }); + + it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => { + const blob = base64OfTypeScriptSource(48 * 1024); + expect(blob).toMatch(/^[A-Za-z0-9]+$/); + + const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`; + expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand); + + const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } }); + expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine); + const serializedToolCallLine = JSON.stringify(toolCallLine); + expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine); + + const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`); + expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`); + }); + + it("scans long dash-separated runs in linear time", () => { + for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) { + expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`)) + .toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`); + expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`)) + .toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`); + } + }); +}); diff --git a/server/src/__tests__/remote-workspace-file-resources.test.ts b/server/src/__tests__/remote-workspace-file-resources.test.ts index 38d31a618f7..393917f0c33 100644 --- a/server/src/__tests__/remote-workspace-file-resources.test.ts +++ b/server/src/__tests__/remote-workspace-file-resources.test.ts @@ -152,15 +152,20 @@ describeDatabase("remote workspace ordinary file routes", () => { }); const runtime = { execute, syncOut } as unknown as Pick; const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: runtime }); + const app = filesApp(ids.company, remoteFiles); + return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise) => { beforeCopy = callback; } }; + } + + function filesApp(companyId: string, remoteFiles: ReturnType) { const app = express(); app.use(express.json()); app.use((req, _res, next) => { - req.actor = { type: "board", userId: "local-board", companyIds: [ids.company], source: "local_implicit", isInstanceAdmin: true } as Request["actor"]; + req.actor = { type: "board", userId: "local-board", companyIds: [companyId], source: "local_implicit", isInstanceAdmin: true } as Request["actor"]; next(); }); app.use("/api", fileResourceRoutes(db, { remoteFiles })); app.use(errorHandler); - return { root, ids, authority, execute, syncOut, remoteFiles, app, beforeCopy: (callback: () => Promise) => { beforeCopy = callback; } }; + return app; } it("lists, resolves, previews, and downloads live bytes without a controller checkout", async () => { @@ -323,7 +328,7 @@ describeDatabase("remote workspace ordinary file routes", () => { { workspace: "execution", path: "docs/result.md" })).rejects.toMatchObject({ status: 422, details: { code: "file_changed" } }); }); - async function savedFileFixture() { + async function savedFileFixture(keepLocation = false) { const f = await fixture(), storageRoot = await fs.mkdtemp(path.join(os.tmpdir(), "saved-file-budget-")); directories.push(storageRoot); const storage = createStorageService(createLocalDiskStorageProvider(path.join(storageRoot, "objects"))); @@ -335,11 +340,113 @@ describeDatabase("remote workspace ordinary file routes", () => { const saved = await uploadWorkspaceSnapshot({ db, storage, companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id, archivePath, manifestDigest: archive.manifestDigest }); await db.update(executionWorkspaceSnapshots).set({ status: "committed" }).where(eq(executionWorkspaceSnapshots.id, snapshot.id)); - await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: null, snapshotId: snapshot.id } } }) + await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, location: keepLocation ? f.authority.location : null, snapshotId: snapshot.id } } }) .where(eq(executionWorkspaces.id, f.ids.workspace)); return { ...f, storage, saved, input: { companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId: snapshot.id } }; } + it("serves committed saved bytes through ordinary routes while the live allocation is unavailable", async () => { + const f = await savedFileFixture(true); + f.execute.mockRejectedValue(new Error("synthetic live allocation unavailable")); + const load = createSavedWorkspaceLoader(db, f.storage), closed: Promise[] = []; + const loadSavedWorkspace = vi.fn(async (input: Parameters[0]) => { + const saved = await load(input); + let finish!: () => void, fail!: (error: unknown) => void; + closed.push(new Promise((resolve, reject) => { finish = resolve; fail = reject; })); + return { ...saved, cleanup: async () => { + try { await saved.cleanup(); finish(); } catch (error) { fail(error); throw error; } + } }; + }); + const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } }); + const app = filesApp(f.ids.company, remoteFiles), base = `/api/issues/${f.ids.issue}/file-resources`; + const preview = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md" }).expect(200); + expect(preview.body).toMatchObject({ content: { data: "Remote useful result\n" }, resource: { source: { kind: "saved", snapshotId: f.saved.id } } }); + const listed = await request(app).get(`${base}/list`).query({ workspace: "execution", path: "docs" }).expect(200); + expect(listed.body).toMatchObject({ workspace: { source: { kind: "saved", snapshotId: f.saved.id } } }); + const resolved = await request(app).get(`${base}/resolve`).query({ workspace: "execution", path: "docs/result.md" }).expect(200); + expect(resolved.body).toMatchObject({ source: { kind: "saved", snapshotId: f.saved.id }, capabilities: { preview: true, download: true } }); + const download = await request(app).get(`${base}/content`).query({ workspace: "execution", path: "docs/result.md", download: true }).expect(200); + expect(download.text).toBe("Remote useful result\n"); + expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled(); + expect(loadSavedWorkspace).toHaveBeenCalledTimes(4); + // A streamed response can finish at the client before its server finally block. + await Promise.all(closed); + for (const loaded of loadSavedWorkspace.mock.results) await expect(fs.stat((await loaded.value).root)).rejects.toMatchObject({ code: "ENOENT" }); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.ids.workspace)); + expect(workspace!.metadata?.remoteWorkspace).toMatchObject({ location: f.authority.location, dirty: false, writerRunId: null }); + }); + + it.each(["dirty", "writer"] as const)("keeps %s work on the live path and never substitutes its older save", async state => { + const f = await savedFileFixture(true); + let writerRunId: string | null = null; + if (state === "writer") { + const [agent] = await db.insert(agents).values({ companyId: f.ids.company, name: "Current workspace writer" }).returning(); + const [run] = await db.insert(heartbeatRuns).values({ companyId: f.ids.company, agentId: agent!.id, status: "running" }).returning(); + writerRunId = run!.id; + } + await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: state === "dirty", writerRunId } } }) + .where(eq(executionWorkspaces.id, f.ids.workspace)); + await fs.writeFile(path.join(f.root, "docs", "result.md"), "Unsaved live result\n"); + const loadSavedWorkspace = vi.fn(createSavedWorkspaceLoader(db, f.storage)); + const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } }); + const app = filesApp(f.ids.company, remoteFiles), route = `/api/issues/${f.ids.issue}/file-resources/content`; + const current = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(200); + expect(current.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Unsaved live result\n" } }); + f.execute.mockRejectedValueOnce(new Error("synthetic live allocation unavailable")); + const unavailable = await request(app).get(route).query({ workspace: "execution", path: "docs/result.md" }).expect(422); + expect(unavailable.body).not.toHaveProperty("content"); expect(loadSavedWorkspace).not.toHaveBeenCalled(); + }); + + it.each([true, false])("preserves the optional saved-loader boundary with a recorded location=%s", async keepLocation => { + const f = await savedFileFixture(keepLocation); + const remoteFiles = createRemoteWorkspaceFiles(db, { environmentRuntime: { execute: f.execute, syncOut: f.syncOut } }); + const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`) + .query({ workspace: "execution", path: "docs/result.md" }).expect(keepLocation ? 200 : 422); + if (keepLocation) { + expect(response.body).toMatchObject({ resource: { source: { kind: "live" } }, content: { data: "Remote useful result\n" } }); + expect(f.execute).toHaveBeenCalled(); + } else { + expect(response.body).not.toHaveProperty("content"); expect(f.execute).not.toHaveBeenCalled(); + } + }); + + it.each(["company", "workspace", "uncommitted"] as const)("rejects a saved snapshot with an invalid %s binding before object retrieval", async invalid => { + const f = await savedFileFixture(true); + if (invalid === "company") { + const [other] = await db.insert(companies).values({ name: "Other saved-file company" }).returning(); + await db.update(executionWorkspaceSnapshots).set({ companyId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id)); + } else if (invalid === "workspace") { + const [other] = await db.insert(executionWorkspaces).values({ companyId: f.ids.company, mode: "isolated_workspace", + strategyType: "project_primary", providerType: "cloud_sandbox", name: "Other saved-file workspace" }).returning(); + await db.update(executionWorkspaceSnapshots).set({ executionWorkspaceId: other!.id }).where(eq(executionWorkspaceSnapshots.id, f.saved.id)); + } else await db.update(executionWorkspaceSnapshots).set({ status: "uploaded" }).where(eq(executionWorkspaceSnapshots.id, f.saved.id)); + const getObject = vi.spyOn(f.storage, "getObject"); + const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace: createSavedWorkspaceLoader(db, f.storage), + environmentRuntime: { execute: f.execute, syncOut: f.syncOut } }); + const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`) + .query({ workspace: "execution", path: "docs/result.md" }); + expect(response.status).toBeGreaterThanOrEqual(400); expect(response.body).not.toHaveProperty("content"); + expect(getObject).not.toHaveBeenCalled(); expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled(); + getObject.mockRestore(); + }); + + it("fences a saved read if a new writer claims the workspace during extraction and removes its scratch files", async () => { + const f = await savedFileFixture(true), load = createSavedWorkspaceLoader(db, f.storage); + let savedRoot: string | undefined; + const loadSavedWorkspace = async (input: Parameters[0]) => { + const saved = await load(input); savedRoot = saved.root; + await db.update(executionWorkspaces).set({ metadata: { remoteWorkspace: { ...f.authority, snapshotId: f.saved.id, dirty: true, epoch: 2 } } }) + .where(eq(executionWorkspaces.id, f.ids.workspace)); + return saved; + }; + const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace, environmentRuntime: { execute: f.execute, syncOut: f.syncOut } }); + const response = await request(filesApp(f.ids.company, remoteFiles)).get(`/api/issues/${f.ids.issue}/file-resources/content`) + .query({ workspace: "execution", path: "docs/result.md" }).expect(422); + expect(response.body.details.code).toBe("workspace_changed"); expect(response.body).not.toHaveProperty("content"); + expect(savedRoot).toBeDefined(); await expect(fs.stat(savedRoot!)).rejects.toMatchObject({ code: "ENOENT" }); + expect(f.execute).not.toHaveBeenCalled(); expect(f.syncOut).not.toHaveBeenCalled(); + }); + it("shares fail-fast saved Files admission across loader instances until owned cleanup completes", async () => { const f = await savedFileFixture(), availableBytes = async () => 8n * 1024n ** 3n; const first = createSavedWorkspaceLoader(db, f.storage, { availableBytes }); @@ -625,7 +732,7 @@ describeDatabase("remote workspace ordinary file routes", () => { }); it.each(["confirmed_stop", "unknown_stop", "dirty", "foreign_receipt"])( - "uses saved bytes only for a clean workspace with an exact stopped receipt (%s)", async state => { + "uses clean saved authority independently of an allocation's stop receipt (%s)", async state => { const f = await fixture(); const snapshotId = randomUUID(); const agentId = randomUUID(); @@ -650,7 +757,7 @@ describeDatabase("remote workspace ordinary file routes", () => { const remoteFiles = createRemoteWorkspaceFiles(db, { loadSavedWorkspace }); const content = workspaceFileResourceService(db, undefined, { remoteFiles }).readContent(f.ids.issue, { workspace: "execution", path: "docs/result.md" }); - if (state === "confirmed_stop") { + if (state !== "dirty") { await expect(content).resolves.toMatchObject({ content: { data: "Saved stopped result\n" }, resource: { source: { kind: "saved", snapshotId, savedAt: "2026-10-30T12:00:00.000Z" } } }); expect(loadSavedWorkspace).toHaveBeenCalledWith({ companyId: f.ids.company, workspaceId: f.ids.workspace, snapshotId }); diff --git a/server/src/__tests__/sandbox-provider-schema-reads.test.ts b/server/src/__tests__/sandbox-provider-schema-reads.test.ts index 55f81f8bfbe..fd82c852bbc 100644 --- a/server/src/__tests__/sandbox-provider-schema-reads.test.ts +++ b/server/src/__tests__/sandbox-provider-schema-reads.test.ts @@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto"; import { eq, sql, type SQL } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { createDb, plugins, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared"; import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js"; import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js"; @@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => { const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey }); async function measure(client: Db, operation: () => Promise) { const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now); installDatabaseResponseCapture(client, { plugins }, capture); try { const value = await operation(); capture.flush(); - const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"); + const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"); return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0), calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) }; } finally { capture.flush(); } diff --git a/server/src/__tests__/storage-s3-multipart.test.ts b/server/src/__tests__/storage-s3-multipart.test.ts index 343bb0ddcd1..6896989b987 100644 --- a/server/src/__tests__/storage-s3-multipart.test.ts +++ b/server/src/__tests__/storage-s3-multipart.test.ts @@ -1,5 +1,8 @@ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, DeleteObjectCommand, PutObjectCommand, S3Client } from "@aws-sdk/client-s3"; import { Readable } from "node:stream"; +import { createHash } from "node:crypto"; +import { createServer, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; import { describe, expect, it, vi } from "vitest"; import { putS3Multipart } from "../storage/s3-multipart.js"; import { createS3StorageProvider } from "../storage/s3-provider.js"; @@ -7,6 +10,94 @@ import { createStorageService } from "../storage/service.js"; import { STORAGE_FAILURE_CLEANUP_MS } from "../storage/failure-cleanup.js"; describe("streamed S3 multipart uploads", () => { + it("sends concurrent exact part bodies through the real S3 HTTP client", async () => { + const received = new Map(); + const responses = new Map(); + let completedParts = ""; + const server = createServer((request, response) => { + const url = new URL(request.url!, "http://localhost"); + const digest = createHash("sha256"); + let text = ""; + request.on("data", chunk => { digest.update(chunk); if (request.method === "POST") text += chunk; }); + request.on("end", () => { + if (request.method === "PUT") { + const number = Number(url.searchParams.get("partNumber")); + received.set(number, digest.digest("hex")); + responses.set(number, response); + } else if (url.searchParams.has("uploads")) { + response.end('upload'); + } else { + completedParts = text; + response.end('complete'); + } + }); + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const client = new S3Client({ region: "us-east-1", endpoint: `http://127.0.0.1:${(server.address() as AddressInfo).port}`, + forcePathStyle: true, requestChecksumCalculation: "WHEN_REQUIRED", credentials: { accessKeyId: "fixture", secretAccessKey: "fixture" } }); + const cancellation = new AbortController(); + const body = Readable.from((async function* () { + for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024, Math.floor(index / 128) + 1); + })()); + const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body, signal: cancellation.signal }).catch(error => error); + try { + await expect.poll(() => received.size, { timeout: 3000 }).toBe(4); + for (const number of [4, 2, 3, 1]) { + expect(received.get(number)).toBe(createHash("sha256").update(Buffer.alloc(8 * 1024 * 1024, number)).digest("hex")); + responses.get(number)!.writeHead(200, { ETag: `part-${number}` }).end(); + } + expect(await saving).toBeUndefined(); + expect([...completedParts.matchAll(/(\d+)<\/PartNumber>/g)].map(match => Number(match[1]))).toEqual([1, 2, 3, 4]); + } finally { + cancellation.abort(); + await saving; + client.destroy(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); + + it("overlaps slow part acknowledgements and completes exact bytes in source order", async () => { + const client = new S3Client({ region: "us-east-1" }); + const received = new Map(); + const acknowledgements = new Map void>(); + let completed = false; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + const chunks: Buffer[] = []; + for await (const chunk of command.input.Body as Readable) chunks.push(chunk); + received.set(command.input.PartNumber, Buffer.concat(chunks)); + await new Promise(resolve => acknowledgements.set(command.input.PartNumber, resolve)); + return { ETag: `part-${command.input.PartNumber}` }; + } + expect(command).toBeInstanceOf(CompleteMultipartUploadCommand); + expect(command.input.MultipartUpload.Parts).toEqual([1, 2, 3, 4].map(PartNumber => ({ PartNumber, ETag: `part-${PartNumber}` }))); + completed = true; + return {}; + }); + const length = 25 * 1024 * 1024; + const body = Readable.from((async function* () { + for (let offset = 0; offset < length; offset += 64 * 1024) yield Buffer.alloc(Math.min(64 * 1024, length - offset), offset / (64 * 1024) % 251); + })()); + const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: length, body }); + try { + await expect.poll(() => received.size, { timeout: 1000 }).toBe(4); + expect(completed).toBe(false); + for (const number of [4, 2, 3, 1]) acknowledgements.get(number)!(); + await saving; + const actual = Buffer.concat([1, 2, 3, 4].map(number => received.get(number)!)); + expect(actual.length).toBe(length); + for (let offset = 0; offset < length; offset += 64 * 1024) { + expect(actual.subarray(offset, Math.min(length, offset + 64 * 1024)).every(byte => byte === offset / (64 * 1024) % 251)).toBe(true); + } + expect(completed).toBe(true); + } finally { + const release = setInterval(() => { for (const acknowledge of acknowledgements.values()) acknowledge(); }, 1); + try { await saving; } finally { clearInterval(release); send.mockRestore(); client.destroy(); } + } + }); + it("streams exact ordered parts without buffering the full response", async () => { const client = new S3Client({ region: "us-east-1" }); const lengths: number[] = []; @@ -33,6 +124,109 @@ describe("streamed S3 multipart uploads", () => { expect(lengths).toEqual([8, 8, 1].map(mb => mb * 1024 * 1024)); } finally { send.mockRestore(); client.destroy(); } }); + it("keeps source backpressure and at most four part requests while acknowledgements stall", async () => { + const client = new S3Client({ region: "us-east-1" }); + let releaseReading!: () => void, releaseAcknowledgements!: () => void; + const reading = new Promise(resolve => { releaseReading = resolve; }); + const acknowledgements = new Promise(resolve => { releaseAcknowledgements = resolve; }); + let yielded = 0, active = 0, peak = 0, received = 0; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + active++; peak = Math.max(peak, active); + await reading; + for await (const chunk of command.input.Body as Readable) expect(chunk.length).toBeLessThanOrEqual(64 * 1024); + received++; + await acknowledgements; + active--; + return { ETag: `part-${command.input.PartNumber}` }; + } + expect(command).toBeInstanceOf(CompleteMultipartUploadCommand); + return {}; + }); + const body = Readable.from((async function* () { + for (let index = 0; index < 1600; index++) { yielded++; yield Buffer.alloc(64 * 1024); } + })()); + const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 100 * 1024 * 1024, body }); + try { + await expect.poll(() => active).toBe(1); + expect(yielded).toBeLessThanOrEqual(3); + releaseReading(); + await expect.poll(() => received).toBe(4); + expect(yielded).toBeLessThanOrEqual(4 * 128 + 3); + expect(peak).toBe(4); + releaseAcknowledgements(); + await saving; + expect(received).toBe(13); + expect(yielded).toBe(1600); + expect(peak).toBe(4); + } finally { releaseReading(); releaseAcknowledgements(); await saving; send.mockRestore(); client.destroy(); } + }); + + it("settles cancelled siblings before aborting a failed multipart upload", async () => { + const client = new S3Client({ region: "us-east-1" }); + const cancellations: AbortSignal[] = []; + let failPart!: () => void, releaseSiblings!: () => void; + const settlement = new Promise(resolve => { releaseSiblings = resolve; }); + let received = 0, settled = 0, aborted = false, completed = false; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + for await (const _chunk of command.input.Body as Readable) {} + received++; + if (command.input.PartNumber === 2) return new Promise((_resolve, reject) => { failPart = () => reject(new Error("second part failed")); }); + cancellations.push(options.abortSignal); + await new Promise(resolve => options.abortSignal.addEventListener("abort", () => resolve(), { once: true })); + await settlement; + settled++; + throw options.abortSignal.reason; + } + if (command instanceof CompleteMultipartUploadCommand) { completed = true; return {}; } + expect(command).toBeInstanceOf(AbortMultipartUploadCommand); + expect(settled).toBe(3); + aborted = true; + return {}; + }); + const body = Readable.from((async function* () { for (let index = 0; index < 512; index++) yield Buffer.alloc(64 * 1024); })()); + const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 32 * 1024 * 1024, body }).catch(error => error); + try { + await expect.poll(() => received).toBe(4); + failPart(); + await expect.poll(() => cancellations.every(signal => signal.aborted)).toBe(true); + expect(aborted).toBe(false); + releaseSiblings(); + expect((await saving).message).toBe("second part failed"); + expect(aborted).toBe(true); + expect(completed).toBe(false); + expect(body.destroyed).toBe(true); + } finally { failPart?.(); releaseSiblings(); await saving; send.mockRestore(); client.destroy(); } + }); + + it.each(["truncated", "excess", "source error"])("rejects %s input without publishing an object", async kind => { + const client = new S3Client({ region: "us-east-1" }); + let aborted = false; + const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + for await (const _chunk of command.input.Body as Readable) {} + return { ETag: `part-${command.input.PartNumber}` }; + } + expect(command).toBeInstanceOf(AbortMultipartUploadCommand); + aborted = true; + return {}; + }); + const expected = { truncated: "S3 upload source was truncated", excess: "S3 upload source exceeds its declared size", "source error": "source unavailable" }; + const body = Readable.from((async function* () { + for (let index = 0; index < (kind === "excess" ? 274 : 270); index++) yield Buffer.alloc(64 * 1024); + if (kind === "source error") throw new Error("source unavailable"); + })()); + try { + await expect(putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "application/octet-stream", contentLength: 17 * 1024 * 1024, body })).rejects.toThrow(expected[kind]); + expect(aborted).toBe(true); + expect(body.destroyed).toBe(true); + } finally { send.mockRestore(); client.destroy(); } + }); + it("aborts an incomplete upload and closes its source on failure", async () => { const client = new S3Client({ region: "us-east-1" }); const send = vi.spyOn(client, "send").mockImplementation(async (command: any) => { @@ -79,4 +273,43 @@ describe("streamed S3 multipart uploads", () => { await expect(storage.putFile({ companyId: "company", namespace: "skill-packages", originalFilename: "next", contentType: "text/plain", body: Buffer.from("next") })).resolves.toMatchObject({ byteSize: 4 }); } finally { send.mockRestore(); vi.useRealTimers(); } }); + + it("bounds cancellation while an asynchronous source read is stalled", async () => { + vi.useFakeTimers(); + const client = new S3Client({ region: "us-east-1" }); + const cancellation = new AbortController(); + let release!: () => void, waiting = false, settled = false; + const sourceGate = new Promise(resolve => { release = resolve; }); + const body = Readable.from((async function* () { + yield Buffer.alloc(64 * 1024); + waiting = true; + await sourceGate; + yield Buffer.alloc(64 * 1024); + })()); + const send = vi.spyOn(client, "send").mockImplementation(async (command: any, options: any) => { + if (command instanceof CreateMultipartUploadCommand) return { UploadId: "upload" }; + if (command instanceof UploadPartCommand) { + (command.input.Body as Readable).resume(); + return new Promise((_resolve, reject) => options.abortSignal.addEventListener("abort", () => reject(options.abortSignal.reason), { once: true })); + } + expect(command).toBeInstanceOf(AbortMultipartUploadCommand); + return {}; + }); + const saving = putS3Multipart(client, "bucket", "key", { objectKey: "key", contentType: "text/plain", contentLength: 128 * 1024, body, signal: cancellation.signal }) + .catch(error => error).finally(() => { settled = true; }); + try { + await vi.advanceTimersByTimeAsync(0); + expect(waiting).toBe(true); + cancellation.abort(new Error("cancelled during source read")); + await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1); + expect(settled).toBe(true); + expect((await saving).message).toBe("cancelled during source read"); + expect(body.destroyed).toBe(true); + } finally { + release(); + await vi.advanceTimersByTimeAsync(STORAGE_FAILURE_CLEANUP_MS + 1); + await saving; + send.mockRestore(); client.destroy(); vi.useRealTimers(); + } + }); }); diff --git a/server/src/__tests__/tool-effective-assignments.test.ts b/server/src/__tests__/tool-effective-assignments.test.ts index 4b99a23eec2..f22b8fd0cb6 100644 --- a/server/src/__tests__/tool-effective-assignments.test.ts +++ b/server/src/__tests__/tool-effective-assignments.test.ts @@ -5,7 +5,7 @@ import { agents, companies, createDb, toolApplications, toolCatalogEntries, toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js"; import { toolAccessService } from "../services/tool-access.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; @@ -76,6 +76,20 @@ describeDatabase("effective tool assignments", () => { expect(full.allowedTools[0].annotations).toEqual(f.read.annotations); }); + it("keeps exclusions local to each profile when effective profiles grant the same tool", async () => { + const f = await fixture(); + await db.update(toolProfiles).set({ defaultAction: "allow" }).where(eq(toolProfiles.id, f.profile.id)); + await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profile.id, + selectorType: "catalog_entry", effect: "exclude", catalogEntryId: f.read.id }); + const [second] = await db.insert(toolProfiles).values({ companyId: f.company.id, profileKey: "second", name: "Second", defaultAction: "deny" }).returning(); + await db.insert(toolProfileBindings).values({ companyId: f.company.id, profileId: second.id, targetType: "agent", targetId: f.agent.id }); + const [include] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: second.id, + selectorType: "catalog_entry", effect: "include", catalogEntryId: f.read.id }).returning(); + expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.read.id, f.write.id]); + await db.delete(toolProfileEntries).where(eq(toolProfileEntries.id, include.id)); + expect((await compare(f)).assignments.allowedTools.map(tool => tool.id)).toEqual([f.write.id]); + }); + it("keeps unresolved exact names and observes changed catalog, profile and binding state", async () => { const f = await fixture(); await db.insert(toolProfileEntries).values([ @@ -149,14 +163,14 @@ describeDatabase("effective tool assignments", () => { const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id); expect(before.allowedTools).toHaveLength(102); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture); let scoped: Awaited>; try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); } finally { capture.flush(); } expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0); expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024); - const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" + const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_catalog_entries" && event.operation === "select"); expect(catalogQueries).toHaveLength(1); expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 }); @@ -192,13 +206,13 @@ describeDatabase("effective tool assignments", () => { const service = toolAccessService(db), before = await compare(f); const savedRules = await db.select().from(toolProfileEntries); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture); try { const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); capture.flush(); - const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries"); - const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings"); + const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries"); + const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings"); expect(rules).toHaveLength(1); expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096); expect(rules[0]).toMatchObject({ calls: 1, rows: 2 }); diff --git a/server/src/__tests__/tool-profile-matching.test.ts b/server/src/__tests__/tool-profile-matching.test.ts new file mode 100644 index 00000000000..4c8498ed6bd --- /dev/null +++ b/server/src/__tests__/tool-profile-matching.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; +import { profileCatalogMatcher, profileEntryMatchesCatalog, type ProfileCatalogMatch, type ProfileEntryMatch } from "../services/tool-access/profiles/projection.js"; + +describe("indexed profile catalog matching", () => { + it("preserves all selector kinds, null matches and exact strings in mixed rule sets", () => { + const catalog: ProfileCatalogMatch[] = Array.from({ length: 75 }, (_, index) => ({ + id: `tool-${index}`, applicationId: index % 3 ? `app-${index % 5}` : null, + connectionId: `connection-${index % 7}`, toolName: index % 4 ? `tool-${index}` : "quoted' path\\tool_🙂", + riskLevel: index % 2 ? "read" : "write", + })); + const selectors = ["application", "connection", "catalog_entry", "tool_name", "risk_level", "unknown"]; + const rules: ProfileEntryMatch[] = catalog.flatMap(entry => selectors.map(selectorType => ({ + selectorType, applicationId: entry.applicationId, connectionId: entry.connectionId, + catalogEntryId: entry.id, toolName: entry.toolName, riskLevel: entry.riskLevel, + }))); + for (const selected of [[], ...rules.map(rule => [rule]), rules.slice(0, 25), rules]) { + const matches = profileCatalogMatcher(selected); + expect(catalog.filter(matches)).toEqual(catalog.filter(entry => selected.some(rule => profileEntryMatchesCatalog(rule, entry)))); + } + }); +}); diff --git a/server/src/__tests__/tool-profile-pending-projection.test.ts b/server/src/__tests__/tool-profile-pending-projection.test.ts new file mode 100644 index 00000000000..6b2764acc44 --- /dev/null +++ b/server/src/__tests__/tool-profile-pending-projection.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from "vitest"; +import { pendingNewToolsForProfile, profileCoversCatalogScope, profileEntryMatchesCatalog, type ProfileCatalogEntry } from "../services/tool-access/profiles/projection.js"; +import type { toolProfiles, toolProfileEntries } from "@paperclipai/db"; + +type Input = Parameters[0]; +const date = new Date("2026-01-01T00:00:00Z"); +const later = new Date("2026-01-02T00:00:00Z"); +function profile(overrides: Partial = {}): typeof toolProfiles.$inferSelect { + return { id: "profile", companyId: "company", profileKey: "selected", name: "Selected", description: null, status: "active", defaultAction: "deny", newToolsReviewedAt: null, metadata: {}, createdAt: date, updatedAt: date, ...overrides }; +} +function rule(overrides: Partial): typeof toolProfileEntries.$inferSelect { + return { id: "rule", companyId: "company", profileId: "profile", selectorType: "catalog_entry", effect: "include", applicationId: null, connectionId: null, catalogEntryId: null, toolName: null, riskLevel: null, conditions: null, createdAt: date, updatedAt: date, ...overrides }; +} +function legacy(input: Input) { + if (input.profile.status !== "active" || input.profile.defaultAction !== "deny") return []; + const catalogById = new Map(input.catalog.map(entry => [entry.id, entry])); + return input.catalog.filter(entry => (entry.status === "active" || entry.status === "quarantined") + && entry.firstSeenAt > (input.profile.newToolsReviewedAt ?? input.profile.createdAt) + && input.entries.some(rule => profileCoversCatalogScope({ entry: rule, catalogEntry: entry, catalogById })) + && !input.entries.some(rule => profileEntryMatchesCatalog(rule, entry))).map(entry => ({ + catalogEntryId: entry.id, applicationId: entry.applicationId, + applicationName: entry.applicationId ? input.applicationsById?.get(entry.applicationId)?.name ?? null : null, + connectionId: entry.connectionId, connectionName: input.connectionsById?.get(entry.connectionId)?.name ?? null, + toolName: entry.toolName, title: entry.title ?? null, description: entry.description ?? null, + capability: entry.riskLevel, riskLevel: entry.riskLevel, addedAt: entry.firstSeenAt, firstSeenAt: entry.firstSeenAt, + })); +} +const catalog: ProfileCatalogEntry[] = Array.from({ length: 48 }, (_, i): ProfileCatalogEntry => ({ + id: `tool-${i}`, applicationId: i % 3 ? `app-${i % 4}` : null, + connectionId: `connection-${i % 6}`, toolName: i % 7 ? `name-${i}` : "quoted' path\\tool_🙂", + riskLevel: i % 2 ? "read" : "write", status: i % 5 ? "active" : "quarantined", + firstSeenAt: i % 11 ? later : date, +})); + +describe("pending profile projection", () => { + it("matches the previous ordered projection for every selector, effect, watermark and nullable scope", () => { + const rules = catalog.flatMap(entry => (["include", "exclude"] as const).flatMap(effect => [ + rule({ effect, selectorType: "application", applicationId: entry.applicationId }), + rule({ effect, selectorType: "connection", connectionId: entry.connectionId }), + rule({ effect, selectorType: "catalog_entry", catalogEntryId: entry.id }), + rule({ effect, selectorType: "tool_name", toolName: entry.toolName }), + rule({ effect, selectorType: "risk_level", riskLevel: entry.riskLevel }), + ])); + const scopes = [rule({ selectorType: "application", applicationId: null }), rule({ selectorType: "connection", connectionId: "connection-1" }), rule({ catalogEntryId: "tool-2" }), rule({ catalogEntryId: "missing" })]; + for (const scope of scopes) for (const selected of [[], ...rules.map(entry => [entry]), rules.slice(0, 50), rules]) { + const input = { profile: profile(), catalog, entries: [scope, ...selected], applicationsById: new Map([["app-1", { id: "app-1", name: "Application" }]]), connectionsById: new Map([["connection-1", { id: "connection-1", name: "Connection" }]]) }; + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + } + for (const changes of [{ status: "archived" as const }, { defaultAction: "allow" as const }, { newToolsReviewedAt: later }]) { + const input = { profile: profile(changes), catalog, entries: scopes }; + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + } + }); + + it("does not broaden an individual tool's null application scope to unrelated null applications", () => { + const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-0" })], catalog }; + const ids = pendingNewToolsForProfile(input).map(entry => entry.catalogEntryId); + expect(ids).toContain("tool-6"); + expect(ids).not.toContain("tool-3"); + expect(pendingNewToolsForProfile(input)).toEqual(legacy(input)); + }); + + it("recomputes catalog references, exclusions and scope after the input changes", () => { + const input = { profile: profile(), entries: [rule({ catalogEntryId: "tool-2" })], catalog }; + expect(pendingNewToolsForProfile(input).length).toBeGreaterThan(0); + expect(pendingNewToolsForProfile({ ...input, catalog: catalog.filter(entry => entry.id !== "tool-2") })).toEqual([]); + expect(pendingNewToolsForProfile({ ...input, entries: [rule({ catalogEntryId: "tool-2", effect: "exclude" })] })).toEqual([]); + }); +}); diff --git a/server/src/__tests__/tool-profile-review-service.test.ts b/server/src/__tests__/tool-profile-review-service.test.ts new file mode 100644 index 00000000000..edfa2a4d33c --- /dev/null +++ b/server/src/__tests__/tool-profile-review-service.test.ts @@ -0,0 +1,36 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { eq } from "drizzle-orm"; +import { createDb, toolCatalogEntries, toolProfileEntries, toolProfiles } from "@paperclipai/db"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createProfileProjectionFixture } from "./helpers/tool-profile-projection-fixture.js"; +import { createToolProfileService } from "../services/tool-access/profiles/service.js"; +import { createToolAppAttention } from "../services/tool-access/catalog/attention.js"; +const support = await getEmbeddedPostgresTestSupport(); +const describeDatabase = support.supported ? describe : describe.skip; +describeDatabase("pending profile tools through profile and attention services", () => { + let db: ReturnType, database: Awaited>; + beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("profile-review-service-"); db = createDb(database.connectionString); }); + afterAll(async () => { await db?.$client.end({ timeout: 1 }); await database?.cleanup(); }); + it("preserves fresh pending counts in both owning services after review and catalog changes", async () => { + const f = await createProfileProjectionFixture(db, { profiles: 2, tools: 8, rules: 4 }); + const now = () => new Date("2026-02-01T00:00:00Z"), service = createToolProfileService(db, now), attention = createToolAppAttention(db, now); + const assertCounts = async (expected: number) => { + const profiles = await service.listProfiles(f.company.id); + expect(profiles.find(p => p.id === f.profiles[0].id)?.newToolsPendingCount).toBe(expected); + expect(profiles.find(p => p.id === f.profiles[1].id)?.newToolsPendingCount).toBe(0); + expect((await service.listProfileNewTools(f.profiles[0].id, f.company.id)).pendingCount).toBe(expected); + expect((await attention(f.company.id)).totals.newToolsPendingReview).toBe(expected); + }; + await assertCounts(4); + const [exclude] = await db.insert(toolProfileEntries).values({ companyId: f.company.id, profileId: f.profiles[0].id, + selectorType: "tool_name", effect: "exclude", toolName: f.catalog[4].toolName }).returning(); + await assertCounts(3); + await db.update(toolCatalogEntries).set({ status: "removed" }).where(eq(toolCatalogEntries.id, f.catalog[5].id)); + await assertCounts(2); + await db.update(toolProfiles).set({ newToolsReviewedAt: now() }).where(eq(toolProfiles.id, f.profiles[0].id)); + await assertCounts(0); + const foreign = await createProfileProjectionFixture(db, { profiles: 2, tools: 5, rules: 2 }); + expect((await service.listProfiles(foreign.company.id)).map(p => p.companyId)).toEqual([foreign.company.id, foreign.company.id]); + expect((await db.select().from(toolProfileEntries).where(eq(toolProfileEntries.id, exclude.id)))[0].effect).toBe("exclude"); + }); +}); diff --git a/server/src/__tests__/voice-call-reports.test.ts b/server/src/__tests__/voice-call-reports.test.ts index fa614a5ed8a..0714158493c 100644 --- a/server/src/__tests__/voice-call-reports.test.ts +++ b/server/src/__tests__/voice-call-reports.test.ts @@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu await quiet(); expect(fetchReport).not.toHaveBeenCalled(); await vi.advanceTimersByTimeAsync(deadline - Date.now()); await vi.waitFor(async () => { - await vi.advanceTimersByTimeAsync(1000); + // Keep time fixed during database I/O. Advancing it on every poll can + // make the first reports due again before the backlog has settled. + const next = coordinator!.nextWakeAt(); + if (next !== null && next < deadline + 60_000) { + await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now())); + } expect(idleWorkSnapshot().active).toBe(0); expect(fetchReport).toHaveBeenCalledTimes(105); - }, { timeout: 5000 }); + }, { interval: 0, timeout: 15_000 }); expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort()); expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId))) .every(row => row.attempts === 1)).toBe(true); await quiet(); } finally { reads.mockRestore(); } - }); + }, 30_000); it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => { await session({ state: "active", endedAt: null }); await session({ endedAt: new Date(Date.now() - day - 1000) }); diff --git a/server/src/__tests__/voice-inbound-retention.test.ts b/server/src/__tests__/voice-inbound-retention.test.ts index 6bdedd256d5..6925c795d39 100644 --- a/server/src/__tests__/voice-inbound-retention.test.ts +++ b/server/src/__tests__/voice-inbound-retention.test.ts @@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet(); }); it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => { + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false }); const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000; await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`); const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id }); diff --git a/server/src/app.ts b/server/src/app.ts index 06adc15edb8..17b4e63e127 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -12,6 +12,7 @@ import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js"; import { createLifecycleDriver } from "./services/agent-lifecycle-driver.js"; import { startAgentLifecycle } from "./services/agent-lifecycle.js"; import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js"; +import { databaseOperationMiddleware } from "./middleware/database-operation.js"; import { staticUiFiles } from "./middleware/static-ui.js"; import { retainedUiAssets } from "./retained-ui-assets.js"; import { createStorageProviderFromConfig } from "./storage/provider-registry.js"; @@ -470,6 +471,7 @@ export async function createApp( }, ) { const app = express(); + app.use(databaseOperationMiddleware); app.use(idleAdmissionMiddleware); const staticUi = express.Router(); app.locals.paperclipDb = db; diff --git a/server/src/index.ts b/server/src/index.ts index ffc3b2d9009..ce02001ecf9 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -81,6 +81,7 @@ import { deliverExecutionStatuses } from "./services/execution-status-delivery.j import { deliverReconciledExecutions, settleUnrecoverableExecutions } from "./services/execution-recovery-resolution.js"; import { reconcileSafeNativeReplacements } from "./services/native-runtime/native-safe-replacement.js"; import { reconcileAbandonedExecutionControl } from "./services/execution-control-reconciliation.js"; +import { acknowledgeNativeStartupCleanup } from "./services/native-runtime/native-startup-cleanup.js"; import { executionControlSchedule, registerExecutionControlWork } from "./services/execution-control-work.js"; import { connectionIntentDeliveryService } from "./services/connection-intent-delivery.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; @@ -1376,7 +1377,12 @@ async function startServerWithDatabaseTeardown( const unsubscribeLocalLoginCleanup = localLoginCleanup.subscribeCleanupReady(localLoginCleanupWork.wake); server.on("close", unsubscribeLocalLoginCleanup); registerExecutionControlWork(deliveryWork, [ - { queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(db), next: executionSchedule.finalization }, + { queue: DELIVERY_QUEUES.executionFinalization, run: () => reconcileAbandonedExecutionControl(db, new Date(), { + startupCleanup: async scope => { + await accountingHeartbeat.cancelRun(scope.runId, "Native startup timed out; cancelling and verifying cleanup"); + await acknowledgeNativeStartupCleanup(db, scope); + }, + }), next: executionSchedule.finalization }, { queue: DELIVERY_QUEUES.executionStatus, run: () => deliverExecutionStatuses(db), next: executionSchedule.status }, { queue: DELIVERY_QUEUES.executionDisposition, run: () => settleUnrecoverableExecutions(db), next: executionSchedule.disposition }, ...(heartbeat ? [ diff --git a/server/src/middleware/database-operation.ts b/server/src/middleware/database-operation.ts new file mode 100644 index 00000000000..b7cbc7abe01 --- /dev/null +++ b/server/src/middleware/database-operation.ts @@ -0,0 +1,5 @@ +import { withDatabaseOperation } from "@paperclipai/db"; +import type { RequestHandler } from "express"; + +export const databaseOperationMiddleware: RequestHandler = (_req, _res, next) => + withDatabaseOperation("http.request", next); diff --git a/server/src/modules/wake-queue/adapters/release-run-read.test.ts b/server/src/modules/wake-queue/adapters/release-run-read.test.ts index 6a669edf617..4ad78f2b1e0 100644 --- a/server/src/modules/wake-queue/adapters/release-run-read.test.ts +++ b/server/src/modules/wake-queue/adapters/release-run-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; @@ -40,7 +40,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => { const scope = { companyId, runId: before.id }; const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const native = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, scope)); @@ -51,7 +51,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => { expect(native.run).not.toHaveProperty("stdoutExcerpt"); expect(native.run).not.toHaveProperty("stderrExcerpt"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(8192); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before); @@ -97,12 +97,12 @@ describe.skipIf(!support.supported)("wake release run reads", () => { contextSnapshot: { ...context, ...launch }, resultJson: evidence }).returning(); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let selected: Awaited>; try { selected = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, { companyId, runId: before.id })); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(8192); expect(selected!.run?.contextSnapshot).toEqual(context); diff --git a/server/src/redaction.ts b/server/src/redaction.ts index 28693d13fa0..dd3edf95680 100644 --- a/server/src/redaction.ts +++ b/server/src/redaction.ts @@ -2,6 +2,7 @@ import { redactCommandText } from "@paperclipai/adapter-utils"; import { isPublicExecutorToolSelector, looksLikeCredentialJwt } from "@paperclipai/adapter-utils/command-redaction"; const SECRET_FIELD_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring|browser[-_]?code|login[-_]?url)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`; +const SECRET_FIELD_NAME_START = String.raw`(? { + const companyId = req.params.companyId as string; + assertBoardOrAgent(req); + assertCompanyAccess(req, companyId); + const connectionId = z.string().uuid().safeParse(req.params.connectionId); + const grantId = z.string().uuid().optional().safeParse(req.query.grantId); + if (!connectionId.success || !grantId.success) throw unprocessable("Invalid connection or grant ID"); + res.setHeader("Cache-Control", "no-store"); + const actor = getActorInfo(req); + res.json(await service.accountIdentity(companyId, responsibleUserForAiRequest(req)!, connectionId.data, grantId.data, { + actorType: actor.actorType, actorId: actor.actorId, heartbeatRunId: actor.runId, + })); + }, + ); router.get( "/companies/:companyId/ai-connections/:connectionId/active-runs", managementGuard("connections:read", "GET /companies/:companyId/ai-connections/:connectionId/active-runs", { personal: true }), async (req, res) => { diff --git a/server/src/routes/issues.ts b/server/src/routes/issues.ts index 01bfe9f81d8..344c923080f 100644 --- a/server/src/routes/issues.ts +++ b/server/src/routes/issues.ts @@ -8380,6 +8380,7 @@ export function issueRoutes( title: row.title, identifier: row.identifier, status: row.status as IssueSummary["status"], + displayStatus: row.displayStatus, updatedAt: row.updatedAt, externalConversationState: row.externalConversationState, })); @@ -8549,10 +8550,10 @@ export function issueRoutes( res.status(400).json({ error: "offset must be a non-negative integer" }); return; } - if (sortField !== undefined && sortField !== "updated" && sortField !== "id") { + if (sortField !== undefined && sortField !== "updated" && sortField !== "created" && sortField !== "id") { res .status(400) - .json({ error: "sortField must be 'updated' or 'id' when provided" }); + .json({ error: "sortField must be 'updated', 'created', or 'id' when provided" }); return; } if (sortDir !== undefined && sortDir !== "asc" && sortDir !== "desc") { @@ -8620,6 +8621,7 @@ export function issueRoutes( readCondition: await issueReadSqlCondition(db, req.actor), attention: attention === "blocked" ? "blocked" : undefined, status: req.query.status as string | string[] | undefined, + displayStatus: req.query.displayStatus as string | string[] | undefined, assigneeAgentId, participantAgentId: uuidFilters.participantAgentId, assigneeUserId, @@ -8658,7 +8660,7 @@ export function issueRoutes( q: req.query.q as string | undefined, limit, offset, - sortField: sortField === "updated" || sortField === "id" ? sortField : undefined, + sortField: sortField === "updated" || sortField === "created" || sortField === "id" ? sortField : undefined, afterId: req.query.afterId as string | undefined, sortDir: sortDir === "asc" || sortDir === "desc" ? sortDir : undefined, updatedSince: rawUpdatedSince, @@ -8870,6 +8872,7 @@ export function issueRoutes( readCondition: await issueReadSqlCondition(db, req.actor), attention: "blocked", status: req.query.status as string | string[] | undefined, + displayStatus: req.query.displayStatus as string | string[] | undefined, assigneeAgentId: req.query.assigneeAgentId as string | undefined, participantAgentId: countUuidFilters.participantAgentId, assigneeUserId: req.query.assigneeUserId as string | undefined, diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index 6b7e02f026e..c766d6c4836 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -11311,6 +11311,17 @@ registerCurrentRoute({ responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound }, }); +registerCurrentRoute({ + "x-paperclip-transport": "rest", + "x-paperclip-connection-management": "connections:read", + method: "get", + path: "/api/companies/{companyId}/ai-connections/{connectionId}/identity", + tags: ["ai-connections"], + summary: "Read the selected authorized AI account's provider email and organization", + query: z.object({ grantId: z.string().uuid().optional() }), + responses: { 200: r.ok(), 401: r.unauthorized, 403: r.forbidden, 404: r.notFound, 422: r.unprocessable }, +}); + registerCurrentRoute({ "x-paperclip-transport": "rest", "x-paperclip-connection-management": "connections:read", diff --git a/server/src/routes/user-profiles.ts b/server/src/routes/user-profiles.ts index 24ed9f9e007..4dc8a9e0109 100644 --- a/server/src/routes/user-profiles.ts +++ b/server/src/routes/user-profiles.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "../services/issue-display-status.js"; import { Router } from "express"; import { and, desc, eq, gte, inArray, isNull, or, sql, type SQLWrapper } from "drizzle-orm"; import type { Db } from "@paperclipai/db"; @@ -347,6 +348,7 @@ export function userProfileRoutes(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index c1fe658aab6..b1c8de72e5d 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -2,7 +2,7 @@ import { notifyInstructionCollection } from "./instruction-collection-notificati import fs from "node:fs/promises"; import path from "node:path"; import { isDeepStrictEqual } from "node:util"; -import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm"; import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, withDedicatedDbConnection, type Db } from "@paperclipai/db"; import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh"; import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget, type PreparedAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target"; @@ -16,7 +16,7 @@ import type { AuthorizationActor } from "./authorization.js"; import type { EnvironmentRuntimeService } from "./environment-runtime.js"; import type { Environment, EnvironmentLease } from "@paperclipai/shared"; import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory } from "./agent-directory-probe.js"; -import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js"; +import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "./remote-execution-termination.js"; import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js"; import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js"; import { logger } from "../middleware/logger.js"; @@ -46,7 +46,7 @@ function baseline(row: Copy) { function actor(row: Copy): AuthorizationActor { return { type: "agent", companyId: row.companyId, agentId: row.agentId, runId: row.runId, onBehalfOfUserId: row.responsibleUserId }; } -export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise, patch: (row: Copy, values: Partial) => Promise, environmentRuntime?: EnvironmentRuntimeService) { +export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise, patch: (row: Copy, values: Partial, condition?: SQL) => Promise, environmentRuntime?: EnvironmentRuntimeService) { const store = agentFileStore(db); async function insertCopy(values: typeof copies.$inferInsert) { return db.transaction(async tx => { @@ -503,7 +503,15 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string return row.state === "superseded" || !row.processStoppedAt || completed.has(row.state) && row.receipt?.cleanupPending === false; } + async function settleCompletedCleanupRetry(row: Copy) { + if (completed.has(row.state) && row.processStoppedAt && !row.receipt?.retainedByRunId && row.receipt?.cleanupPending === false && row.nextAttemptAt) { + return patch(row, { nextAttemptAt: null }, sql`${copies.receipt} = ${JSON.stringify(row.receipt)}::jsonb + AND ${copies.processStoppedAt} IS NOT NULL`); + } + return row; + } async function release(row: Copy, target?: AdapterExecutionTarget | null) { + row = await settleCompletedCleanupRetry(row); if (releaseIsNoop(row) || !await owns(row)) return; const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true; if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return; @@ -559,7 +567,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string : eq(environmentLeases.environmentId, row.location.slice("remote:".length)), )); if (leases.length !== 1) return false; - const lease = leases[0]!; + const lease = await resolveRemoteTerminationLease(db, leases[0]!); if (lease.environmentId !== null && row.location !== `remote:${lease.environmentId}`) return false; const remoteCwd = cleanup?.remoteCwd ?? lease.metadata?.remoteCwd; return typeof remoteCwd === "string" && @@ -588,7 +596,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string cleanup?.leaseId ? eq(environmentLeases.id, cleanup.leaseId) : eq(environmentLeases.environmentId, row.location.slice("remote:".length)), )); if (leases.length !== 1) return null; - const lease = leases[0]!; + const lease = await resolveRemoteTerminationLease(db, leases[0]!); const destroyedWithoutEnvironment = lease.environmentId === null && hasRemoteTerminationReceipt(lease) && (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed"; if (row.location !== `remote:${lease.environmentId}` && !destroyedWithoutEnvironment) return null; @@ -658,10 +666,12 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string checkpointWarm: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => current.receipt?.warm === true ? checkpoint(current, target) : Promise.resolve(current), "agent_directory_checkpoint"), collectStopped: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => collectStopped(current, target), "agent_directory_collect"), release: async (row: Copy) => { - const current = await get(row.companyId, row.runId); + let current = await get(row.companyId, row.runId); + if (!current) return; + current = await settleCompletedCleanupRetry(current); // This check changes no receipt or bytes. A concurrent stop merely leaves // cleanup to its owner or the recovery sweep; it grants no new authority. - if (!current || releaseIsNoop(current)) return; + if (releaseIsNoop(current)) return; await serial(current, release, "agent_directory_release"); }, }; diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index 0d990331d4a..586bd63dbfb 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -129,8 +129,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen }; return db.transaction(tx => write(tx as unknown as Db)); } - async function patch(row: Copy, values: Partial) { - const updated = await tryPatch(row, values); + async function patch(row: Copy, values: Partial, condition?: SQL) { + const updated = await tryPatch(row, values, false, condition); // A late cleanup must not overwrite an explicit resolution or a newer // baseline. Canonical content has its own independent head CAS. return updated ?? (await get(row.companyId, row.runId))!; @@ -499,8 +499,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen function directoryCleanupEntry(row: Copy | null): DirectoryCleanupEntry | null { return row && ["saved", "unchanged", "resolved", "unavailable"].includes(row.state) && row.processStoppedAt && isAgentDirectoryCopy(row) && !row.receipt?.retainedByRunId - && row.receipt?.cleanupPending !== false - && (Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true) + && (row.receipt?.cleanupPending === false ? row.nextAttemptAt !== null + : Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true) ? { id: row.runId, companyId: row.companyId, dueAt: row.nextAttemptAt?.getTime() ?? 0 } : null; } function directoryCleanupPredicate() { @@ -508,8 +508,9 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen AND ${copies.processStoppedAt} IS NOT NULL AND ${copies.receipt}->>'schema' = 'paperclip.agent-files.v1' AND coalesce(${copies.receipt}->>'retainedByRunId', '') = '' - AND ${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false' - AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')`; + AND ((${copies.receipt}->>'cleanupPending' = 'false' AND ${copies.nextAttemptAt} IS NOT NULL) + OR (${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false' + AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')))`; } async function loadDirectoryCleanup(scope: DirectoryCleanupScope | undefined, admitted: () => boolean) { const entries: DirectoryCleanupEntry[] = []; diff --git a/server/src/services/ai-account-identity.test.ts b/server/src/services/ai-account-identity.test.ts new file mode 100644 index 00000000000..eb8e67edd2a --- /dev/null +++ b/server/src/services/ai-account-identity.test.ts @@ -0,0 +1,72 @@ +import { createServer } from "node:http"; +import { once } from "node:events"; +import { describe, expect, it, vi } from "vitest"; +import { probeAiAccountIdentity } from "./ai-account-identity.js"; + +const metadata = { provider: "anthropic", method: "subscription" } as const; +function fixture(body: unknown, status = 200) { + return vi.fn().mockResolvedValue(Response.json(body, { status })); +} + +describe("AI account identity", () => { + it("returns only safe account details from the selected bearer profile", async () => { + const request = fixture({ account: { uuid: "private-seat", email: "avery@example.test", secret: "provider-private" }, + organization: { uuid: "private-organization", name: "Example team" }, access_token: "private-token" }); + const result = await probeAiAccountIdentity(metadata, "selected-credential", request); + expect(result).toMatchObject({ status: "available", accountLabel: "avery@example.test", organizationLabel: "Example team" }); + expect(request).toHaveBeenCalledWith("https://api.anthropic.com/api/oauth/profile", expect.objectContaining({ + redirect: "error", signal: expect.any(AbortSignal), + headers: { Authorization: "Bearer selected-credential", "anthropic-beta": "oauth-2025-04-20" }, + })); + for (const secret of ["private-seat", "private-organization", "provider-private", "private-token", "selected-credential"]) { + expect(JSON.stringify(result)).not.toContain(secret); + } + }); + + it.each([401, 403, 429, 500])("reports HTTP %s without exposing the provider response", async status => { + const result = await probeAiAccountIdentity(metadata, "selected-credential", fixture({ error: "private-provider-error" }, status)); + expect(result).toMatchObject({ accountLabel: null, organizationLabel: null }); + expect(result.status).toBe(status === 401 || status === 403 ? "unavailable" : "error"); + if (status === 403) expect(result).toMatchObject({ errorCode: "permission_denied", message: expect.stringContaining("did not permit") }); + expect(JSON.stringify(result)).not.toContain("private-provider-error"); + }); + + it.each([{}, { account: { email: "not-an-email" } }, { account: { email: "avery@example.test\nprivate" } }])("does not infer an email from an incomplete profile %j", async profile => { + expect(await probeAiAccountIdentity(metadata, "selected-credential", fixture(profile))).toMatchObject({ + status: "unavailable", errorCode: "invalid_response", accountLabel: null, + }); + }); + + it("does not call a provider for API keys or other sign-in methods", async () => { + const request = fixture({}); + expect((await probeAiAccountIdentity({ provider: "anthropic", method: "api_key" }, "", request)).errorCode).toBe("unsupported"); + expect((await probeAiAccountIdentity({ provider: "openai", method: "subscription" }, "", request)).errorCode).toBe("unsupported"); + expect(request).not.toHaveBeenCalled(); + }); + + it("aborts a stalled real response body while unrelated requests remain responsive", async () => { + const server = createServer((req, res) => { + if (req.url === "/profile") { res.writeHead(200, { "Content-Type": "application/json" }); res.flushHeaders(); } + else res.end("responsive"); + }); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + const origin = `http://127.0.0.1:${address.port}`; + const timeout = AbortSignal.timeout.bind(AbortSignal); + const deadline = vi.spyOn(AbortSignal, "timeout").mockImplementation(() => timeout(100)); + try { + const pending = probeAiAccountIdentity(metadata, "selected-credential", (_url, init) => fetch(`${origin}/profile`, init)); + expect(await (await fetch(`${origin}/unrelated`)).text()).toBe("responsive"); + const result = await pending; + expect(result.status).toBe("error"); + expect(result.accountLabel).toBeNull(); + expect(deadline).toHaveBeenCalledWith(15_000); + } finally { + deadline.mockRestore(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + }); +}); diff --git a/server/src/services/ai-account-identity.ts b/server/src/services/ai-account-identity.ts new file mode 100644 index 00000000000..b93fb6da02c --- /dev/null +++ b/server/src/services/ai-account-identity.ts @@ -0,0 +1,43 @@ +import type { AiConnectionAccountIdentity, AiConnectionMetadata } from "@paperclipai/shared"; +import { readUsage } from "./ai-connection-usage.js"; + +type Observation = Omit; +type FailureCode = NonNullable; +const messages: Record = { + unsupported: "Account details are not available for this sign-in method.", + connection_unavailable: "Reconnect or enable this connection to check its account details.", + authentication_required: "Reconnect this account to check its account details.", + permission_denied: "Anthropic did not permit this connection to read the account email.", + rate_limited: "Anthropic limited this account check. Try again later.", + provider_unavailable: "Could not read the account details. Try again later.", + invalid_response: "Anthropic did not return an account email.", +}; +const object = (value: unknown): Record => value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +const label = (value: unknown): string | null => typeof value === "string" && value.trim().length > 0 + && value.length <= 254 && !/[\u0000-\u001f\u007f]/.test(value) ? value.trim() : null; + +/** Never uses ambient CLI identity or exposes provider IDs, tokens, or raw responses. */ +export async function probeAiAccountIdentity( + metadata: AiConnectionMetadata, + credential: string, + request: typeof fetch = fetch, +): Promise { + const base = { checkedAt: new Date().toISOString(), accountLabel: null, organizationLabel: null }; + if (metadata.provider !== "anthropic" || metadata.method !== "subscription" || metadata.routing) { + return { ...base, status: "unavailable", errorCode: "unsupported", message: messages.unsupported }; + } + try { + const profile = await readUsage("https://api.anthropic.com/api/oauth/profile", { + Authorization: `Bearer ${credential}`, "anthropic-beta": "oauth-2025-04-20", + }, request); + const accountLabel = label(object(profile.account).email); + if (!accountLabel || !/^[^\s@/\\]+@[^\s@/\\]+\.[^\s@/\\]+$/.test(accountLabel)) { + return { ...base, status: "unavailable", errorCode: "invalid_response", message: messages.invalid_response }; + } + return { ...base, status: "available", accountLabel, organizationLabel: label(object(profile.organization).name) }; + } catch (error) { + const code = object(error).code; + const errorCode: FailureCode = typeof code === "string" && Object.hasOwn(messages, code) ? code as FailureCode : "provider_unavailable"; + return { ...base, status: errorCode === "authentication_required" || errorCode === "permission_denied" ? "unavailable" : "error", errorCode, message: messages[errorCode] }; + } +} diff --git a/server/src/services/ai-connections.ts b/server/src/services/ai-connections.ts index d76f9606c9e..6314a5265d8 100644 --- a/server/src/services/ai-connections.ts +++ b/server/src/services/ai-connections.ts @@ -36,6 +36,7 @@ import { type AiConnectionAttribution, type AiConnectionMetadata, type AiManagedConnectionSummary, + type AiConnectionAccountIdentity, type AiConnectionUsage, type CreateAiConnection, type AiConnectionLoginIntent, @@ -49,6 +50,7 @@ import { logger } from "../middleware/logger.js"; import { logActivity } from "./activity-log.js"; import { secretService } from "./secrets.js"; import { probeAiConnectionUsage } from "./ai-connection-usage.js"; +import { probeAiAccountIdentity } from "./ai-account-identity.js"; /** Same human audience displayed by the existing Connections identity controls. */ function canUseCredential( @@ -225,28 +227,46 @@ export function aiConnectionService(db: Db) { }); }); } - /** On-demand account observation for either runner; never probes ambient CLI auth. */ - async function probeUsage(companyId: string, userId: string, connectionId: string, grantId?: string): Promise { + async function readableAccount(companyId: string, userId: string, connectionId: string, grantId?: string) { if (!(await membership(companyId, userId))) throw forbidden("An active company member is required"); const candidates = (await rows(companyId)).filter((row) => row.connection.id === connectionId && (!grantId || row.grant.id === grantId)); const audience = await db.select().from(connectionGrantMembers).where(eq(connectionGrantMembers.companyId, companyId)); const visible = candidates.filter(({ grant }) => canUseCredential(grant, userId, audience.filter((member) => member.grantId === grant.id))); if (!visible.length) throw notFound("AI connection not found"); - if (visible.length > 1) throw unprocessable("Choose a credential grant for this usage check"); + if (visible.length > 1) throw unprocessable("Choose a credential grant for this account check"); const row = visible[0]!; const metadata = aiConnectionMetadataSchema.safeParse(row.connection.config.ai); if (!metadata.success) throw notFound("AI connection not found"); - const identity = { connectionId: row.connection.id, grantId: row.grant.id, ...metadata.data }; + return { row, metadata: metadata.data }; + } + /** On-demand account observation for either runner; never probes ambient CLI auth. */ + async function probeUsage(companyId: string, userId: string, connectionId: string, grantId?: string): Promise { + const { row, metadata } = await readableAccount(companyId, userId, connectionId, grantId); + const identity = { connectionId: row.connection.id, grantId: row.grant.id, ...metadata }; const empty = { ...identity, checkedAt: new Date().toISOString(), source: null, planType: null, limits: [], overage: null }; if (row.grant.status !== "active" || !row.connection.enabled || row.connection.status !== "active" || row.connection.healthStatus !== "ok" || aiSubscriptionNeedsIsolatedLogin(row.connection.config)) { return { ...empty, status: "unavailable", errorCode: "connection_unavailable", message: "Reconnect or enable this AI connection before checking usage." }; } - if (!supportsAiConnectionUsage(metadata.data.provider, metadata.data.method)) { - return { ...identity, ...await probeAiConnectionUsage(metadata.data, "") }; + if (!supportsAiConnectionUsage(metadata.provider, metadata.method)) { + return { ...identity, ...await probeAiConnectionUsage(metadata, "") }; } const value = await credential(row); - return { ...identity, ...await probeAiConnectionUsage(metadata.data, value) }; + return { ...identity, ...await probeAiConnectionUsage(metadata, value) }; + } + async function accountIdentity(companyId: string, userId: string, connectionId: string, grantId?: string, + audit?: { actorType: "user" | "agent" | "system"; actorId: string; heartbeatRunId?: string | null }): Promise { + const { row, metadata } = await readableAccount(companyId, userId, connectionId, grantId); + const identity = { connectionId: row.connection.id, grantId: row.grant.id }; + if (row.grant.status !== "active" || !row.connection.enabled || row.connection.status !== "active" || + row.connection.healthStatus !== "ok" || aiSubscriptionNeedsIsolatedLogin(row.connection.config)) { + return { ...identity, status: "unavailable", checkedAt: new Date().toISOString(), accountLabel: null, organizationLabel: null, + errorCode: "connection_unavailable", message: "Reconnect or enable this connection to check its account details." }; + } + // Unsupported methods must not open their credentials just to report this limit. + const value = metadata.provider === "anthropic" && metadata.method === "subscription" && !metadata.routing + ? await credential(row, 0, { actorType: "user", actorId: userId, ...audit, responsibleUserId: userId }) : ""; + return { ...identity, ...await probeAiAccountIdentity(metadata, value) }; } async function select(input: { companyId: string; @@ -1218,5 +1238,5 @@ export function aiConnectionService(db: Db) { return [{ ...row, summary }]; }); } - return { list, selectDecision, selectFastResponse, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, save, setDefault, membership, markAuthenticationFailed }; + return { list, selectDecision, selectFastResponse, selectPromptInjectionPrevention, quotaAccounts, subscriptionAccounts: (companyId: string, userId: string) => quotaAccounts(companyId, userId, true), refreshQuotaCredential, select, credential, runtimeCredential, probeUsage, accountIdentity, save, setDefault, membership, markAuthenticationFailed }; } diff --git a/server/src/services/budget-tracing.ts b/server/src/services/budget-tracing.ts index c0f525a61b1..f4b9212ca8f 100644 --- a/server/src/services/budget-tracing.ts +++ b/server/src/services/budget-tracing.ts @@ -1,18 +1,27 @@ import { trace, type Span } from "@opentelemetry/api"; +import { withDatabaseOperation } from "@paperclipai/db"; + +type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover" + | "budget.accounting.settlement" | "budget.accounting.transaction" + | "budget.admission.evaluate" | "budget.admission.reserve" + | "budget.cancellation.deliver" | "budget.enforcement.deliver" + | "budget.enforcement.scope" | "budget.exposure.evaluate"; /** Operator-gated OpenTelemetry only; these spans contain no prompt, receipt, * credential, or provider error text. Diagnostics never change accounting. */ -export async function withBudgetSpan(name: string, work: () => Promise, attributes?: Record) { - if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work(); - let operation: Promise | undefined; - const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => { - try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ } +export function withBudgetSpan(name: BudgetOperation, work: () => Promise, attributes?: Record) { + return withDatabaseOperation(name, async () => { + if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work(); + let operation: Promise | undefined; + const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => { + try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ } + }); + try { + const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span)); + // Some external SDK implementations can fail before or after the callback. + // Own the accounting promise so either failure still executes work once. + void Promise.resolve(traced).catch(() => undefined); + } catch { /* Preserve the accounting operation if span creation fails. */ } + return operation ?? run(); }); - try { - const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span)); - // Some external SDK implementations can fail before or after the callback. - // Own the accounting promise so either failure still executes work once. - void Promise.resolve(traced).catch(() => undefined); - } catch { /* Preserve the accounting operation if span creation fails. */ } - return operation ?? run(); } diff --git a/server/src/services/chat-channels/authorization-run-read.test.ts b/server/src/services/chat-channels/authorization-run-read.test.ts index 3d56b695222..5e573a5974e 100644 --- a/server/src/services/chat-channels/authorization-run-read.test.ts +++ b/server/src/services/chat-channels/authorization-run-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -49,7 +49,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => { const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId)); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const selected = await db.transaction(tx => readChatAuthorizationRun(tx, scope, true)); @@ -58,7 +58,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => { expect(selected.run).not.toHaveProperty("runnerProfileJson"); expect(selected.run).not.toHaveProperty("stdoutExcerpt"); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBeGreaterThan(0); expect(summary.responseValueBytes).toBeLessThan(4096); const retry = await db.transaction(tx => readChatAuthorizationRun(tx, scope, false)); diff --git a/server/src/services/chat-channels/github/webhook-ingress.test.ts b/server/src/services/chat-channels/github/webhook-ingress.test.ts index ab868d63c5c..975f0b810de 100644 --- a/server/src/services/chat-channels/github/webhook-ingress.test.ts +++ b/server/src/services/chat-channels/github/webhook-ingress.test.ts @@ -2,7 +2,7 @@ import { createHmac } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { applyPendingMigrations, chatActions, createDb } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js"; import { cleanupIngressFixture, githubIngressFixture as createFixture } from "../../../__tests__/helpers/github-webhook-ingress.js"; @@ -36,7 +36,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh const f = await githubIngressFixture(db, createGitHubWebhookIngress, 768 * 1024); const action = await f.insertAction(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions }, capture); try { expect((await f.service.processGitHubWebhookIngress(action.id))?.status).toBe(202); } finally { capture.flush(); } @@ -59,7 +59,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh // A corrupt body must not let a nonclaimant overwrite a live or delayed claim. const action = await f.insertAction({ status, result, payload: { ...f.payload, bodySha256: "0".repeat(64) } }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatActions }, capture); let response: Response | null; try { response = await f.service.processGitHubWebhookIngress(action.id); } @@ -67,7 +67,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh expect(response?.status ?? null).toBe(accepted ? 202 : null); expect(await f.readAction(action.id)).toEqual(action); expect(f.calls).toEqual([]); - expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024); + expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024); }); it.each(["received", "failed", "processing"])("settles eligible malformed %s payloads without replay", async status => { diff --git a/server/src/services/company-search.ts b/server/src/services/company-search.ts index fb27743e0d2..17055aa3146 100644 --- a/server/src/services/company-search.ts +++ b/server/src/services/company-search.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { externalConversationStateSql } from "./slack-conversation-state.js"; import { and, desc, eq, gte, inArray, isNotNull, isNull, notInArray, or, sql } from "drizzle-orm"; import type { SQL } from "drizzle-orm"; @@ -21,6 +22,7 @@ import { COMPANY_ARTIFACTS_MAX_QUERY_LENGTH, ISSUE_PRIORITIES, ISSUE_STATUSES, + ISSUE_DISPLAY_STATUSES, SYSTEM_ISSUE_DOCUMENT_KEYS, type CompanyArtifact, type CompanySearchArtifactSummary, @@ -50,6 +52,7 @@ type IssueSearchRow = { title: string; description: string | null; status: string; + displayStatus: string; priority: string; assigneeAgentId: string | null; assigneeUserId: string | null; @@ -193,6 +196,7 @@ function emptySearchCounts(): Record { function emptyFilterOptionCounts(): CompanySearchFilterOptionCounts { return { status: {}, + displayStatus: {}, priority: {}, assigneeAgentId: {}, assigneeUserId: {}, @@ -224,7 +228,7 @@ function updatedWithinStart(value: string | undefined, now = new Date()): Date | } function issueOnlyFiltersActive(query: CompanySearchQuery) { - return query.status.length > 0 + return (query.status.length > 0 || query.displayStatus.length > 0) || query.priority.length > 0 || query.assigneeAgentId !== undefined || Boolean(query.assigneeUserId) @@ -236,7 +240,7 @@ function issueOnlyFiltersActive(query: CompanySearchQuery) { function activeIssueFilters(query: CompanySearchQuery): Array<{ key: CompanySearchIssueFilterKey; values: string[] }> { const filters: Array<{ key: CompanySearchIssueFilterKey; values: string[] }> = []; - if (query.status.length > 0) filters.push({ key: "status", values: query.status }); + if (query.status.length > 0 || query.displayStatus.length > 0) filters.push({ key: "status", values: [...query.status, ...query.displayStatus] }); if (query.assigneeAgentId !== undefined) filters.push({ key: "assigneeAgentId", values: [query.assigneeAgentId ?? "null"] }); if (query.assigneeUserId) filters.push({ key: "assigneeUserId", values: [query.assigneeUserId] }); if (query.projectId) filters.push({ key: "projectId", values: [query.projectId] }); @@ -251,6 +255,7 @@ function queryWithoutFilter(query: CompanySearchQuery, key: CompanySearchIssueFi return { ...query, status: key === "status" ? [] : query.status, + displayStatus: key === "status" ? [] : query.displayStatus, priority: key === "priority" ? [] : query.priority, assigneeAgentId: key === "assigneeAgentId" ? undefined : query.assigneeAgentId, assigneeUserId: key === "assigneeUserId" ? undefined : query.assigneeUserId, @@ -265,6 +270,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery return { ...query, status: [], + displayStatus: [], priority: [], assigneeAgentId: undefined, assigneeUserId: undefined, @@ -277,6 +283,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery function issueFilterConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] { const conditions: SQL[] = []; + if (omit !== "status" && query.displayStatus.length) conditions.push(inArray(issueDisplayStatusSql(), query.displayStatus)); if (omit !== "status" && query.status.length > 0) { conditions.push(query.status.length === 1 ? eq(issues.status, query.status[0]!) : inArray(issues.status, query.status)); } @@ -315,6 +322,7 @@ function issueFilterConditions(companyId: string, query: CompanySearchQuery, omi // matched-issues CTE (plain columns, no drizzle table references). function matchedFacetConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] { const conditions: SQL[] = []; + if (omit !== "status" && query.displayStatus.length) conditions.push(sql`m.display_status = ANY(${sqlTextArray(query.displayStatus)})`); if (omit !== "status" && query.status.length > 0) { conditions.push(sql`m.status = ANY(${sqlTextArray(query.status)})`); } @@ -447,6 +455,7 @@ function issueResult(row: IssueSearchRow, prefix: string, normalizedQuery: strin identifier: row.identifier, title: row.title, status: row.status as CompanySearchIssueSummary["status"], + displayStatus: row.displayStatus as CompanySearchIssueSummary["displayStatus"], priority: row.priority as CompanySearchIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -626,7 +635,7 @@ export function companySearchService(db: Db) { conditions.length > 0 ? sql`WHERE ${sql.join(conditions, sql` AND `)}` : sql``; // Count branches must match the result branch's column list; the // trailing NULLs pad the issue data columns. - const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`; + const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`; const branches: SQL[] = []; const wantResultRows = scopeIncludesIssues(scope) @@ -652,6 +661,7 @@ export function companySearchService(db: Db) { (SELECT issue_text.description FROM issues issue_text WHERE issue_text.id = m.id AND issue_text.company_id = ${companyId}) AS description, m.status, + m.display_status AS "displayStatus", m.priority, m.assignee_agent_id AS "assigneeAgentId", m.assignee_user_id AS "assigneeUserId", @@ -690,6 +700,7 @@ export function companySearchService(db: Db) { GROUP BY 2 `; branches.push(facetBranch("facet:status", sql`m.status`, "status")); + branches.push(facetBranch("facet:displayStatus", sql`m.display_status`, "status")); branches.push(facetBranch("facet:priority", sql`m.priority`, "priority")); branches.push(facetBranch("facet:assigneeAgentId", sql`m.assignee_agent_id`, "assigneeAgentId", [sql`m.assignee_agent_id IS NOT NULL`])); branches.push(facetBranch("facet:assigneeUserId", sql`m.assignee_user_id`, "assigneeUserId", [sql`m.assignee_user_id IS NOT NULL`])); @@ -757,6 +768,7 @@ export function companySearchService(db: Db) { title: row.title, description: row.description, status: row.status, + displayStatus: row.displayStatus, priority: row.priority, assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -780,6 +792,9 @@ export function companySearchService(db: Db) { if (row.kind === "type:issue") aggregates.typeCounts.issue = count; else if (row.kind === "type:comment") aggregates.typeCounts.comment = count; else if (row.kind === "type:document") aggregates.typeCounts.document = count; + else if (row.kind === "facet:displayStatus" && row.value && (ISSUE_DISPLAY_STATUSES as readonly string[]).includes(row.value)) { + aggregates.filterOptionCounts.displayStatus![row.value as keyof NonNullable] = count; + } else if (row.kind === "facet:status" && row.value && (ISSUE_STATUSES as readonly string[]).includes(row.value)) { aggregates.filterOptionCounts.status[row.value as keyof CompanySearchFilterOptionCounts["status"]] = count; } else if (row.kind === "facet:priority" && row.value && (ISSUE_PRIORITIES as readonly string[]).includes(row.value)) { diff --git a/server/src/services/connection-management.ts b/server/src/services/connection-management.ts index cf65d2afb34..39a0c67488f 100644 --- a/server/src/services/connection-management.ts +++ b/server/src/services/connection-management.ts @@ -32,7 +32,7 @@ type PreparedAuthorization = { expandedTargets: Set; referencedConnections: string[]; currentGatewayResource: ConnectionManagementResource | undefined; - savedNonAiConnection: boolean; + savedConnection: boolean; }; type ConnectionCapability = { allowed: boolean; reason: string; operation: string }; @@ -88,7 +88,7 @@ export function connectionManagementService(db: Db) { const expandedTargets = new Set(options.targetAgentIds ?? []); const referencedConnections: string[] = []; let currentGatewayResource: ConnectionManagementResource | undefined; - let savedNonAiConnection = false; + let savedConnection = false; if (resource.gatewayId && !options.gatewayContextChecked) { const [gateway] = await db.select().from(toolMcpGateways).where(and(eq(toolMcpGateways.id, resource.gatewayId), eq(toolMcpGateways.companyId, resource.companyId))); if (!gateway) throw forbidden("Gateway is unavailable in this company"); @@ -159,7 +159,7 @@ export function connectionManagementService(db: Db) { if (resource.connectionId) { const [connection] = await db.select().from(toolConnections).where(and(eq(toolConnections.id, resource.connectionId), eq(toolConnections.companyId, resource.companyId))); if (!connection) throw forbidden("Connection is unavailable in this company"); - savedNonAiConnection = connection.connectionPurpose !== "ai"; + savedConnection = true; resource = { ...resource, selfServiceUserId: connection.createdByUserId, purpose: connection.connectionPurpose === "ai" ? "ai" : connection.connectionPurpose === "channel" ? "chat" : "tool", provider: typeof connection.config.sourceTemplateKey === "string" ? connection.config.sourceTemplateKey @@ -241,7 +241,7 @@ export function connectionManagementService(db: Db) { } } return { actor, action, resource, options, responsibleUserId, workContext, capturedRun, - expandedTargets, referencedConnections, currentGatewayResource, savedNonAiConnection }; + expandedTargets, referencedConnections, currentGatewayResource, savedConnection }; } async function evaluateTarget(prepared: PreparedAuthorization, targetAgentId: string | undefined): Promise { @@ -388,7 +388,9 @@ export function connectionManagementService(db: Db) { } const resolved = prepared.resource; const isCanonicalAgentId = (id: string) => isUuidLike(id) && id === id.toLowerCase(); - const eligible = actor.type === "board" && prepared.savedNonAiConnection + // Direct AI accounts use the same target checks. Pools with members still + // use the full evaluator through referencedConnections below. + const eligible = actor.type === "board" && prepared.savedConnection && [resolved.profileId, resolved.gatewayId, resolved.routineId, resolved.projectId, resolved.issueId].every(id => id == null) && !Object.hasOwn(resolved, "requestedProvider") && !Object.hasOwn(resolved, "requestedInstallCompanyIds") && !resolved.selfServiceAgentInstalls diff --git a/server/src/services/delivery-work-coordinator.ts b/server/src/services/delivery-work-coordinator.ts index 3e1f1b95b2b..42638c33e34 100644 --- a/server/src/services/delivery-work-coordinator.ts +++ b/server/src/services/delivery-work-coordinator.ts @@ -1,4 +1,5 @@ import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db/work-signals"; +import { withDatabaseOperation } from "@paperclipai/db"; import { createWorkScheduler } from "./work-scheduler.js"; import { type DeliveryQueue } from "./delivery-work-notifications.js"; import { beginIdleTrackedWork } from "./task-admission.js"; @@ -70,7 +71,7 @@ export function createDeliveryWorkCoordinator(input: { const attempt = new AbortController(); let runFailed = false; controller = attempt; - running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) + running = withDatabaseOperation(`delivery.${queue}`, () => Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue)) .then(() => workerStopped || !(input.canRun() || (task.runDuringDrain && input.canReconcile?.())) ? undefined : task.run(attempt.signal).catch(error => { runFailed = true; throw error; })).then(() => workerStopped ? false : task.hasPending()).then(pending => { pendingWork = pending || databaseWorkPending(input.owner, queue); if (pendingWork) schedule(task.retryMs); @@ -98,7 +99,7 @@ export function createDeliveryWorkCoordinator(input: { controller = null; finish(); if (dirty) schedule(0); - }); + })); } function wake() { if (workerStopped) return; diff --git a/server/src/services/durable-chat-wakeup.ts b/server/src/services/durable-chat-wakeup.ts index 2870eac88ba..367386ffaca 100644 --- a/server/src/services/durable-chat-wakeup.ts +++ b/server/src/services/durable-chat-wakeup.ts @@ -1,6 +1,6 @@ import { createHash } from "node:crypto"; -import { agentWakeupRequests, chatActions, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db"; -import { and, eq, inArray, or, sql, type SQLWrapper } from "drizzle-orm"; +import { agentWakeupRequests, chatActions, chatConversations, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db"; +import { and, eq, exists, inArray, or, sql, type SQLWrapper } from "drizzle-orm"; import { HttpError } from "../errors.js"; export interface CommittedChatResponseAuthorizationInput { @@ -233,6 +233,22 @@ export interface DeferredChatWakeScope { issueId: string; requestId: string; } + +/** Identify saved input by its canonical receipt, never by a caller's source + * label. This is selection only; the inbound owner rechecks live authority. */ +export function deferredChatInputPredicate(db: Db) { + return exists(db.select({ id: chatActions.id }).from(chatActions) + .innerJoin(chatConversations, and(eq(chatConversations.companyId, chatActions.companyId), + eq(chatConversations.id, chatActions.conversationId))) + .innerJoin(chatEndpoints, and(eq(chatEndpoints.companyId, chatActions.companyId), + eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.id, chatConversations.endpointId))) + .where(and(eq(chatActions.companyId, agentWakeupRequests.companyId), eq(chatActions.id, agentWakeupRequests.id), + eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"), + eq(chatEndpoints.assignedAgentId, agentWakeupRequests.agentId), eq(chatEndpoints.externalExecutionPolicy, "restricted"), + sql`${chatConversations.issueId}::text = ${agentWakeupRequests.payload}->>'issueId'`, + sql`${chatActions.payload}->>'issueId' = ${agentWakeupRequests.payload}->>'issueId'`, + sql`${chatActions.payload}->>'agentId' = ${agentWakeupRequests.agentId}::text`))); +} const deferredChatWakeOwners = new WeakMap Promise>(); /** The live inbound owner reauthorizes the original source. Cleanup owns no chat authority. */ diff --git a/server/src/services/environment-cleanup-state.ts b/server/src/services/environment-cleanup-state.ts index eae1d13fb4e..e9d1fa6bf5f 100644 --- a/server/src/services/environment-cleanup-state.ts +++ b/server/src/services/environment-cleanup-state.ts @@ -1,9 +1,19 @@ -import { eq, sql } from "drizzle-orm"; +import { and, eq, or, sql } from "drizzle-orm"; import { environmentLeases, sandboxCreationOperations, type Db } from "@paperclipai/db"; import { hasDeferredEnvironmentCleanup } from "./environment-cleanup-notifications.js"; export const ENVIRONMENT_CLEANUP_RETRY_MS = 5 * 60_000; +export function recoverableSandboxCreationCondition() { + return or(eq(sandboxCreationOperations.state, "pending"), and( + eq(sandboxCreationOperations.state, "needs_reconciliation"), + sql`coalesce(${sandboxCreationOperations.metadata}->>'creationReconciliationReason', '') <> 'company_deleted'`, + sql`${sandboxCreationOperations.metadata}->>'runnerTaskLease' = 'true'`, + sql`jsonb_typeof(${sandboxCreationOperations.metadata}->'runnerTaskCreationContextId') = 'string' and ${sandboxCreationOperations.metadata}->>'runnerTaskCreationContextId' <> ''`, + sql`jsonb_typeof(${sandboxCreationOperations.metadata}->'creationProviderLeaseId') = 'string' and ${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' <> ''`, + )); +} + export function pendingCleanupDeadlineSql(now = Date.now()) { const deadline = sql`case when ${environmentLeases.metadata}->>'pendingCleanupInFlight' = 'true' @@ -21,7 +31,7 @@ export async function nextEnvironmentCleanupAt(db: Db, backoffMs = ENVIRONMENT_C extract(epoch from ${environmentLeases.updatedAt}) * 1000 + ${backoffMs}, ${pendingCleanupDeadlineSql()}))` }).from(environmentLeases).where(eq(environmentLeases.status, "pending_cleanup")); const [creation] = await db.select({ next: sql`min(extract(epoch from ${sandboxCreationOperations.nextAttemptAt}) * 1000)` }) - .from(sandboxCreationOperations).where(eq(sandboxCreationOperations.state, "pending")); + .from(sandboxCreationOperations).where(recoverableSandboxCreationCondition()); const deadlines = [row?.next, creation?.next].filter((value): value is string => value != null).map(Number); const saved = deadlines.length ? Math.min(...deadlines) : null; const buffered = hasDeferredEnvironmentCleanup(db) ? Date.now() + backoffMs : null; diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 30acbbe1db5..7ffdb2c379a 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -3382,7 +3382,7 @@ function createPluginEnvironmentDriver( async releaseRunLease(input) { await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"release"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId))); const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true); + (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); await workerManager.call(plugin.id, "environmentReleaseLease", { driverKey, @@ -3425,7 +3425,7 @@ function createPluginEnvironmentDriver( async destroyRunLease(input) { await db.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"pluginLeaseCleanupMethod":"destroy"}'::jsonb` }).where(and(eq(environmentLeases.id, input.lease.id), eq(environmentLeases.companyId, input.lease.companyId))); const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true); + (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); const { pluginKey, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); await destroyPluginEnvironmentLease({ db, @@ -3462,7 +3462,7 @@ function createPluginEnvironmentDriver( async retryPendingSandboxTeardown(input) { const taskStopped = await settleDispatchedEnvironmentTask(db, workerManager, input.lease) || - (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true); + (input.lease.metadata?.runnerTaskLease === true && input.lease.metadata?.runnerTaskDispatched !== true && !input.lease.metadata?.runnerTaskStartup && !input.lease.metadata?.runnerTaskStartupCleanup); const { plugin, driverKey, driverConfig } = await resolvePluginDriverForRelease(input); const environmentId = input.environment?.id ?? readString(input.lease.metadata?.environmentId); if (!environmentId) throw new Error("Plugin lease cleanup has no saved environment identity"); @@ -3778,7 +3778,7 @@ export function environmentRuntimeService( }, /** Dispatch plugin-provided Runner execution, including resource preparation and task lifecycle. */ - async task(input: { companyId: string; leaseId: string; operation: PluginEnvironmentTaskOperation }) { + async task(input: { companyId: string; leaseId: string; operation: PluginEnvironmentTaskOperation; executionContext?: unknown }) { if (!options.pluginWorkerManager) throw new Error("Plugin-provided Runner execution worker manager unavailable"); return executeEnvironmentTask(db, options.pluginWorkerManager, input); }, diff --git a/server/src/services/environment-task-runtime.ts b/server/src/services/environment-task-runtime.ts index 3595d00b430..1ef71b839d2 100644 --- a/server/src/services/environment-task-runtime.ts +++ b/server/src/services/environment-task-runtime.ts @@ -1,15 +1,17 @@ import { attachSandboxCreation, environmentService } from "./environments.js"; -import { createOwnedSandbox, recordSandboxCreationResource } from "./sandbox-creation-operations.js"; +import { createOwnedSandbox, queueSandboxCreationCleanup, recordSandboxCreationResource } from "./sandbox-creation-operations.js"; import { writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; import { releaseProviderRecoveryTransfers, stageProviderFreshWorkspace } from "./provider-fresh-workspace.js"; import { readWorkspaceProviderBinding } from "@paperclipai/shared"; import { settlePluginTaskCredentialHandoff } from "./plugin-task-credential-writeback.js"; import { and, eq, inArray, or, sql } from "drizzle-orm"; import { environmentLeases, executionWorkspaces, environments, heartbeatRuns, issues, projects, type Db } from "@paperclipai/db"; -import { freshWorkspaceSchema, environmentTaskOperationSchema, parseEnvironmentTaskResult, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk"; +import { freshWorkspaceSchema, environmentTaskOperationSchema, parseEnvironmentTaskResult, + type HostToWorkerMethodName, type HostToWorkerMethods, type PluginEnvironmentTaskOperation } from "@paperclipai/plugin-sdk"; import { pluginRegistryService } from "./plugin-registry.js"; import { hasRemoteTerminationReceipt, remoteExecutionHasStopped, remoteTerminationReceipt } from "./remote-execution-termination.js"; import type { PluginWorkerManager } from "./plugin-worker-manager.js"; +import { NativeExecutionContext } from "./native-runtime/native-execution-context.js"; export async function pluginEnvironmentSupportsTasks(db: Db, workers: PluginWorkerManager, lease: { companyId: string; metadata: Record | null }) { const metadata = lease.metadata ?? {}; @@ -31,6 +33,7 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage companyId: string; leaseId: string; operation: PluginEnvironmentTaskOperation; + executionContext?: unknown; }) { const operation = environmentTaskOperationSchema.parse(input.operation); const [row] = await db.select({ lease: environmentLeases, environment: environments, run: heartbeatRuns }) @@ -40,6 +43,13 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage .where(and(eq(environmentLeases.id, input.leaseId), eq(environmentLeases.companyId, input.companyId))).limit(1); if (!row || !row.lease.providerLeaseId) throw new Error("Plugin-provided Runner execution lease unavailable"); const { lease, environment, run } = row; + const context = input.executionContext; + if (context !== undefined) { + if (!(context instanceof NativeExecutionContext) || !run) { + throw new Error("native_execution_context_unavailable"); + } + context.assertScope({ companyId: input.companyId, agentId: run.agentId, runId: run.id }); + } const taskId = row.lease.providerLeaseId; const metadata = lease.metadata ?? {}; const probe = metadata.runnerTaskProbe as { runId?: unknown; sessionId?: unknown; expiresAtMs?: unknown } | undefined; @@ -128,6 +138,19 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage !workers.getWorker(plugin.id)?.supportedMethods.includes("environmentTask")) { throw new Error("Plugin-provided Runner execution provider unavailable"); } + const callWorker = async (method: M, params: HostToWorkerMethods[M][0], timeout?: number): Promise => { + if (!context) { + const cleanup = metadata.runnerTaskStartupCleanup as { contextId?: unknown; requiresPreparationStop?: unknown } | undefined; + if (["stop", "status", "complete", "credentials"].includes(operation.kind) && typeof cleanup?.contextId === "string" && typeof cleanup.requiresPreparationStop === "boolean") { + return workers.call(plugin.id, method, params, timeout, undefined, undefined, cleanup.contextId); + } + return timeout === undefined ? workers.call(plugin.id, method, params) : workers.call(plugin.id, method, params, timeout); + } + const settled = context.trackWorker(); + const result = await workers.call(plugin.id, method, params, Math.max(1, Math.min(timeout ?? 15_000, context.deadlineAtMs - Date.now())), undefined, settled, context.id); + context.assertOpen(); + return result; + }; const issue = scopesProjects && lease.issueId ? await db.select({ id: issues.id }).from(issues) .where(and(eq(issues.id, lease.issueId), eq(issues.companyId, input.companyId))).limit(1).then(rows => rows[0]) : null; if (scopesProjects && lease.issueId && !issue) throw new Error("Plugin-provided Runner execution issue unavailable"); @@ -219,7 +242,7 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage } try { if (operation.kind === "describe" && metadata.runnerTaskReused === true) { - const state = parseEnvironmentTaskResult({kind:"status"}, taskId, await workers.call(plugin.id, "environmentTask", { + const state = parseEnvironmentTaskResult({kind:"status"}, taskId, await callWorker("environmentTask", { driverKey: metadata.driverKey, companyId: input.companyId, environmentId: environment?.id ?? null, issueId: lease.issueId, config: driverConfig, taskId, runId: run?.id ?? null, agentId: run?.agentId ?? null, projectIds, lease: {providerLeaseId: taskId, metadata}, operation:{kind:"status"}, @@ -233,37 +256,74 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage runnerTaskSessionRetained:state.sessionStateAvailable === true})}::jsonb`, }).where(and(eq(environmentLeases.id, metadata.runnerTaskPreviousLeaseId), eq(environmentLeases.companyId, input.companyId))); const replacementDriverKey = metadata.driverKey; + let abandoned = false; const fresh = await createOwnedSandbox(db, { companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId, issueId: lease.issueId, executionWorkspaceId: lease.executionWorkspaceId, provider: lease.provider!, pluginId: plugin.id, driverKey: metadata.driverKey, config: {}, requestKey: taskId, - metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey, runnerTaskLease: true }, - }, identity => workers.call(plugin.id, "environmentAcquireLease", {driverKey:replacementDriverKey, - companyId:input.companyId,environmentId:environment!.id,issueId:lease.issueId,config:driverConfig, - runId:identity.runId,creationOperationId:identity.id,agentId:run!.agentId,adapterType:"paperclip_runner", - ...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? {requestedExpiresAt:metadata.runnerTaskRequestedExpiresAt} : {})})); + metadata: { driver: "plugin", pluginId: plugin.id, pluginKey: plugin.pluginKey, driverKey: metadata.driverKey, + runnerTaskLease: true, agentId: run!.agentId, + ...(context ? { runnerTaskCreationContextId: context.id } : {}) }, + }, async identity => { + const params = { driverKey: replacementDriverKey, companyId: input.companyId, environmentId: environment!.id, + issueId: lease.issueId, config: driverConfig, runId: identity.runId, creationOperationId: identity.id, + agentId: run!.agentId, adapterType: "paperclip_runner", + ...(typeof metadata.runnerTaskRequestedExpiresAt === "string" ? { requestedExpiresAt: metadata.runnerTaskRequestedExpiresAt } : {}) }; + if (!context) return workers.call(plugin.id, "environmentAcquireLease", params, 30_000); + const settled = context.trackWorker(); + let observedResource: Parameters[1] | undefined; + let dispatched = false; + try { + return await context.wait(() => { + dispatched = true; + return workers.call(plugin.id, "environmentAcquireLease", params, 30_000, undefined, settled, context.id, + async result => { + if (!result.providerLeaseId || result.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable"); + const resource = { id: identity.id, companyId: input.companyId, environmentId: environment!.id, + providerLeaseId: result.providerLeaseId, providerMetadata: result.metadata }; + await recordSandboxCreationResource(db, resource); + observedResource = resource; + if (abandoned || !context.active || context.signal.aborted || Date.now() >= context.deadlineAtMs) { + await queueSandboxCreationCleanup(db, resource); + } + }); + }); + } catch (error) { + if (!dispatched) settled(); + abandoned = true; + if (observedResource) await queueSandboxCreationCleanup(db, observedResource); + throw error; + } + }); if (!fresh.providerLeaseId || fresh.providerLeaseId === taskId) throw new Error("Plugin replacement task identity unavailable"); - await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId, - environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId }); const {runnerTaskIdentity:_identity,runnerTaskPriorIdentity:_prior,runnerTaskCredentialHandoff:_handoff, runnerTaskResumeFrom:_resume,runnerTaskPreviousLeaseId:_previous,...rest}=metadata; const replacementMetadata={...rest,providerMetadata:fresh.metadata ?? {},runnerTaskReused:false,runnerTaskDispatched:false, runnerTaskExecutionStopped:false,runnerTaskSessionRetained:false}; - { + try { + context?.assertOpen(); + if (!context) await recordSandboxCreationResource(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId, + environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId }); const expiresAt=fresh.expiresAt ? new Date(fresh.expiresAt) : null; const deadline=typeof metadata.runnerTaskRequestedExpiresAt === "string" ? Date.parse(metadata.runnerTaskRequestedExpiresAt) : null; if ((expiresAt && (!Number.isFinite(expiresAt.getTime()) || expiresAt.getTime() <= Date.now())) || (deadline !== null && (!Number.isFinite(deadline) || !expiresAt || expiresAt.getTime() > deadline))) throw new Error("Plugin replacement task expiry is not bounded"); await attachSandboxCreation(db, String(fresh.metadata!.creationOperationId), { companyId: input.companyId, environmentId: environment!.id, heartbeatRunId: lease.heartbeatRunId }, async tx => { + context?.assertOpen(); const [updated] = await tx.update(environmentLeases).set({providerLeaseId:fresh.providerLeaseId,expiresAt, metadata:{...replacementMetadata,creationOperationId:fresh.metadata!.creationOperationId}, }).where(and(eq(environmentLeases.id,lease.id),eq(environmentLeases.providerLeaseId,taskId), - eq(environmentLeases.status,"active"))).returning({id:environmentLeases.id}); + eq(environmentLeases.status,"active"), sql`coalesce(${environmentLeases.metadata}->>'runnerTaskRetiring', 'false') != 'true'`)).returning({id:environmentLeases.id}); if (!updated) throw new Error("Plugin task replacement ownership changed"); - return (await environmentService(tx).getLeaseById(updated.id))!; + const attached = (await environmentService(tx).getLeaseById(updated.id))!; + context?.assertOpen(); + return attached; }); - + } catch (error) { + if (context) await queueSandboxCreationCleanup(db, { id: String(fresh.metadata!.creationOperationId), companyId: input.companyId, + environmentId: environment!.id, providerLeaseId: fresh.providerLeaseId, providerMetadata: fresh.metadata }); + throw error; } return executeEnvironmentTask(db, workers, input); } @@ -275,14 +335,17 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage return snapshot.credential; }); } - const result = await workers.call(plugin.id, "environmentTask", { + const params = { driverKey: metadata.driverKey, companyId: input.companyId, environmentId: environment?.id ?? null, issueId: lease.issueId, config: driverConfig, taskId, runId: run?.id ?? (activeProbe ? String(probe.runId) : null), agentId: run?.agentId ?? null, projectIds, lease: { providerLeaseId: lease.providerLeaseId, metadata: lease.metadata ?? undefined }, ...(workspaceBinding ? { workspace: workspaceBinding } : {}), ...(freshWorkspace && scopesProjects ? { fresh: freshWorkspace } : {}), operation, - }, operation.kind === "copilot_metadata" ? Math.max(1, Math.min(40_000, Number(probe?.expiresAtMs) - Date.now())) : 15_000); + }; + const timeout = operation.kind === "copilot_metadata" ? Math.max(1, Math.min(40_000, Number(probe?.expiresAtMs) - Date.now())) + : 15_000; + const result = await callWorker("environmentTask", params, timeout); const parsed = parseEnvironmentTaskResult(operation, taskId, result); if (operation.kind === "read_file") { const [current] = await db.select({ lease: environmentLeases, run: heartbeatRuns }).from(environmentLeases) @@ -316,7 +379,9 @@ export async function executeEnvironmentTask(db: Db, workers: PluginWorkerManage } if ((parsed.kind === "status" || parsed.kind === "copilot_metadata") && parsed.executionStopped === true) { await db.update(environmentLeases).set({ - metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ runnerTaskExecutionStopped: true, runnerTaskSessionRetained: parsed.kind === "status" && parsed.sessionStateAvailable === true })}::jsonb`, + metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ runnerTaskExecutionStopped: true, + runnerTaskSessionRetained: parsed.kind === "status" && parsed.sessionStateAvailable === true, + ...(parsed.kind === "status" && parsed.preparationStopped === true ? { runnerTaskPreparationStopped: true } : {}) })}::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, input.companyId))); } if (operation.kind === "complete") { @@ -353,16 +418,22 @@ export async function settleDispatchedEnvironmentTask(db: Db, workers: PluginWor metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || '{"runnerTaskRetiring":true}'::jsonb`, }).where(and(eq(environmentLeases.id, lease.id), eq(environmentLeases.companyId, lease.companyId))) .returning({ metadata: environmentLeases.metadata, heartbeatRunId: environmentLeases.heartbeatRunId }); - if (current?.metadata?.runnerTaskDispatched !== true) { + const startup = current?.metadata?.runnerTaskStartup; + const startupCleanup = current?.metadata?.runnerTaskStartupCleanup as { requiresPreparationStop?: boolean } | undefined; + if (current?.metadata?.runnerTaskDispatched !== true && !startup && !startupCleanup) { if (current?.metadata) await settlePluginTaskCredentialHandoff(db, { companyId: lease.companyId, leaseId: lease.id }, current.metadata, async () => { throw new Error("Undispatched task must not request credentials"); }); return current?.metadata?.runnerTaskExecutionStopped === true; } const scope = { companyId: lease.companyId, leaseId: lease.id }; try { + const contextId = (startupCleanup as { contextId?: unknown } | undefined)?.contextId ?? (startup as { contextId?: unknown } | undefined)?.contextId; + const draining = typeof contextId === "string" ? workers.drainStartup?.(contextId) : undefined; + void draining?.catch(() => undefined); await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "stop" } }); const state = await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "status" } }); - if (state.kind !== "status" || state.executionStopped !== true) throw new Error("Plugin Runner termination is unconfirmed"); + if (state.kind !== "status" || state.executionStopped !== true || ((startup || startupCleanup?.requiresPreparationStop) && state.preparationStopped !== true)) throw new Error("Plugin Runner termination is unconfirmed"); + await draining; await executeEnvironmentTask(db, workers, { ...scope, operation: { kind: "complete" } }); return true; } catch { diff --git a/server/src/services/execution-blocker.ts b/server/src/services/execution-blocker.ts index c22437f1a47..4e57d9b0c88 100644 --- a/server/src/services/execution-blocker.ts +++ b/server/src/services/execution-blocker.ts @@ -16,7 +16,7 @@ import { hasVerifiedWorkspaceRepair, workspaceRecoveryDisplay } from "./workspac const blockerContextKeys = ["issueId", "taskId", "source", "connectionIntentResolution", "interactionKind", "interactionStatus", "mutation", "interactionId"] as const; -const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync"] as const; +const blockerProfileKeys = ["adapterDispatch", "nativeExecutionInput", "nativeWorkspaceSync", "remoteWorkspace"] as const; const blockerContextColumn = sql | null>`case when ${heartbeatRuns.contextSnapshot} is null then null else ( select coalesce(jsonb_object_agg(field.key, field.value), '{}'::jsonb) diff --git a/server/src/services/execution-control-reconciliation.test.ts b/server/src/services/execution-control-reconciliation.test.ts index 10923cd5bc1..1cd52611df5 100644 --- a/server/src/services/execution-control-reconciliation.test.ts +++ b/server/src/services/execution-control-reconciliation.test.ts @@ -1,7 +1,7 @@ import { resolveExternalTestDatabaseUrl } from "../__tests__/helpers/external-test-database.js"; import { enableRunFailureReporting } from "../__tests__/helpers/run-failure-reporting.js"; import { randomUUID } from "node:crypto"; -import { eq } from "drizzle-orm"; +import { eq, inArray } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, issueRecoveryActions, issueThreadInteractions, nativeRunFinalizations, nativeRunResults, statusDecisions, workAssessments } from "@paperclipai/db"; import { @@ -78,6 +78,55 @@ describeEmbeddedPostgres("reconcileAbandonedExecutionControl reports a genuine f return { companyId, agentId, issueId, runId }; } + it("recognizes pending cleanup selected only by the run deadline batch", async () => { + const f = await seedAbandonedRunFixture(), now = new Date(); + const cleanupIds = Array.from({ length: 51 }, () => randomUUID()); + const hidden = cleanupIds[50]!; + await db.insert(heartbeatRuns).values(cleanupIds.map(id => ({ id, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "running", + executionControlDeadlineAt: id === hidden ? new Date(now.getTime() - 120_000) : null }))); + await db.insert(nativeRunFinalizations).values(cleanupIds.map(runId => ({ runId, companyId: f.companyId, + issueId: f.issueId, phase: "terminal_failure", failureCode: "native_startup_cleanup_pending", + controlDeadlineAt: new Date(now.getTime() - (runId === hidden ? 60_000 : 120_000)) }))); + const attempted: string[] = []; + try { + await reconcileAbandonedExecutionControl(db, now, { startupCleanup: async ({ runId }) => { attempted.push(runId); throw new Error("Provider unavailable"); } }); + expect(attempted).toContain(hidden); + expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, hidden)))[0].status).toBe("running"); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, hidden)))[0].controlDeadlineAt! > now).toBe(true); + } finally { + await db.update(nativeRunFinalizations).set({ controlDeadlineAt: null }).where(inArray(nativeRunFinalizations.runId, cleanupIds)); + await db.update(heartbeatRuns).set({ executionControlDeadlineAt: null }).where(inArray(heartbeatRuns.id, cleanupIds)); + } + }); + + it("moves failed startup cleanup retries behind later expired runs", async () => { + const f = await seedAbandonedRunFixture(); + const now = new Date(); + const cleanupIds = Array.from({ length: 50 }, () => randomUUID()); + const firstDeadline = new Date(now.getTime() - 120_000); + await db.insert(heartbeatRuns).values(cleanupIds.map(id => ({ id, companyId: f.companyId, agentId: f.agentId, + nativeIssueId: f.issueId, runtimeMode: "native", status: "running", executionControlDeadlineAt: firstDeadline, + contextSnapshot: { issueId: f.issueId } }))); + await db.insert(nativeRunFinalizations).values(cleanupIds.map(runId => ({ runId, companyId: f.companyId, + issueId: f.issueId, phase: "terminal_failure", failureCode: "native_startup_cleanup_pending", controlDeadlineAt: firstDeadline }))); + const attempted: string[] = []; + const startupCleanup = async ({ runId }: { runId: string }) => { attempted.push(runId); throw new Error("Provider unavailable"); }; + try { + await reconcileAbandonedExecutionControl(db, now, { startupCleanup }); + await reconcileAbandonedExecutionControl(db, now, { startupCleanup }); + expect(new Set(attempted)).toEqual(new Set(cleanupIds)); + expect(attempted).toHaveLength(50); + const [later] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId)); + expect(later?.status).toBe("failed"); + const retries = await db.select().from(nativeRunFinalizations).where(inArray(nativeRunFinalizations.runId, cleanupIds)); + expect(retries.every(row => row.controlDeadlineAt && row.controlDeadlineAt > now)).toBe(true); + } finally { + await db.update(nativeRunFinalizations).set({ controlDeadlineAt: null }).where(inArray(nativeRunFinalizations.runId, cleanupIds)); + await db.update(heartbeatRuns).set({ executionControlDeadlineAt: null }).where(inArray(heartbeatRuns.id, cleanupIds)); + } + }); + it("reports exactly one Sentry event for a genuine finalization-deadline failure", async () => { const { runId } = await seedAbandonedRunFixture(); const captureCallsBefore = mockCaptureRunFailure.mock.calls.length; diff --git a/server/src/services/execution-control-reconciliation.ts b/server/src/services/execution-control-reconciliation.ts index 621fd210cfc..0133f215d27 100644 --- a/server/src/services/execution-control-reconciliation.ts +++ b/server/src/services/execution-control-reconciliation.ts @@ -8,7 +8,7 @@ import { notifyInstructionCollection } from "./instruction-collection-notificati import { notifyExecutionRunWork } from "./execution-control-notifications.js"; import { randomUUID } from "node:crypto"; import { logger } from "../middleware/logger.js"; -import { and, eq, isNotNull, lte, sql } from "drizzle-orm"; +import { and, asc, eq, isNotNull, lte, sql } from "drizzle-orm"; import { agents, heartbeatRuns, @@ -22,17 +22,25 @@ import { hasLiveLegacyController } from "./legacy-controller-lease.js"; import { reportRunFailure } from "./run-failure-report.js"; import { getNativeReviewAssignment, readNativeReviewAssignmentContext } from "./native-runtime/native-review-participant.js"; import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-workspace-recovery.js"; +import { EXECUTION_RECONCILIATION_INTERVAL_MS } from "./execution-control-deadline.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "./delivery-work-notifications.js"; +import { NATIVE_STARTUP_CLEANUP_PENDING, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-runtime/native-startup-cleanup.js"; /** Only newly recorded control deadlines are eligible. Upgrades never replay ambiguous historical runs. */ export async function reconcileAbandonedExecutionControl( db: Db, now = new Date(), + options: { startupCleanup?: (scope: { companyId: string; runId: string }) => Promise } = {}, ) { const nativeDue = await db .select({ runId: nativeRunFinalizations.runId, issueId: nativeRunFinalizations.issueId, companyId: nativeRunFinalizations.companyId, + failureCode: nativeRunFinalizations.failureCode, + leaseOwner: nativeRunFinalizations.leaseOwner, + controllerGeneration: nativeRunFinalizations.controllerGeneration, + attempt: nativeRunFinalizations.attempt, }) .from(nativeRunFinalizations) .where( @@ -41,20 +49,28 @@ export async function reconcileAbandonedExecutionControl( lte(nativeRunFinalizations.controlDeadlineAt, now), ), ) + .orderBy(asc(nativeRunFinalizations.controlDeadlineAt), asc(nativeRunFinalizations.runId)) .limit(50); const controlDue = await db .select({ runId: heartbeatRuns.id, companyId: heartbeatRuns.companyId, context: heartbeatRuns.contextSnapshot, + nativeIssueId: nativeRunFinalizations.issueId, + failureCode: nativeRunFinalizations.failureCode, + leaseOwner: nativeRunFinalizations.leaseOwner, + controllerGeneration: nativeRunFinalizations.controllerGeneration, + attempt: nativeRunFinalizations.attempt, }) .from(heartbeatRuns) + .leftJoin(nativeRunFinalizations, eq(nativeRunFinalizations.runId, heartbeatRuns.id)) .where( and( isNotNull(heartbeatRuns.executionControlDeadlineAt), lte(heartbeatRuns.executionControlDeadlineAt, now), ), ) + .orderBy(asc(heartbeatRuns.executionControlDeadlineAt), asc(heartbeatRuns.id)) .limit(50); const due = [ ...new Map( @@ -62,10 +78,10 @@ export async function reconcileAbandonedExecutionControl( ...nativeDue, ...controlDue.map((row) => ({ ...row, - issueId: + issueId: row.nativeIssueId ?? ( typeof row.context?.issueId === "string" ? row.context.issueId - : null, + : null), })), ].map((row) => [row.runId, row]), ).values(), @@ -78,6 +94,25 @@ export async function reconcileAbandonedExecutionControl( while (nextCandidate < due.length) { const candidate = due[nextCandidate++]!; try { + if (candidate.failureCode === NATIVE_STARTUP_CLEANUP_PENDING) { + if (!options.startupCleanup) continue; + try { + await options.startupCleanup({ companyId: candidate.companyId, runId: candidate.runId }); + } finally { + await db.transaction(async tx => { + if (candidate.issueId) await tx.select({ id: issues.id }).from(issues).where(eq(issues.id, candidate.issueId)).for("update"); + const [owner] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, candidate.runId)).for("update"); + if (owner?.failureCode !== NATIVE_STARTUP_CLEANUP_PENDING || !owner.controlDeadlineAt || owner.controlDeadlineAt > now || + owner.leaseOwner !== candidate.leaseOwner || owner.controllerGeneration !== candidate.controllerGeneration || owner.attempt !== candidate.attempt) return; + const cleanupRetryAt = new Date(Math.max(now.getTime(), Date.now()) + EXECUTION_RECONCILIATION_INTERVAL_MS); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); + await tx.update(nativeRunFinalizations).set({ controlDeadlineAt: cleanupRetryAt }).where(eq(nativeRunFinalizations.runId, candidate.runId)); + await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: cleanupRetryAt }).where(and(eq(heartbeatRuns.id, candidate.runId), + isNotNull(heartbeatRuns.executionControlDeadlineAt), lte(heartbeatRuns.executionControlDeadlineAt, now))); + }); + } + continue; + } // Set inside the transaction only when the write below genuinely // transitions the run into "failed". Read after the transaction // commits, so a rolled-back write never reports a false failure. @@ -128,6 +163,18 @@ export async function reconcileAbandonedExecutionControl( ].some((deadline) => deadline && deadline <= now) ) return false; + if (coordinator?.failureCode === NATIVE_STARTUP_CLEANUP_PENDING) return false; + const startupContext = coordinator && readNativeStartupContext(run, coordinator); + if (startupContext) { + if (!await promoteNativeStartupCleanup(tx as unknown as Db, run, coordinator!, now)) { + const retryAt = new Date(Math.max(now.getTime() + EXECUTION_RECONCILIATION_INTERVAL_MS, + startupContext.deadlineAtMs, coordinator!.leaseExpiresAt?.getTime() ?? 0)); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); + await tx.update(nativeRunFinalizations).set({ controlDeadlineAt: retryAt }).where(eq(nativeRunFinalizations.runId, run.id)); + await tx.update(heartbeatRuns).set({ executionControlDeadlineAt: retryAt }).where(eq(heartbeatRuns.id, run.id)); + } + return false; + } await tx .update(heartbeatRuns) .set({ executionControlDeadlineAt: null }) diff --git a/server/src/services/execution-preparation-timing.ts b/server/src/services/execution-preparation-timing.ts index 9ee7f8a189e..4ba7aa69457 100644 --- a/server/src/services/execution-preparation-timing.ts +++ b/server/src/services/execution-preparation-timing.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from "node:async_hooks"; +import { withDatabaseOperation } from "@paperclipai/db"; import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { createHash } from "node:crypto"; import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api"; @@ -156,7 +157,7 @@ function measure(runId: string, name: string, work: () => T | Promise, ind ended: false, details: !independent && previous?.runId === runId ? previous.details : new Map(), }; - operation = Promise.resolve().then(() => + operation = withDatabaseOperation(name, () => Promise.resolve().then(() => ownership.run(owned, () => runWithRuntimeParent(owned.parent, work))).then(value => { try { span?.end(); } catch {} owned.ended = true; @@ -166,10 +167,10 @@ function measure(runId: string, name: string, work: () => T | Promise, ind try { span?.end(); } catch {} owned.ended = true; throw error; - }); + })); return operation; }; - if (!endpointEnabled()) return (async () => work())(); + if (!endpointEnabled()) return withDatabaseOperation(name, async () => work()); try { const parent = independent ? undefined : activeParent ?? getRunExecutionTraceContext(runId); const trigger = parent ? undefined : trace.getActiveSpan()?.spanContext(); @@ -202,20 +203,22 @@ export function withRunWorkspaceFinalizationTiming(runId: string, work: () => export function withRunResponsePresentationTiming(runId: string, work: () => T | Promise): Promise { const owned = ownership.getStore(); if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) { - return (async () => work())(); + return withDatabaseOperation("task.response_presentation", async () => work()); } return measure(runId, "task.response_presentation", work, true); } -export async function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise { - const owned = ownership.getStore(); - if (!endpointEnabled() || !owned || owned.ended) return work(); - const count = owned.details.get(phase) ?? 0; - if (count >= 16) return work(); - owned.details.set(phase, count + 1); - // A startup step can be more specific than the owning run phase. Keep that - // causal parent while the child publishes its own context to plugin workers. - return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined); +export function measureRunPhase(phase: ExecutionPhase, work: () => T | Promise, counts?: { fileCount?: number; bytes?: number }): Promise { + return withDatabaseOperation(phase, async () => { + const owned = ownership.getStore(); + if (!endpointEnabled() || !owned || owned.ended) return work(); + const count = owned.details.get(phase) ?? 0; + if (count >= 16) return work(); + owned.details.set(phase, count + 1); + // A startup step can be more specific than the owning run phase. Keep that + // causal parent while the child publishes its own context to plugin workers. + return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined); + }); } export function beginRunPhase(phase: ExecutionPhase): (failed?: boolean) => void { diff --git a/server/src/services/explicit-native-continuation.ts b/server/src/services/explicit-native-continuation.ts index 084a25e457f..c93ec6b8681 100644 --- a/server/src/services/explicit-native-continuation.ts +++ b/server/src/services/explicit-native-continuation.ts @@ -52,7 +52,7 @@ const preflightProfile = sql`case then jsonb_build_object('adapterType', field.value->'adapterType') else field.value end), '{}'::jsonb) from jsonb_each(case when jsonb_typeof(${heartbeatRuns.runnerProfileJson}) = 'object' then ${heartbeatRuns.runnerProfileJson} else '{}'::jsonb end) as field(key, value) - where field.key in ('adapterDispatch', 'nativeWorkspaceSync') + where field.key in ('adapterDispatch', 'nativeWorkspaceSync', 'remoteWorkspace') ) end`; const continuationRunFields = { id: heartbeatRuns.id, companyId: heartbeatRuns.companyId, agentId: heartbeatRuns.agentId, diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 9212fc686c8..b9878f8ae20 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -398,6 +398,7 @@ import { isNativeRunnerOwnershipHeld, NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE, } from "./native-runtime/native-runner-ownership.js"; +import { NativeStartupCleanupPendingError, requestNativeStartupCleanup, verifyNativeStartupCleanup, readNativeStartupCleanup } from "./native-runtime/native-startup-cleanup.js"; import { emitAgentTaskRun, emitAgentTaskRunById, @@ -1436,6 +1437,14 @@ export function heartbeatService( } const runControl = createHeartbeatRunControl(db, { + verifyStartupCleanup: async run => { + await verifyNativeStartupCleanup(db, run, () => envOrchestrator.releaseForRun({ + heartbeatRunId: run.id, companyId: run.companyId, agentId: run.agentId, status: "expired", + cancelActiveWork: true, providerResourceDisposition: "destroy", + })); + const marker = readNativeStartupCleanup(run)!; + await options.pluginWorkerManager?.retireStartup?.(marker.contextId); + }, getRun, getRunCleanupState, getRunExecutionWait, @@ -1759,7 +1768,7 @@ export function heartbeatService( }, }); })().catch((error) => { - if (!nativeRecoveryDeadlines.has(runId)) { + if (!nativeRecoveryDeadlines.has(runId) || nativeRecoveryDeadlines.get(runId) === pending) { scheduleNativeSessionResumeDispatch(runId, new Date(Date.now() + 30_000), mode); } logger.error( @@ -1956,6 +1965,19 @@ export function heartbeatService( Parameters[0] | null = null; let nativeSessionResumeScheduled = false; let nativeOwnershipHeld = false; + let nativeStartupCleanupScheduled = false; + let nativeStartupCleanupError: NativeStartupCleanupPendingError | undefined; + const requestStartupCleanup = async (error: NativeStartupCleanupPendingError) => { + nativeStartupCleanupError = error; + try { + nativeStartupCleanupScheduled = await requestNativeStartupCleanup(db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, + contextId: error.contextId, deadlineAtMs: error.deadlineAtMs, leaseOwner: error.leaseOwner, + leaseId: error.leaseId, requiresPreparationStop: error.requiresPreparationStop }); + } catch (writeError) { + scheduleNativeSessionResumeDispatch(run.id, new Date(Date.now() + 30_000), "restart"); + throw writeError; + } + }; let instructionWorkingCopy: ReturnType | null = null; let nativeDispatchStarted = false; let nativeWorkspaceFinalizeScheduled = false; @@ -3527,6 +3549,10 @@ export function heartbeatService( })); } catch (err) { await persistUsageCaptureFailure?.(); + if (err instanceof NativeStartupCleanupPendingError) { + await requestStartupCleanup(err); + return; + } if (err instanceof NativeControllerDetachedForRestartError) { nativeSessionResumeScheduled = true; return; @@ -3659,7 +3685,12 @@ export function heartbeatService( }); } } catch (outerErr) { - if ( + if (nativeStartupCleanupScheduled || nativeStartupCleanupError || outerErr instanceof NativeStartupCleanupPendingError) { + const pending = outerErr instanceof NativeStartupCleanupPendingError ? outerErr : nativeStartupCleanupError; + if (!nativeStartupCleanupScheduled && pending) await requestStartupCleanup(pending).catch(error => { + logger.warn({ err: error, runId: run.id }, "Native startup cleanup intent will retry through owned recovery"); + }); + } else if ( nativeOwnershipHeld || outerErr instanceof NativeRunnerOwnershipUnverifiedError ) { @@ -3696,7 +3727,7 @@ export function heartbeatService( ); } else { await preserveNativeWorkspaceOnFailure(); - if (!nativeOwnershipHeld) await runCompletion.failRunSetup({ + if (!nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError) await runCompletion.failRunSetup({ run, runId, outerErr, signal: executionControl.controller.signal, providerDispatchStarted: legacyAdapterEntered || nativeDispatchStarted, identityRedactor, readFailureReportSecrets, @@ -3746,7 +3777,7 @@ export function heartbeatService( latestRun && !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && - !nativeOwnershipHeld + !nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError ) { latestRun = await terminalizeRunOnLeaseRelease(latestRun).catch( (terminalizeErr) => { @@ -3772,7 +3803,7 @@ export function heartbeatService( if ( !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && - !nativeOwnershipHeld + !nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError ) { // Keep launchers during same-run recovery. At a terminal boundary all // operations have settled; clean before the remote lease can be stopped. @@ -3895,7 +3926,7 @@ export function heartbeatService( }).where(and(eq(heartbeatRuns.id, run.id), eq(heartbeatRuns.status, "cancelled"))); } if (latestRun?.runtimeMode === "native" && isHeartbeatRunTerminalStatus(latestRun.status) && - !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && !nativeOwnershipHeld) { + !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && !nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError) { await settleNativeRunProcess(db, run.id).catch(error => { logger.warn({ err: error, runId: run.id }, "failed to release settled native process identity"); }); @@ -3932,7 +3963,7 @@ export function heartbeatService( caseHandoffCleanupSettled = hasStoppedLocalProcessTree(latestRun); } if (caseHandoffCleanupSettled && latestRun && !nativeSessionResumeScheduled && !nativeWorkspaceFinalizeScheduled && - !nativeOwnershipHeld && isHeartbeatRunTerminalStatus(latestRun.status)) { + !nativeOwnershipHeld && !nativeStartupCleanupScheduled && !nativeStartupCleanupError && isHeartbeatRunTerminalStatus(latestRun.status)) { const handoff = await pendingCaseHandoffForRun(db, run.id); if (handoff) await recordCaseHandoffExecutionReleased(db, run.id, handoff.issueId); } diff --git a/server/src/services/heartbeat/native-stop-evidence-read.test.ts b/server/src/services/heartbeat/native-stop-evidence-read.test.ts index d53a5d4c417..9a139a5cc85 100644 --- a/server/src/services/heartbeat/native-stop-evidence-read.test.ts +++ b/server/src/services/heartbeat/native-stop-evidence-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { heartbeatRuns, issues, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq, sql } from "drizzle-orm"; import { describe, expect, it, vi } from "vitest"; import { isAcknowledgedNativeStop } from "../acknowledged-native-stop.js"; @@ -31,7 +31,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read contextSnapshot: { savedInput: bulk }, runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk, resultJson: result }).returning(); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); let evidence: Awaited>; try { evidence = await readNativeStopExecutionEvidence(db, company.id, id); } @@ -68,7 +68,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read runtimeMode: "native", status: "cancelled" }).returning(); await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(saved)}::jsonb where id = ${run.id}`); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { const evidence = await readNativeStopExecutionEvidence(db, company.id, run.id); diff --git a/server/src/services/heartbeat/queue.ts b/server/src/services/heartbeat/queue.ts index 18b5033ea39..5f8bd897f65 100644 --- a/server/src/services/heartbeat/queue.ts +++ b/server/src/services/heartbeat/queue.ts @@ -2299,7 +2299,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`, )); const retry = readTaskRetryIntent(receipt?.payload); - if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || + if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || retry.queuedChatRequestId || retry.actorId !== receipt.requestedByActorId) return null; const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId && durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId; @@ -2518,6 +2518,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende // retain distinct durable receipts even when the same gate blocks them. const coalesceExecutionWait = opts.requestedByActorType === "system" && + opts.allowRunCoalescing !== false && !receiptRequest && !wakeCommentId && queuedCommentIdsFromRunContext(enrichedContextSnapshot).length === 0 && @@ -4369,7 +4370,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)), )).then(rows => rows[0] ?? null) : null; - const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false; + // External input owns an exact authorized source batch. Board input + // must retain its separate receipt, including during normal resume. + const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false; // A resumed receipt already passed admission under this issue lock. // Consume it with its successor even when chat keeps other messages // in separate turns, or finalization will deliver it a second time. diff --git a/server/src/services/heartbeat/recovery.test.ts b/server/src/services/heartbeat/recovery.test.ts index 1573f3bdd49..73a665bb53d 100644 --- a/server/src/services/heartbeat/recovery.test.ts +++ b/server/src/services/heartbeat/recovery.test.ts @@ -168,13 +168,13 @@ describe("heartbeat recovery module callbacks", () => { expect(database.access).not.toHaveBeenCalled(); }); - it("persists an authentication ownership hold before its event without terminating or retrying", async () => { + it.each(["adopted_runner_authentication_timeout"] as const)("persists a %s ownership hold before its event without terminating or retrying", async reason => { const database = guardedDatabase(); const deps = callbacks(database.db); const run = { id: "run", runtimeMode: "native", status: "running" } as Run; const blocked = { ...run, nativePhase: "terminal_failure", errorCode: NATIVE_OWNERSHIP_UNVERIFIED_ERROR_CODE }; deps.setRunStatusFromLive.mockResolvedValue({ updated: true, run: blocked }); - expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason: "adopted_runner_authentication_timeout" })).toBe(blocked); + expect(await createHeartbeatRecovery(database.db, deps).markNativeOwnershipUnverified(run, { reason })).toBe(blocked); expect(deps.setRunStatusFromLive).toHaveBeenCalledWith("run", "running", ["running"], expect.objectContaining({ nativePhase: "terminal_failure" })); expect(deps.appendRunEvent).toHaveBeenCalledAfter(deps.setRunStatusFromLive); expect(deps.cancelHeartbeatNativeRun).not.toHaveBeenCalled(); @@ -207,6 +207,42 @@ describePostgres("heartbeat recovery module ownership", () => { return (await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, issueId, invocationSource: "automation", status: "running", runtimeMode: "legacy" }).returning())[0]; } + it("routes an expired pre-capability startup to cleanup before generic orphan ownership holds", async () => { + const run = await seedRun(), leaseId = randomUUID(), environmentId = randomUUID(), leaseOwner = randomUUID(); + const deadlineAtMs = Date.now() - 1_000, contextId = randomUUID(); + await db.update(agents).set({ adapterType: "paperclip_runner" }).where(eq(agents.id, run.agentId)); + await db.update(issues).set({ executionRunId: run.id }).where(eq(issues.id, run.issueId!)); + await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: run.issueId, nativePhase: "observed", + runnerInstanceId: randomUUID(), nativeSessionId: randomUUID(), runnerProfileJson: { nativeStartupContext: { + schema: "paperclip.native-startup-context.v1", companyId: run.companyId, agentId: run.agentId, runId: run.id, + contextId, deadlineAtMs, leaseId, leaseOwner, controllerGeneration: 1, attempt: 1 } } }).where(eq(heartbeatRuns.id, run.id)); + await db.insert(nativeRunFinalizations).values({ runId: run.id, companyId: run.companyId, issueId: run.issueId!, + phase: "observed", attempt: 1, leaseOwner, controllerGeneration: 1, leaseExpiresAt: new Date(Date.now() - 1_000), + controllerBootId: randomUUID(), controllerPid: process.pid, controllerProcessStartedAt: new Date() }); + await db.insert(environments).values({ id: environmentId, name: environmentId, driver: "plugin", config: {} }); + await db.insert(environmentLeases).values({ id: leaseId, companyId: run.companyId, environmentId, heartbeatRunId: run.id, + issueId: run.issueId, provider: "plugin", providerLeaseId: "early-task", status: "active", metadata: { runnerTaskLease: true } }); + const deps = callbacks(db); + deps.getRun.mockImplementation(async id => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0] ?? null); + native.claims.mockImplementationOnce(claimNativeRestartRecoveries); + try { + await createHeartbeatRecovery(db, deps).recoverActiveExecutions({ runId: run.id }); + expect(native.claims).toHaveBeenCalledWith(expect.objectContaining({ runIds: [run.id] })); + const [current] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + expect(current).toMatchObject({ status: "running", errorCode: "native_startup_cleanup_pending", + resultJson: { nativeStartupCleanup: { contextId, state: "pending" } } }); + expect(deps.executeRun).not.toHaveBeenCalled(); + expect(deps.setRunStatusFromLive).not.toHaveBeenCalled(); + } finally { + await db.delete(environmentLeases).where(eq(environmentLeases.id, leaseId)); + await db.delete(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); + await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, run.id)); + await db.delete(activityLog).where(eq(activityLog.runId, run.id)); + await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, run.issueId!)); + await db.delete(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)); + } + }); + async function ownedProcess() { const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", `paperclip-recovery-${randomUUID()}`], { stdio: "ignore", detached: true, env: { PATH: process.env.PATH }, diff --git a/server/src/services/heartbeat/recovery.ts b/server/src/services/heartbeat/recovery.ts index c72ff89214b..48d1a532c27 100644 --- a/server/src/services/heartbeat/recovery.ts +++ b/server/src/services/heartbeat/recovery.ts @@ -1,3 +1,4 @@ +import { readNativeStartupCleanup, readNativeStartupContext } from "../native-runtime/native-startup-cleanup.js"; import { pendingFailedNativeRetryCancellationCondition } from "../native-runtime/native-cancellation-request.js"; import { notifyLiveRunRecovery } from "../live-run-recovery-notifications.js"; import { registerLiveRecoveryWork } from "./live-recovery-work.js"; @@ -1038,7 +1039,9 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD ? "Native restart recovery is waiting for safe ownership evidence" : disposition.kind === "already_finalized" ? "Native restart recovery found an already-finalized result" - : "Native restart recovery blocked ambiguous or conflicting ownership", + : disposition.reason === "native_startup_cleanup_pending" + ? "Native startup timed out; cancellation and verified cleanup are queued" + : "Native restart recovery blocked ambiguous or conflicting ownership", payload: { restartKind: isClaim ? disposition.restartKind : restartKind, recoveryRequestId: isClaim @@ -2251,6 +2254,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD nativeControllerProcessStartedAt: nativeRunFinalizations.controllerProcessStartedAt, nativeControllerLeaseExpiresAt: nativeRunFinalizations.leaseExpiresAt, + nativeLeaseOwner: nativeRunFinalizations.leaseOwner, + nativeControllerGeneration: nativeRunFinalizations.controllerGeneration, + nativeAttempt: nativeRunFinalizations.attempt, + nativeResultId: nativeRunFinalizations.resultId, nativeWorkspaceFinalizationPending: pendingNativeWorkspaceFinalizationCondition(db), }) .from(heartbeatRuns) @@ -2271,7 +2278,10 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD && !isNativeRunnerOwnershipHeld(run) && !activeRunExecutions.has(run.id) && !runningProcesses.has(run.id), ).map(({ run }) => run.id); - const remoteRecoveryRunIds = new Set(await listExpiredNativeRunnerAuthorities(db, now, expiredNativeRunIds)); + const earlyStartupRunIds = activeRuns.filter(row => expiredNativeRunIds.includes(row.run.id) && + readNativeStartupContext(row.run, { leaseOwner: row.nativeLeaseOwner, controllerGeneration: row.nativeControllerGeneration ?? 0, + attempt: row.nativeAttempt ?? 0, resultId: row.nativeResultId })).map(row => row.run.id); + const remoteRecoveryRunIds = new Set([...earlyStartupRunIds, ...await listExpiredNativeRunnerAuthorities(db, now, expiredNativeRunIds)]); if (remoteRecoveryRunIds.size) { const recovery = await recoverNativeRunsAfterRestart(now, [...remoteRecoveryRunIds]); for (const claim of recovery.claims) resumedRunIds.add(claim.runId); @@ -2321,6 +2331,7 @@ export function createHeartbeatRecovery(db: Db, dependencies: HeartbeatRecoveryD nativeControllerLeaseExpiresAt, nativeWorkspaceFinalizationPending, } of activeRuns) { + if (run.runtimeMode === "native" && readNativeStartupCleanup(run)) continue; if (parseObject(run.contextSnapshot).nativeQuestionCancellation) continue; // Native recovery owns these dispositions. Its writes happened after this // inventory was read; do not reap or overwrite them using the old row. diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 0553b686a42..835ac1fa078 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -123,7 +123,8 @@ import { } from "../native-runtime/index.js"; import { runningProcesses } from "../../adapters/index.js"; import { parseObject } from "../../adapters/utils.js"; -import { isNativeRunnerOwnershipHeld } from "../native-runtime/native-runner-ownership.js"; +import { isNativeRunnerOwnershipHeld, nativeRunnerOwnershipNotHeldCondition } from "../native-runtime/native-runner-ownership.js"; +import { readNativeStartupCleanup, acknowledgeNativeStartupCleanup } from "../native-runtime/native-startup-cleanup.js"; import { withCurrentBudgetEnforcement, type BudgetEnforcementScope, @@ -177,6 +178,7 @@ export interface HeartbeatRunControlDependencies extends Pick, "mergeRunStopMetadataForAgent" | "setRunStatusFromLive" | "setRunStatus" | "setWakeupStatus" | "appendRunEvent">, Pick { + verifyStartupCleanup?: (run: typeof heartbeatRuns.$inferSelect) => Promise; options: { closeWarmNativeSessionsForRun?: (input: { runId: string; @@ -763,6 +765,10 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont eq(agentWakeupRequests.requestedByActorType, "user"), )); const intent = readTaskRetryIntent(wake?.payload); + if (intent?.queuedChatRequestId) { + await reconcileTaskChatRetryReceipts(db, { companyId, requestId }); + return; + } if (!wake || !intent || intent.chatActionId || intent.canonicalIntentId || intent.generationId !== wake.id || intent.actorId !== wake.requestedByActorId) return; const issueId = readNonEmptyString(wake.payload?.issueId); @@ -788,6 +794,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`, + sql`${agentWakeupRequests.payload}->'taskRetry'->>'queuedChatRequestId' is null`, notInArray(issues.status, ["done", "cancelled"]))) .orderBy(asc(agentWakeupRequests.updatedAt)).limit(50); const chatEnabled = !retries.some(retry => retry.conversationAgentId) @@ -1394,13 +1401,19 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont } let terminationSettled = false; try { - await cancelHeartbeatNativeRun({ - db, - runId: run.id, - reason, - runtimeMode: run.runtimeMode, - ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), - }); + const startupCleanupRun = run.runtimeMode === "native" ? await getRun(run.id) : null; + if (startupCleanupRun && readNativeStartupCleanup(startupCleanupRun) && !control && !running && !activeRunExecutions.has(run.id)) { + if (!dependencies.verifyStartupCleanup) throw conflict("Native startup cleanup worker unavailable"); + await dependencies.verifyStartupCleanup(startupCleanupRun); + } else { + await cancelHeartbeatNativeRun({ + db, + runId: run.id, + reason, + runtimeMode: run.runtimeMode, + ...(options.cancellationRequestId ? { cancellationRequestId: options.cancellationRequestId } : {}), + }); + } if (running) { await terminateHeartbeatRunProcess({ pid: running.child.pid, @@ -1461,12 +1474,13 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont } const finishedAt = new Date(); - const persistedCancellationResult = - run.runtimeMode === "native" - ? await getRun(run.id).then((current) => - parseObject(current?.resultJson), - ) - : {}; + let persistedCancellationRun = run.runtimeMode === "native" ? await getRun(run.id) : null; + if (persistedCancellationRun && readNativeStartupCleanup(persistedCancellationRun)) { + if (!dependencies.verifyStartupCleanup) throw conflict("Native startup cleanup worker unavailable"); + await dependencies.verifyStartupCleanup(persistedCancellationRun); + persistedCancellationRun = await getRun(run.id); + } + const persistedCancellationResult = parseObject(persistedCancellationRun?.resultJson); return await setRunStatusFromLive( run.id, "cancelled", @@ -1515,12 +1529,18 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont persistedCancellationResult.nativeCancellation, } : {}), + ...(Object.hasOwn(persistedCancellationResult, "nativeStartupCleanup") + ? { nativeStartupCleanup: persistedCancellationResult.nativeStartupCleanup } : {}), }, } : {}), }, undefined, - pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) : undefined, + and(pendingNativeRetry ? nativeRetryCancellationCommitCondition(persistedCancellationResult) + : persistedCancellationRun && readNativeStartupCleanup(persistedCancellationRun) ? and(nativeRunnerOwnershipNotHeldCondition(), + sql`not exists (select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${run.id} and ${nativeRunFinalizations.resultId} is not null)`) + : undefined, + run.runtimeMode === "native" ? sql`coalesce(${heartbeatRuns.resultJson}->'nativeStartupCleanup'->>'state', '') <> 'pending'` : undefined), ); } catch (error) { if (processCancellationSettlement) { @@ -1552,6 +1572,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont const cancelled = cancellation.run; if (cancellation.updated && cancelled) { + if (readNativeStartupCleanup(cancelled)) await acknowledgeNativeStartupCleanup(db, { companyId: cancelled.companyId, runId: cancelled.id }); await setWakeupStatus(run.wakeupRequestId, "cancelled", { finishedAt: cancelled.finishedAt ?? new Date(), error: reason, diff --git a/server/src/services/heartbeat/run-lifecycle.ts b/server/src/services/heartbeat/run-lifecycle.ts index 8f4c0dc7595..5cf432a9134 100644 --- a/server/src/services/heartbeat/run-lifecycle.ts +++ b/server/src/services/heartbeat/run-lifecycle.ts @@ -1,3 +1,4 @@ +import { readNativeStartupCleanup } from "../native-runtime/native-startup-cleanup.js"; import { notifyStrandedAgentStatus } from "../live-run-recovery-notifications.js"; import { notifyRunDispatch } from "../run-dispatch-notifications.js"; import { preserveAcceptedRunUsageProjectionSql } from "../usage-receipts.js"; @@ -1159,6 +1160,9 @@ export function createHeartbeatLifecycle(db: Db, dependencies: HeartbeatLifecycl .where(eq(heartbeatRuns.id, runId)) .then((rows) => rows[0] ?? null); const previousStatus = previous?.run ?? null; + if (status === "failed" && patch?.errorCode === "process_lost" && previousStatus?.runtimeMode === "native" && readNativeStartupCleanup(previousStatus)) { + return { updated: false, run: previousStatus }; + } if (previousStatus && (status === "cancelled" || status === "interrupted")) { patch = { ...patch, resultJson: cancellationResultJson(previousStatus, status, patch?.resultJson, patch?.errorCode, patch?.error) }; diff --git a/server/src/services/heartbeat/run-state.test.ts b/server/src/services/heartbeat/run-state.test.ts index 28faa0efa6d..7769b779da8 100644 --- a/server/src/services/heartbeat/run-state.test.ts +++ b/server/src/services/heartbeat/run-state.test.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import type { Db } from "@paperclipai/db"; import { agents, @@ -186,13 +186,13 @@ describePostgres("heartbeat run state database wiring", () => { stdoutExcerpt: payload, stderrExcerpt: payload, }).returning(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); const read = await state.getRunExecutionWait(companyId, run.id); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(4096); expect(read).toMatchObject({ id: run.id, companyId, agentId: agent.id, status: "cancelled", contextSnapshot: context, @@ -228,7 +228,7 @@ describePostgres("heartbeat run state database wiring", () => { resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload, }).returning(); const observed = createDb(database.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); @@ -236,7 +236,7 @@ describePostgres("heartbeat run state database wiring", () => { id: run.id, companyId, scopeKind: "issue", issueId: issue.id, executionWorkspaceId: workspaceId, }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(1); expect(summary.responseValueBytes).toBeLessThan(1024); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); @@ -260,7 +260,7 @@ describePostgres("heartbeat run state database wiring", () => { }).returning(); const observed = createDb(database.connectionString); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns }, capture); try { const state = extracted.createHeartbeatRunState(observed); @@ -270,7 +270,7 @@ describePostgres("heartbeat run state database wiring", () => { runtimeModeResolvedAt: null, startedAt: run.startedAt, createdAt: run.createdAt, contextSnapshot: { issueId: issue.id }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(2048); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); @@ -321,7 +321,7 @@ describePostgres("heartbeat run state database wiring", () => { await state.getRunWorkspaceSettlement(run.id); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); expect(await state.getRunCleanupState(run.id)).toEqual({ id: run.id, companyId, agentId: agent.id, runtimeMode: run.runtimeMode, @@ -334,7 +334,7 @@ describePostgres("heartbeat run state database wiring", () => { contextSnapshot: { timerClaimWasFirstHeartbeat: true }, }); capture.flush(); - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBe(2); expect(summary.responseValueBytes).toBeLessThan(2000); expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run); diff --git a/server/src/services/heartbeat/runtime-execution.ts b/server/src/services/heartbeat/runtime-execution.ts index 1ec069325c1..268ebf4aa41 100644 --- a/server/src/services/heartbeat/runtime-execution.ts +++ b/server/src/services/heartbeat/runtime-execution.ts @@ -1,3 +1,4 @@ +import { NativeStartupCleanupPendingError } from "../native-runtime/native-startup-cleanup.js"; import { NativeControllerRecoveryWaitingError } from "../native-runtime/native-controller-recovery.js"; import { usesNativeUsageDelivery } from "../native-runtime/native-usage-accounting.js"; @@ -1072,15 +1073,21 @@ export async function executeHeartbeatRuntime(db: Db, input: HeartbeatRuntimeExe && !(nativeExecution && "access" in nativeExecution.workspace && nativeExecution.workspace.access === "none")) { input.effects.onProviderResourceDisposition("stop_and_retain"); } - await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx - .update(heartbeatRuns) - .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) - .where( - and( - eq(heartbeatRuns.id, run.id), - eq(heartbeatRuns.status, "running"), - ), - ), { companyId: run.companyId }); + try { + await writeExecutionControl(db, DELIVERY_QUEUES.executionFinalization, tx => tx + .update(heartbeatRuns) + .set({ executionControlDeadlineAt: new Date(Date.now() + 60_000) }) + .where( + and( + eq(heartbeatRuns.id, run.id), + eq(heartbeatRuns.status, "running"), + ), + ), { companyId: run.companyId }); + } catch (writeError) { + if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr; + throw writeError; + } + if (adapterErr instanceof NativeStartupCleanupPendingError) throw adapterErr; if ( issueRef && context.resumeSessionGoalHeartbeat === true && diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index beb43a34b0f..255d0a5806f 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -7,7 +7,7 @@ import { pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { captureDirectorySnapshot, directorySnapshotSha256, serializeDirectorySnapshot } from "@paperclipai/adapter-utils/workspace-restore-merge"; import type { AdapterExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; -import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, type Db } from "@paperclipai/db"; +import { agentWakeupRequests, heartbeatRunEvents, companyObservability, issueRecoveryActions, agents, authUsers, companyMemberships, companies, createDb, applyPendingMigrations, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, environmentLeases, environments, executionWorkspaces, executionWorkspaceRepositoryAccess, heartbeatRuns, issues, issueAccessGrants, nativeRunFinalizations, plugins, projects, projectWorkspaces, workspaceOperations, type Db } from "@paperclipai/db"; import { eq, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -170,15 +170,16 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace }); it.each([ - { kind: "empty", repoUrl: null, branchMode: null, fresh: false, ephemeral: true }, - { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false }, - { kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false }, - { kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false }, - { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false }, - { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false }, - { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true }, - ] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh) without materializing a controller checkout", async (scenario) => { - const { kind, repoUrl, branchMode, fresh } = scenario; + { kind: "empty", repoUrl: null, branchMode: null, fresh: false, strategyChange: false, ephemeral: true }, + { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: null, fresh: false, strategyChange: false }, + { kind: "git", repoUrl: "ssh://git@github.com/example/remote.git", branchMode: null, fresh: false, strategyChange: false }, + { kind: "git", repoUrl: "git@github.com:example/remote.git", branchMode: null, fresh: false, strategyChange: false }, + { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "existing", fresh: false, strategyChange: false }, + { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: false, strategyChange: false }, + { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true, strategyChange: false }, + { kind: "git", repoUrl: "https://github.com/example/remote.git", branchMode: "create", fresh: true, strategyChange: true }, + ] as const)("reserves a $kind remote workspace ($repoUrl, branch: $branchMode, fresh: $fresh, strategy change: $strategyChange) without materializing a controller checkout", async (scenario) => { + const { kind, repoUrl, branchMode, fresh, strategyChange } = scenario; const { input, source, issue, envOrchestrator } = await fixture(); const [savedEnvironment] = await db.insert(environments).values({ name: `Remote ${randomUUID()}`, driver: "sandbox", status: "active", config: { provider: "daytona" } }).returning(); @@ -194,7 +195,7 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace if (branchMode) { input.policy.requestedExecutionWorkspaceMode = "isolated_workspace"; input.policy.effectiveExecutionWorkspaceMode = "isolated_workspace"; - input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree", baseRef: "main", + input.config.mergedConfig = strategyChange ? { workspaceStrategy: { type: "project_primary" } } : { workspaceStrategy: { type: "git_worktree", baseRef: "main", ...(branchMode === "existing" ? { existingBranch: "feature/diverged" } : { branchTemplate: "test-{{issue.identifier}}" }) } }; } if (kind === "git") { @@ -243,9 +244,44 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace expect(workspace.mode).toBe(branchMode ? "isolated_workspace" : "shared_workspace"); expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { kind }, writerRunId: input.run.id }); if (branchMode) expect(workspace.metadata?.remoteWorkspace).toMatchObject({ source: { - ref: "main", branchMode, branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1", + ref: "main", branchMode, + ...(strategyChange ? {} : { branchName: branchMode === "existing" ? "feature/diverged" : "test-WSP-1" }), } }); await expect(access(workspace.cwd!)).rejects.toThrow(); + if ((fresh || strategyChange) && branchMode === "create") { + expect(workspace.strategyType).toBe("git_worktree"); + await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }) + .where(eq(heartbeatRuns.id, input.run.id)); + const [saved] = await db.update(executionWorkspaces).set({ metadata: { + ...workspace.metadata, + remoteWorkspace: { ...workspace.metadata!.remoteWorkspace as object, writerRunId: null, dirty: false }, + } }).where(eq(executionWorkspaces.id, workspace.id)).returning(); + const [nextRun] = await db.insert(heartbeatRuns).values({ companyId: issue.companyId, + agentId: input.agent.id, issueId: issue.id, scopeKind: "issue", status: "running", runtimeMode: "legacy" }).returning(); + input.run = nextRun; + input.task.freshWorkspace = null; + input.task.issueRef = { ...input.task.issueRef!, executionWorkspaceId: saved.id, executionWorkspacePreference: "reuse_existing" }; + input.reuse.existingExecutionWorkspace = toExecutionWorkspace(saved); + input.reuse.reusableExistingExecutionWorkspace = toExecutionWorkspace(saved); + input.reuse.requestedShouldReuseExisting = true; + input.reuse.workspaceReuseRequest = resolveExecutionWorkspaceReuseRequestForIssue({ + issueExecutionWorkspaceId: saved.id, issueExecutionWorkspacePreference: "reuse_existing", existingExecutionWorkspaceStatus: "active", + }); + if (strategyChange) input.config.mergedConfig = { workspaceStrategy: { type: "git_worktree" } }; + await expect(prepareHeartbeatWorkspace(db, input)).rejects.toThrow("reached remote acquisition"); + const operations = await db.select().from(workspaceOperations).where(eq(workspaceOperations.heartbeatRunId, nextRun.id)); + const freshness = operations.filter(operation => operation.phase === "workspace_config_freshness"); + if (strategyChange) { + expect(freshness).toHaveLength(1); + expect(freshness[0].metadata).toMatchObject({ action: "replace", changedCategories: ["strategy"] }); + } else expect(freshness).toEqual([]); + const [retained] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, saved.id)); + expect(retained.id).toBe(workspace.id); + expect(retained.metadata?.configFingerprint).toMatchObject({ + workspaceHash: (saved.metadata?.configFingerprint as { workspaceHash: string }).workspaceHash, + }); + expect(retained.metadata?.remoteWorkspace).toMatchObject({ writerRunId: nextRun.id }); + } } finally { for (const [key, value] of previous) { if (value === undefined) delete process.env[key]; else process.env[key] = value; diff --git a/server/src/services/heartbeat/workspace-preparation.ts b/server/src/services/heartbeat/workspace-preparation.ts index e4c76adb48b..824822ff1a2 100644 --- a/server/src/services/heartbeat/workspace-preparation.ts +++ b/server/src/services/heartbeat/workspace-preparation.ts @@ -543,7 +543,7 @@ async function prepareHeartbeatWorkspaceLocked(db: Db, input: HeartbeatWorkspace const skipHostProvision = copyOnlyProvisioning || hostProvisionDriver === "sandbox" || hostProvisionDriver === "ssh" || hostProvisionDriver === "plugin"; const hostExecutionWorkspaceConfig = isDotRun ? {} : stripHostWorkspaceProvision({ - config: fresh ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig, + config: fresh && !remoteAuthority ? { ...mergedConfig, workspaceStrategy: { type: "project_primary" } } : mergedConfig, skipHostProvision, }); const executionWorkspaceBase = { diff --git a/server/src/services/idle-sleep-safety.ts b/server/src/services/idle-sleep-safety.ts index 88f193cae55..9573395d18b 100644 --- a/server/src/services/idle-sleep-safety.ts +++ b/server/src/services/idle-sleep-safety.ts @@ -75,7 +75,7 @@ const WORK_CHECKS = [ state NOT IN ('saved', 'unchanged', 'resolved', 'unavailable', 'conflict') OR process_stopped_at IS NULL OR next_attempt_at IS NOT NULL OR receipt ? 'baseline' OR receipt->>'cleanupPending' = 'true'`), - workCheck("sandbox_creation_operations", `SELECT 1 FROM sandbox_creation_operations WHERE state = 'pending'`), + workCheck("sandbox_creation_operations", `SELECT 1 FROM sandbox_creation_operations WHERE state IN ('pending', 'needs_reconciliation')`), workCheck("budget_reservations", `SELECT 1 FROM budget_reservations WHERE state NOT IN ('settled', 'released')`), workCheck("budget_policies", `SELECT 1 FROM budget_policies WHERE enforcement_version > enforcement_delivered_version`), workCheck("agents.lifecycle", `SELECT 1 FROM agents WHERE lifecycle_state IN diff --git a/server/src/services/issue-display-status.ts b/server/src/services/issue-display-status.ts new file mode 100644 index 00000000000..5c9bdefb54b --- /dev/null +++ b/server/src/services/issue-display-status.ts @@ -0,0 +1,37 @@ +import { sql, type SQL } from "drizzle-orm"; +import { issues } from "@paperclipai/db"; +import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME, type IssueDisplayStatus } from "@paperclipai/shared"; + +/** Read-only projection, shared by task reads, search, filters and counts. + * The persisted deadline is the scheduler's source of truth. Locks and recovery + * take precedence; a retained monitor alone must not disguise active work. + */ +export function issueDisplayStatusSql(): SQL { + // Drizzle removes column qualifiers in a single-table SELECT. Keep these + // outer references explicitly qualified so correlated reads cannot bind to + // the run/recovery table's own id or company_id instead. + const issueId = sql`"issues"."id"`; + const companyId = sql`"issues"."company_id"`; + return sql`CASE WHEN + ${issues.status} = 'in_progress' + AND ${issues.assigneeAgentId} IS NOT NULL AND ${issues.assigneeUserId} IS NULL + AND ${issues.monitorNextCheckAt} IS NOT NULL + AND ${issues.executionRunId} IS NULL AND ${issues.checkoutRunId} IS NULL + AND ${issues.executionState}->>'currentParticipant' IS NULL + AND COALESCE(${issues.executionState}#>>'{monitor,status}', 'scheduled') = 'scheduled' + AND ${issues.executionPolicy}->'monitor' IS NOT NULL + AND COALESCE(${issues.executionPolicy}#>>'{monitor,serviceName}', '') <> ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME} + AND NOT EXISTS ( + SELECT 1 FROM heartbeat_runs display_run + WHERE display_run.company_id = ${companyId} + AND display_run.issue_id = ${issueId} + AND display_run.status IN ('queued', 'running') + ) + AND NOT EXISTS ( + SELECT 1 FROM issue_recovery_actions display_recovery + WHERE display_recovery.company_id = ${companyId} + AND display_recovery.source_issue_id = ${issueId} + AND display_recovery.status IN ('active', 'escalated') + ) + THEN 'monitoring' ELSE ${issues.status} END`; +} diff --git a/server/src/services/issue-summaries.ts b/server/src/services/issue-summaries.ts index 485bb2250db..645ff8a86c2 100644 --- a/server/src/services/issue-summaries.ts +++ b/server/src/services/issue-summaries.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { and, eq, inArray, isNull, type SQL } from "drizzle-orm"; import { issues, type Db } from "@paperclipai/db"; import { ISSUE_SUMMARIES_MAX_IDS } from "@paperclipai/shared"; @@ -18,6 +19,7 @@ export async function listIssueSummaries( title: issues.title, identifier: issues.identifier, status: issues.status, + displayStatus: issueDisplayStatusSql(), updatedAt: issues.updatedAt, externalConversationState: externalConversationStateSql(), projectId: issues.projectId, diff --git a/server/src/services/issues.ts b/server/src/services/issues.ts index e3f66a720b5..3bd4b50fa24 100644 --- a/server/src/services/issues.ts +++ b/server/src/services/issues.ts @@ -1,4 +1,6 @@ import { recordMcpAttachmentCleanup, type McpAttachmentCleanup } from "./public-mcp/upload-cleanup.js"; +import type { IssueDisplayStatus } from "@paperclipai/shared"; +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js"; import { finishedIdleWake } from "./idle-work-proof.js"; import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js"; @@ -1901,6 +1903,7 @@ export function parseStatusFilter( export interface IssueFilters { attention?: "blocked"; status?: string | readonly string[]; + displayStatus?: string | readonly string[]; /** * Filter by assignee agent ID. * - `string` (UUID): match issues assigned to that agent. @@ -1945,14 +1948,14 @@ export interface IssueFilters { q?: string; limit?: number; offset?: number; - sortField?: "updated" | "id"; + sortField?: "updated" | "created" | "id"; afterId?: string; sortDir?: "asc" | "desc"; /** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */ updatedSince?: string; } -type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null }; +type IssueRow = typeof issues.$inferSelect & { displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null }; type IssueLabelRow = typeof labels.$inferSelect; type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect; type IssueActiveRunRow = { @@ -2560,12 +2563,13 @@ function blockerWorkspaceFacts(companyId: string, workspaceScope: SQL) { } /** An issue's own operation takes precedence over the legacy NULL fallback. - * Only a strictly later successful finalize clears either barrier. */ + * A later successful finalize clears either barrier. A successful operation + * enclosed by its owning finalize is also saved, although it started later. */ function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, workspaceId: SQL) { return sql`${workspaceId} is not null and exists ( select 1 from (values (1)) seed(value) left join lateral ( - select phase, status, started_at from workspace_operations blocker_operation + select phase, status, started_at, finished_at, heartbeat_run_id from workspace_operations blocker_operation where blocker_operation.company_id = ${companyId}::uuid and blocker_operation.execution_workspace_id = ${workspaceId} and blocker_operation.issue_id = ${issueId} @@ -2576,6 +2580,20 @@ function pendingFinalizeBlockerCondition(companyId: string, issueId: SQL, worksp and coalesce(latest_blocker_operation.status, blocker_workspace_facts.status) = 'succeeded') and (blocker_workspace_facts.finalized_at is null or blocker_workspace_facts.finalized_at <= coalesce(latest_blocker_operation.started_at, blocker_workspace_facts.started_at)) + and not ( + latest_blocker_operation.status = 'succeeded' + and latest_blocker_operation.finished_at is not null + and exists ( + select 1 from workspace_operations owning_finalize + where owning_finalize.company_id = ${companyId}::uuid + and owning_finalize.execution_workspace_id = ${workspaceId} + and owning_finalize.issue_id = ${issueId} + and owning_finalize.heartbeat_run_id = latest_blocker_operation.heartbeat_run_id + and owning_finalize.phase = 'workspace_finalize' and owning_finalize.status = 'succeeded' + and owning_finalize.started_at <= latest_blocker_operation.started_at + and owning_finalize.finished_at >= latest_blocker_operation.finished_at + ) + ) )`; } @@ -2930,7 +2948,7 @@ function myLastTouchAtExpr(companyId: string, userId: string) { COALESCE(${myLastCommentAt}, to_timestamp(0)), COALESCE(${myLastReadAt}, to_timestamp(0)), COALESCE(CASE WHEN ${issues.createdByUserId} = ${userId} AND ${humanCreatedIssueCondition()} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0)), - COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.updatedAt} ELSE NULL END, to_timestamp(0)) + COALESCE(CASE WHEN ${issues.assigneeUserId} = ${userId} THEN ${issues.createdAt} ELSE NULL END, to_timestamp(0)) ) `; } @@ -2997,6 +3015,7 @@ function unreadForUserCondition(companyId: string, userId: string) { FROM ${issueComments} WHERE ${issueComments.issueId} = ${issues.id} AND ${issueComments.companyId} = ${companyId} + AND ${issueComments.deletedAt} IS NULL AND ( ${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId} @@ -3119,8 +3138,10 @@ export function deriveIssueUserContext( issue.createdByUserId === userId && !isAgentCreatedIssue(issue) ? normalizeDate(issue.createdAt) : null; + // Assignment establishes inbox participation, not a read. Other people's + // comments also advance updatedAt, so only creation is a stable fallback. const assignedTouchAt = - issue.assigneeUserId === userId ? normalizeDate(issue.updatedAt) : null; + issue.assigneeUserId === userId ? normalizeDate(issue.createdAt) : null; const myLastTouchAt = [myLastCommentAt, myLastReadAt, createdTouchAt, assignedTouchAt] .filter((value): value is Date => value instanceof Date) @@ -3219,6 +3240,11 @@ function issueListOrderBy( }, ) { if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)]; + if (sortField === "created") { + return sortDir === "asc" + ? [asc(issues.createdAt), asc(issues.id)] + : [desc(issues.createdAt), desc(issues.id)]; + } const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId); if (sortField === "updated") { const activityOrder = @@ -3604,6 +3630,7 @@ function appendBlockerAttentionEdges( } type IssueRelationSummaryRow = { + displayStatus?: IssueDisplayStatus; relatedId: string; identifier: string | null; title: string; @@ -3621,6 +3648,7 @@ function summarizeIssueRelationRow( identifier: row.identifier, title: row.title, status: row.status as IssueRelationIssueSummary["status"], + displayStatus: row.displayStatus, priority: row.priority as IssueRelationIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -3736,6 +3764,7 @@ async function terminalExplicitBlockersByRoot( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -5040,6 +5069,7 @@ const issueListSelect = { END `, status: issues.status, + displayStatus: issueDisplayStatusSql(), statusVersion: issues.statusVersion, lastStatusDecisionId: issues.lastStatusDecisionId, workMode: issues.workMode, @@ -5125,7 +5155,8 @@ async function userCommentStatsForIssues( lastExternalCommentAt: sql` MAX( CASE - WHEN ${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId} + WHEN ${issueComments.deletedAt} IS NULL + AND (${issueComments.authorUserId} IS NULL OR ${issueComments.authorUserId} <> ${userId}) THEN ${issueComments.createdAt} END ) @@ -5280,6 +5311,7 @@ async function blockedByMapForIssues( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -5302,6 +5334,7 @@ async function blockedByMapForIssues( identifier: row.identifier, title: row.title, status: row.status as IssueRelationIssueSummary["status"], + displayStatus: row.displayStatus, priority: row.priority as IssueRelationIssueSummary["priority"], assigneeAgentId: row.assigneeAgentId, assigneeUserId: row.assigneeUserId, @@ -6421,6 +6454,8 @@ async function blockedInboxIssueConditions( filters?.lowTrustBoundary, ); if (lowTrustCondition) conditions.push(lowTrustCondition); + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length > 0) { conditions.push( @@ -6873,7 +6908,7 @@ export function issueService(db: Db) { async function getIssueByUuid(id: string) { const row = await retryIdempotentDatabaseOperation(() => db - .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() }) .from(issues) .where(eq(issues.id, id)) .then((rows) => rows[0] ?? null)); @@ -6884,7 +6919,7 @@ export function issueService(db: Db) { async function getIssueByIdentifier(identifier: string) { const row = await retryIdempotentDatabaseOperation(() => db - .select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() }) + .select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() }) .from(issues) .where(eq(issues.identifier, identifier.toUpperCase())) .then((rows) => rows[0] ?? null)); @@ -7483,6 +7518,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -7503,6 +7539,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -8194,6 +8231,9 @@ export function issueService(db: Db) { if (filters?.sortField === "id" && filters.attention) { throw unprocessable("ID ordering is not supported for blocked attention lists"); } + if (filters?.sortField === "created" && filters.attention) { + throw unprocessable("Creation-time ordering is not supported for blocked attention lists"); + } if (filters?.afterId !== undefined && ( !isUuidLike(filters.afterId) || filters.sortField !== "id" || filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0 @@ -8282,6 +8322,8 @@ export function issueService(db: Db) { filters?.lowTrustBoundary, ); if (lowTrustCondition) conditions.push(lowTrustCondition); + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length === 1) { conditions.push(eq(issues.status, statuses[0])); @@ -8575,6 +8617,8 @@ export function issueService(db: Db) { conditions.push(nonIdleSlackIssueCondition()); } } + const displayStatuses = parseStatusFilter(filters?.displayStatus); + if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses)); const statuses = parseStatusFilter(filters?.status); if (statuses.length === 1) conditions.push(eq(issues.status, statuses[0]!)); @@ -8896,6 +8940,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, @@ -9523,6 +9568,7 @@ export function issueService(db: Db) { identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, diff --git a/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts b/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts index 7caef7cf216..6f9a222e39e 100644 --- a/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts +++ b/server/src/services/native-runtime/__tests__/helpers/committed-replay.ts @@ -1,5 +1,5 @@ import { heartbeatRuns, type Db } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { expect } from "vitest"; @@ -21,11 +21,11 @@ export async function verifyCommittedReplayPayloadReads( const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { await replay(); } finally { capture.flush(); } - const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; + const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!; expect(summary.completedQueries).toBeGreaterThan(0); expect(summary.responseValueBytes).toBeLessThan(100_000); const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); diff --git a/server/src/services/native-runtime/agent-chat-response-recovery.test.ts b/server/src/services/native-runtime/agent-chat-response-recovery.test.ts index aca26894574..918653a5fd1 100644 --- a/server/src/services/native-runtime/agent-chat-response-recovery.test.ts +++ b/server/src/services/native-runtime/agent-chat-response-recovery.test.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import { and, eq, sql } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, authUsers, companies, completionContracts, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues, - nativeRunFinalizations, nativeRunResults, chatTaskHandoffs, chatCompletionDeliveries, agentWakeupRequests, type Db } from "@paperclipai/db"; + nativeRunFinalizations, nativeRunResults, issueThreadInteractions, chatTaskHandoffs, chatCompletionDeliveries, agentWakeupRequests, type Db } from "@paperclipai/db"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; import { NativeRunCoordinatorStore } from "./native-run-coordinator-store.js"; @@ -26,7 +26,7 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => }, 60_000); afterAll(async () => { await db?.$client.end(); await database?.cleanup(); }); - async function fixture(conversation = true, options: { succeed?: boolean } = {}) { + async function fixture(conversation = true, options: { succeed?: boolean; disposition?: "done" | "needs_review" } = {}) { const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); const sessionId = randomUUID(), runnerId = randomUUID(), userId = `fixture-${randomUUID()}`, wakeId = randomUUID(); await db.insert(authUsers).values({ id: userId, name: "Conversation owner", email: `${userId}@example.test`, @@ -53,10 +53,12 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => const prpEvent = { runId, normalizedSessionId: sessionId, turnId: "hire-turn", sourceEventId: "fixture-final-answer", payload: { kind: "agentMessage", channel: "final", providerPhase: "final", text: reply } }; await db.insert(heartbeatRunEvents).values({ companyId, runId, agentId, seq: 170, eventType: "item.completed", payload: { prpEvent } }); - await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: "done", + const disposition = options.disposition ?? "done"; + await port.completeRun({ result: { schema: "paperclip.run_result.v1", reportedWorkDisposition: disposition, summary: "A completed hire, with its final message preserved separately.", completionClaim: { contractRevision: "test", - objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], attentionRequests: [], artifacts: [] }, - terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: "done" }, turnId: "hire-turn" }); + objectiveSatisfied: true, criteria: [], remainingWork: [] }, evidence: [], verification: [], + attentionRequests: disposition === "needs_review" ? [{ kind: "review", ownerClass: "human", summary: "Review the completed work." }] : [], artifacts: [] }, + terminal: { schema: "paperclip.prp.terminal.v1", runTerminalState: "succeeded", turnTerminalState: "completed", reportedWorkDisposition: disposition }, turnId: "hire-turn" }); await finalizeNativeRun({ db, runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: false }); expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, runId)))[0].phase).toBe("committed"); if (options.succeed !== false) await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date("2020-01-01T00:02:00Z") }).where(eq(heartbeatRuns.id, runId)); @@ -641,6 +643,38 @@ describe.skipIf(!support.supported)("committed Agent Chat reply recovery", () => expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply })]); expect((await issue(f.issueId)).conversationAgentId).toBeNull(); }); + it("recovers a deferred review reply once without resolving the review or rerunning work", async () => { + const f = await fixture(false, { disposition: "needs_review" }); + const before = await issue(f.issueId); + const reviews = await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, f.issueId)); + expect(before.status).toBe("in_review"); + expect(reviews).toEqual([expect.objectContaining({ status: "pending" })]); + expect(await comments(f.runId)).toEqual([]); + const work = nativeResponsePresentationWork(db), entries = await work.load(f.input); + expect(entries).toHaveLength(1); + await expect(work.check(entries[0])).resolves.toBeNull(); + expect(await comments(f.runId)).toEqual([expect.objectContaining({ body: f.reply, createdByRunId: f.runId })]); + expect(await issue(f.issueId)).toEqual({ ...before, updatedAt: expect.any(Date) }); + expect(await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.issueId, f.issueId))).toEqual(reviews); + expect(await work.load(f.input)).toEqual([]); + await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false); + expect(await comments(f.runId)).toHaveLength(1); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.nativeIssueId, f.issueId))).toHaveLength(1); + }); + it.each(["missing-final", "wrong-turn", "digest", "skip-comment"])( + "keeps deferred review response recovery closed for %s", async fault => { + const f = await fixture(false, { disposition: "needs_review" }); + if (fault === "missing-final") await db.delete(heartbeatRunEvents).where(eq(heartbeatRunEvents.runId, f.runId)); + if (fault === "wrong-turn") await db.update(heartbeatRunEvents).set({ payload: { prpEvent: { ...f.prpEvent, turnId: "another-turn" } } }) + .where(and(eq(heartbeatRunEvents.runId, f.runId), eq(heartbeatRunEvents.seq, 170))); + if (fault === "digest") await db.update(nativeRunResults).set({ canonicalSha256: "invalid-digest" }).where(eq(nativeRunResults.runId, f.runId)); + if (fault === "skip-comment") await db.update(heartbeatRuns).set({ contextSnapshot: { issueId: f.issueId, skipIssueComment: true } }).where(eq(heartbeatRuns.id, f.runId)); + const before = await issue(f.issueId); + await expect(recoverCommittedNativeChatResponse(db, f.input)).resolves.toBe(false); + expect(await comments(f.runId)).toEqual([]); + expect(await issue(f.issueId)).toEqual(before); + }, + ); it("rolls back a partial reply write without changing the waiting state or consuming recovery", async () => { const f = await fixture(), abort = new Error("fixture-commit-aborted"); const failing = new Proxy(db, { get(target, key) { return key === "transaction" diff --git a/server/src/services/native-runtime/assigned-mcp-tools.test.ts b/server/src/services/native-runtime/assigned-mcp-tools.test.ts index a5708ba1668..cfcef3b18e2 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.test.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.test.ts @@ -19,6 +19,50 @@ describe("assigned MCP runner tools", () => { const searchName = "paperclip_search_assigned_tools"; const callName = "paperclip_call_assigned_tool"; + it.each(["", " \t\n"])("discovers and calls assigned tools through MCP with a blank schema reference %j", async schemaTool => { + const github = descriptor("github_bot:read_pull_request"); + const f = fixture([github, descriptor("calendar.search")]); + const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" }); + const bridge = await startOpenCodeMcpBridge({ tools: assigned.definitions(), handler: call => assigned.execute(call) }); + let id = 0; + const call = async (name: string, args: Record) => { + const response = await fetch(bridge.url, { + method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" }, + body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }), + }); + const result = (await response.json()).result; + expect(result.isError).not.toBe(true); + return JSON.parse(result.content[0].text); + }; + try { + const page = await call(searchName, { query: "github", schemaTool }); + expect(page.tools).toHaveLength(1); + expect(page.tools[0]).toMatchObject({ description: expect.stringContaining(github.name), inputSchema: github.parametersSchema }); + expect(await call(searchName, { query: "github", schemaTool, schemaOffset: 0 })).toEqual(page); + expect((await call(searchName, { query: "", schemaTool })).tools).toHaveLength(2); + await call(callName, { name: page.tools[0].name, arguments: { query: "fixture" } }); + expect(f.executeTool).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({ + tool: github.name, parameters: { query: "fixture" }, gatewayPublicId: f.gatewayPublicId, sessionToken: f.bearerToken, + })); + f.listToolsForNamedGateway.mockResolvedValue([]); + expect(await call(searchName, { query: "github", schemaTool })).toEqual({ tools: [], nextOffset: null }); + } finally { + await bridge.close(); + } + }); + + it("explains how to recover from an unknown schema reference or an offset without a reference", async () => { + const f = fixture([descriptor("github_bot:read_pull_request")]); + const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" }); + await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool: "unknown" } })) + .rejects.toThrow("Omit schemaTool to search assigned tools"); + for (const schemaTool of [undefined, "", " \t\n"]) { + await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool, schemaOffset: 1 } })) + .rejects.toThrow("A positive schemaOffset requires an exact inputSchemaRef in schemaTool"); + } + expect(f.executeTool).not.toHaveBeenCalled(); + }); + it.each([undefined, "on_demand"] as const)("advertises usable page limits through MCP discovery (%s)", async delivery => { const f = fixture(Array.from({ length: 25 }, (_, i) => descriptor(`operation_${i}`))); const assigned = await createAssignedMcpTools({ ...f, delivery }); diff --git a/server/src/services/native-runtime/assigned-mcp-tools.ts b/server/src/services/native-runtime/assigned-mcp-tools.ts index 00473c3b5d5..6e5bf20bf23 100644 --- a/server/src/services/native-runtime/assigned-mcp-tools.ts +++ b/server/src/services/native-runtime/assigned-mcp-tools.ts @@ -21,8 +21,8 @@ const ON_DEMAND_TOOLS: ToolDefinition[] = [ query: { type: "string", maxLength: 200 }, offset: { type: "integer", minimum: 0 }, limit: { type: "integer", minimum: 1, maximum: 20, default: 5, description: "Maximum results per page (1 to 20; default 5)." }, - schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result." }, - schemaOffset: { type: "integer", minimum: 0 }, + schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result. Omit or leave blank for a normal tool search." }, + schemaOffset: { type: "integer", minimum: 0, description: "Schema chunk offset. A positive offset requires an exact inputSchemaRef in schemaTool." }, }, required: ["query"], }, @@ -116,14 +116,17 @@ export async function createAssignedMcpTools(input: { const offset = args.offset ?? 0; const limit = args.limit ?? 5; const schemaOffset = args.schemaOffset ?? 0; + const hasSchemaTool = typeof args.schemaTool === "string" && args.schemaTool.trim().length > 0; if (typeof args.query !== "string" || args.query.length > 200 || typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 || typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 20 || (args.schemaTool !== undefined && typeof args.schemaTool !== "string") || - typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0 || - (args.schemaOffset !== undefined && args.schemaTool === undefined)) { + typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0) { throw new Error("assigned_mcp_tool_invalid_arguments"); } + if (schemaOffset > 0 && !hasSchemaTool) { + throw new Error("assigned_mcp_tool_invalid_arguments: A positive schemaOffset requires an exact inputSchemaRef in schemaTool. Omit schemaTool and schemaOffset to search assigned tools."); + } // Intersect fresh grants with this session's pinned catalog. Revocations // take effect immediately; newly assigned tools require a new session. const current = new Map(deduplicateToolDescriptors(await input.gateway.listToolsForNamedGateway({ @@ -135,9 +138,9 @@ export async function createAssignedMcpTools(input: { return fresh && permits(tool) && permits(tool, currentWorkMode) && permits(fresh) && permits(fresh, currentWorkMode); }); - if (typeof args.schemaTool === "string") { + if (hasSchemaTool) { const selected = authorized.find(([name]) => name === args.schemaTool); - if (!selected) throw new Error("assigned_mcp_tool_unknown"); + if (!selected) throw new Error("assigned_mcp_tool_unknown: schemaTool must match an authorized inputSchemaRef. Omit schemaTool to search assigned tools, then use an exact inputSchemaRef from the results."); const schema = JSON.stringify(selected[1].parametersSchema); const end = schemaOffset + schemaChunkCharacters; return { diff --git a/server/src/services/native-runtime/chat-attachment-reuse.test.ts b/server/src/services/native-runtime/chat-attachment-reuse.test.ts index d0de5ab04c9..7870a8033d9 100644 --- a/server/src/services/native-runtime/chat-attachment-reuse.test.ts +++ b/server/src/services/native-runtime/chat-attachment-reuse.test.ts @@ -29,7 +29,7 @@ import { toolApplications, toolConnections, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; import { createLocalDiskStorageProvider } from "../../storage/local-disk-provider.js"; @@ -322,7 +322,7 @@ describe("native same-conversation chat attachment reuse", () => { const deliveries = await db.select().from(chatDeliveries).where(eq(chatDeliveries.endpointId, endpointId)); const bulk = randomBytes(64 * 1024).toString("base64"); const events: Array> = []; - let capture: ReturnType | undefined; + let capture: ReturnType | undefined; try { await db.update(chatEndpoints).set({ communicationInstructions: bulk, setup: { ...endpoint.setup, unusedContent: bulk } as typeof endpoint.setup, @@ -335,7 +335,7 @@ describe("native same-conversation chat attachment reuse", () => { .where(eq(chatDeliveries.id, delivery.id)); } const now = Date.now(); - capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatConversations, chatEndpoints, chatEndpointResources, chatExternalPrincipals, chatIdentityLinks, companyMemberships }, capture); for (const lockMode of ["read", "blocking", "nonblocking"] as const) { diff --git a/server/src/services/native-runtime/external-chat-wait.integration.test.ts b/server/src/services/native-runtime/external-chat-wait.integration.test.ts index f2951d3b8c0..3bb91b68a38 100644 --- a/server/src/services/native-runtime/external-chat-wait.integration.test.ts +++ b/server/src/services/native-runtime/external-chat-wait.integration.test.ts @@ -8,7 +8,7 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { and, eq, ne, sql } from "drizzle-orm"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { agentWakeupRequests, @@ -922,7 +922,7 @@ describe("native external-chat response wait", () => { await authorizeChatConversationForBoundRun(db, fixture, contextSnapshot, "read"); const observed = createDb(temporary?.connectionString ?? externalTestDatabaseUrl!); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { issues, heartbeatRuns }, capture); try { expect(await listExternalReviewWaits(observed, fixture.companyId, [fixture.issueId])).toEqual(governed ? [] : [ @@ -5462,7 +5462,7 @@ describe("native external-chat response wait", () => { if (reply) await db.update(chatActions).set({ result: { ...reply.result, savedProviderOutput: payload } }).where(eq(chatActions.id, reply.id)); const observed = createDb(temporary?.connectionString ?? externalTestDatabaseUrl!); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, chatActions }, capture); try { const context = await new PaperclipRunnerToolAuthority(observed, fixture) diff --git a/server/src/services/native-runtime/native-completion-feedback.test.ts b/server/src/services/native-runtime/native-completion-feedback.test.ts index df586909b15..ad9ba109e86 100644 --- a/server/src/services/native-runtime/native-completion-feedback.test.ts +++ b/server/src/services/native-runtime/native-completion-feedback.test.ts @@ -1,4 +1,4 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { agents, companies, createDb, documentRevisions, executionWorkspaces, executionWorkspaceRepositories, heartbeatRuns, issues, issueThreadInteractions, issueTreeHolds, issueDocuments } from "@paperclipai/db"; @@ -19,11 +19,11 @@ describe("native final-response feedback", () => { let db: ReturnType; beforeAll(async () => { temporary = await startRecoveryPostgres("native-final-response-"); db = createDb(temporary.connectionString); }); afterAll(async () => { await temporary?.cleanup(); }); - async function fixture() { - const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); - await db.insert(companies).values({ id: companyId, name: "Feedback", issuePrefix: `FB${companyId.slice(0, 8).toUpperCase()}` }); + async function fixture(existingCompanyId?: string) { + const companyId = existingCompanyId ?? randomUUID(), agentId = randomUUID(), issueId = randomUUID(), runId = randomUUID(); + if (!existingCompanyId) await db.insert(companies).values({ id: companyId, name: "Feedback", issuePrefix: `FB${companyId.slice(0, 8).toUpperCase()}` }); await db.insert(agents).values({ id: agentId, companyId, name: "Worker", adapterType: "paperclip_runner", status: "active" }); - await db.insert(issues).values({ id: issueId, companyId, identifier: `FB${companyId.slice(0, 8).toUpperCase()}-1`, title: "Save the task document", status: "in_progress", assigneeAgentId: agentId }); + await db.insert(issues).values({ id: issueId, companyId, identifier: `FB${companyId.slice(0, 8).toUpperCase()}-${existingCompanyId ? 2 : 1}`, title: "Save the task document", status: "in_progress", assigneeAgentId: agentId }); await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, nativeIssueId: issueId, status: "running", runtimeMode: "native", contextSnapshot: { issueId } }); await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, issueId)); const authority = new PaperclipRunnerToolAuthority(db, { companyId, agentId, issueId, runId }); @@ -32,6 +32,17 @@ describe("native final-response feedback", () => { } }) as { document: { id: string; latestRevisionId: string }; documentHref: string }; return { companyId, agentId, issueId, runId, saved }; } + async function publishedFileReport(value: Awaited>, objective = "Save a document on this task. Export the results as a CSV."): Promise { + await db.update(issues).set({ description: objective }).where(eq(issues.id, value.issueId)); + const bytes = Buffer.from("category,total\nverified,3\n"); + const authority = new PaperclipRunnerToolAuthority(db, { companyId: value.companyId, agentId: value.agentId, + issueId: value.issueId, runId: value.runId, workspaceRoot: "/workspace/test", executionTargetKind: "remote", readRemoteTaskFile: async () => bytes }); + const receipt = await authority.execute({ tool: "register_deliverable", callId: "csv", arguments: { + idempotencyKey: "csv", filename: "results.csv", contentType: "text/csv", contentRef: "results.csv", + byteSize: bytes.length, sha256: createHash("sha256").update(bytes).digest("hex"), title: "Results CSV", + } }) as { downloadPath: string }; + return { ...done, evidence: [{ ref: `${receipt.downloadPath}#download` }] }; + } it("rejects a duplicate approval report for the exact pending tool-action card", async () => { const value = await fixture(), interactionId = randomUUID(); await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: value.companyId, issueId: value.issueId, @@ -211,6 +222,55 @@ describe("native final-response feedback", () => { expect(feedback).toContain("When compatible with the requested response format"); expect(feedback).not.toContain("publish secrets"); }); + it.each(["evidence", "artifacts", "criterion"] as const)("accepts its generated saved-document link in %s alongside a published file", async location => { + const value = await fixture(); + const report = await publishedFileReport(value); + const feedback = await nativeCompletionFeedback(db, value.runId, report); + const href = /\[Saved document\]\(([^)]+)\)/u.exec(feedback)?.[1]; + expect(href).toBe(value.saved.documentHref); + const referenced = location === "evidence" ? { ...report, evidence: [...report.evidence, { ref: href! }] } + : location === "artifacts" ? { ...report, artifacts: [{ kind: "document", ref: href! }] } + : { ...report, completionClaim: { ...report.completionClaim, criteria: [{ criterionId: "document", status: "satisfied" as const, evidenceRefs: [href!] }] } }; + await expect(nativeCompletionFeedback(db, value.runId, referenced)) + .resolves.toContain("Completion report accepted"); + }); + it("accepts its generated document guidance link when the objective requests only a CSV", async () => { + const value = await fixture(); + await db.update(issues).set({ title: "Export the results as a CSV" }).where(eq(issues.id, value.issueId)); + const report = await publishedFileReport(value, "Export the results as a CSV."); + const feedback = await nativeCompletionFeedback(db, value.runId, report); + const href = /\[Saved document\]\(([^)]+)\)/u.exec(feedback)?.[1]; + expect(href).toBe(value.saved.documentHref); + await expect(nativeCompletionFeedback(db, value.runId, { + ...report, evidence: [...report.evidence, { ref: href! }], + })).resolves.toContain("Completion report accepted"); + }); + it("retains foreign task, foreign company, bogus link, and local file guards", async () => { + const current = await fixture(), otherTask = await fixture(current.companyId), otherCompany = await fixture(); + const report = await publishedFileReport(current); + for (const ref of [otherTask.saved.documentHref, otherCompany.saved.documentHref, + current.saved.documentHref.replace("#document-output", "#document-missing"), + current.saved.documentHref.replace("/issues/", "/not-issues/"), + "//files.example.test/results.csv", "/tmp/results.csv", "results.csv"]) { + await expect(nativeCompletionFeedback(db, current.runId, { ...report, evidence: [...report.evidence, { ref }] })) + .rejects.toThrow("workspace-only file"); + } + }); + it("does not accept the link after its current revision loses publication proof", async () => { + const value = await fixture(), report = await publishedFileReport(value); + const referenced = { ...report, evidence: [...report.evidence, { ref: value.saved.documentHref }] }; + await expect(nativeCompletionFeedback(db, value.runId, referenced)).resolves.toContain("Completion report accepted"); + await documentService(db).upsertIssueDocument({ format: "markdown", issueId: value.issueId, key: "output", title: "Replacement", + body: "Revision without a matching native publication receipt", baseRevisionId: value.saved.document.latestRevisionId, + createdByAgentId: value.agentId, createdByRunId: value.runId }); + await expect(nativeCompletionFeedback(db, value.runId, referenced)).rejects.toThrow("requested task document is only a workspace file"); + }); + it("does not use a published document link as the requested downloadable file", async () => { + const value = await fixture(); + await db.update(issues).set({ description: "Save a document on this task. Export the results as a CSV." }).where(eq(issues.id, value.issueId)); + await expect(nativeCompletionFeedback(db, value.runId, { ...done, evidence: [{ ref: value.saved.documentHref }] })) + .rejects.toThrow("no accessible delivery evidence"); + }); it("does not link stale saved revisions", async () => { const value = await fixture(); await documentService(db).upsertIssueDocument({ format: "markdown", issueId: value.issueId, key: "output", title: "Updated", body: "New version", diff --git a/server/src/services/native-runtime/native-completion-feedback.ts b/server/src/services/native-runtime/native-completion-feedback.ts index 55198db45e9..bd269a9df21 100644 --- a/server/src/services/native-runtime/native-completion-feedback.ts +++ b/server/src/services/native-runtime/native-completion-feedback.ts @@ -17,7 +17,6 @@ import { and, eq, inArray, notInArray } from "drizzle-orm"; import { approvals, agents, - companies, heartbeatRuns, completionContracts, issueApprovals, @@ -40,10 +39,7 @@ function record(value: unknown): Record { /** Reconstruct links from current task state, never from provider-supplied URLs. */ async function savedDocumentLinks(db: Db, run: typeof heartbeatRuns.$inferSelect, issue: typeof issues.$inferSelect) { const saved = await publishedTaskDocuments(db, { companyId: run.companyId, issueId: issue.id }); - if (!saved.length) return []; - const [company] = await db.select({ issuePrefix: companies.issuePrefix }).from(companies).where(eq(companies.id, run.companyId)); - if (!company) return []; - return saved.map(document => `[Saved document](/${encodeURIComponent(company.issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}#document-${encodeURIComponent(document.key)})`); + return saved.map(document => `[Saved document](${document.href})`); } /** Read current constraints before accepting the report, not a premature status commit. */ diff --git a/server/src/services/native-runtime/native-deliverable-feedback.ts b/server/src/services/native-runtime/native-deliverable-feedback.ts index b761f61a249..b9369cd2959 100644 --- a/server/src/services/native-runtime/native-deliverable-feedback.ts +++ b/server/src/services/native-runtime/native-deliverable-feedback.ts @@ -1,5 +1,5 @@ import { and, eq } from "drizzle-orm"; -import { assets, documents, documentRevisions, heartbeatRuns, issueDocuments, issueAttachments, issueWorkProducts, mcpAttachmentUploads, dotRunnerAssignments, type Db } from "@paperclipai/db"; +import { assets, companies, documents, documentRevisions, heartbeatRuns, issueDocuments, issueAttachments, issues, issueWorkProducts, mcpAttachmentUploads, dotRunnerAssignments, type Db } from "@paperclipai/db"; import type { PrpStructuredRunResult } from "../../vendor/paperclip-runner/index.js"; function evidenceRefs(value: unknown): string[] { @@ -184,8 +184,11 @@ export function explicitlyRequestsTaskDocumentOutput(objective: string): boolean */ export async function publishedTaskDocuments(db: Db, binding: { companyId: string; issueId: string }) { const saved = await db.select({ id: documents.id, revisionId: documents.latestRevisionId, - key: issueDocuments.key, resultJson: heartbeatRuns.resultJson }) + key: issueDocuments.key, resultJson: heartbeatRuns.resultJson, + issuePrefix: companies.issuePrefix, issueIdentifier: issues.identifier }) .from(issueDocuments) + .innerJoin(issues, and(eq(issues.id, issueDocuments.issueId), eq(issues.companyId, binding.companyId))) + .innerJoin(companies, eq(companies.id, issues.companyId)) .innerJoin(documents, and(eq(documents.id, issueDocuments.documentId), eq(documents.companyId, binding.companyId))) .innerJoin(documentRevisions, and(eq(documentRevisions.id, documents.latestRevisionId), eq(documentRevisions.documentId, documents.id), eq(documentRevisions.companyId, binding.companyId))) @@ -197,7 +200,8 @@ export async function publishedTaskDocuments(db: Db, binding: { companyId: strin const receipt = record(value), result = record(receipt.result), published = record(result.document); return receipt.operationId === "write_document" && ["applied", "duplicate"].includes(String(result.disposition)) && published.id === document.id && published.latestRevisionId === document.revisionId; - })).map(({ id, revisionId, key }) => ({ id, revisionId, key })); + })).map(({ id, revisionId, key, issuePrefix, issueIdentifier }) => ({ id, revisionId, key, + href: `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issueIdentifier ?? binding.issueId)}#document-${encodeURIComponent(key)}` })); } /** Files cited as completed output must be reachable outside the agent workspace. */ @@ -209,17 +213,21 @@ export async function validateNativeDeliverableEvidence( if (result.reportedWorkDisposition !== "done") return; const fileRequested = explicitlyRequestsFileOutput(binding.objective); const taskDocumentRequested = explicitlyRequestsTaskDocumentOutput(binding.objective); - const publishedTaskDocument = taskDocumentRequested && (await publishedTaskDocuments(db, binding)).length > 0; const artifactRefs = new Set(evidenceRefs(result.artifacts)); const refs = new Set([ ...evidenceRefs(result.evidence), ...artifactRefs, ...result.completionClaim.criteria.flatMap(({ evidenceRefs }) => evidenceRefs), ]); + const citesTaskDocument = [...refs].some(ref => /^\/[^/?#]+\/issues\/[^/?#]+#document-[^/?#]+$/u.test(ref.trim())); + const publishedDocuments = taskDocumentRequested || citesTaskDocument ? await publishedTaskDocuments(db, binding) : []; + const publishedTaskDocument = taskDocumentRequested && publishedDocuments.length > 0; + const documentHrefs = new Set(publishedDocuments.map(document => document.href)); let registeredAttachment = false; for (const value of refs) { if (typeof value !== "string") continue; const ref = value.trim(); + if (documentHrefs.has(ref)) continue; const attachmentPath = /^\/api\/attachments\/([^/?#]+)\/content(?:[?#].*)?$/u.exec(ref); if (ref.startsWith("deliverable:") || attachmentPath) { const id = attachmentPath?.[1] ?? ref.slice("deliverable:".length); diff --git a/server/src/services/native-runtime/native-execution-context-worker.test.ts b/server/src/services/native-runtime/native-execution-context-worker.test.ts index 3fb4d77d8bc..2c511ce108f 100644 --- a/server/src/services/native-runtime/native-execution-context-worker.test.ts +++ b/server/src/services/native-runtime/native-execution-context-worker.test.ts @@ -32,8 +32,8 @@ const invocations = []; const send = (id, result) => process.stdout.write(JSON.stringify({jsonrpc:"2.0",id,result})+"\\n"); readline.createInterface({input:process.stdin,crlfDelay:Infinity}).on("line", line => { const message = JSON.parse(line); - if (message.method === "initialize") send(message.id, {ok:true,supportedMethods:["environmentTask","configChanged","health"]}); - else if (message.method === "environmentTask" || message.method === "environmentDestroyLease") { + if (message.method === "initialize") send(message.id, {ok:true,supportedMethods:["environmentTask","environmentAcquireLease","configChanged","health"]}); + else if (message.method === "environmentTask" || message.method === "environmentDestroyLease" || message.method === "environmentAcquireLease") { held.push(message.id); invocations.push(message.paperclipInvocation?.id); if (message.params.config.hostWrite) process.stdout.write(JSON.stringify({jsonrpc:"2.0",id:"host-"+message.id,method:"companies.get",params:{companyId:"company"},paperclipInvocationId:message.paperclipInvocation.id})+"\\n"); @@ -43,7 +43,7 @@ readline.createInterface({input:process.stdin,crlfDelay:Infinity}).on("line", li else if (message.method === "configChanged") { if (message.params.config.exit) process.exit(1); if (message.params.config.hostAfter) process.stdout.write(JSON.stringify({jsonrpc:"2.0",id:"late-host",method:"companies.get",params:{companyId:"company"},paperclipInvocationId:invocations.at(-1)})+"\\n"); - if (message.params.config.release) for (const id of held.splice(0)) send(id, {kind:"description",taskId:"task"}); + if (message.params.config.release) for (const id of held.splice(0)) { send(id, {providerLeaseId:"late-task",kind:"description",taskId:"task"}); if (message.params.config.duplicate) send(id, {providerLeaseId:"late-task",kind:"description",taskId:"task"}); } send(message.id, {}); } else if (message.method === "shutdown") {send(message.id, {});setImmediate(()=>process.exit(0));} }); @@ -59,6 +59,70 @@ async function startWorker(hostHandlers: Record = {}) { } describe("Core startup responsibility across real worker IPC", () => { + it("drops only a dead acquisition reply placeholder while retaining real writes until verified retirement", async () => { + let release!: () => void, writing!: () => void; + const gate = new Promise(resolve => { release = resolve; }), entered = new Promise(resolve => { writing = resolve; }); + const manager = await startWorker({ "companies.get": async () => { writing(); await gate; return {}; } }); + const settled = vi.fn(), observe = vi.fn(async () => {}); + try { + const acquisition = manager.call(manifest.id, "environmentAcquireLease", { companyId: "company", environmentId: "environment", + driverKey: "tasks", config: { hostWrite: true }, runId: "run", agentId: "agent", adapterType: "paperclip_runner" }, + 10, undefined, settled, "dead-acquire", observe); + const rejected = expect(acquisition).rejects.toThrow("timed out"); + await entered; await rejected; + await manager.stopWorker(manifest.id); + let drained = false; + const draining = manager.drainStartup!("dead-acquire").then(() => { drained = true; }); + await new Promise(resolve => setImmediate(resolve)); + expect(drained).toBe(false); + expect(settled).not.toHaveBeenCalled(); + release(); await draining; + expect(observe).not.toHaveBeenCalled(); + expect(settled).not.toHaveBeenCalled(); + await manager.retireStartup!("dead-acquire"); + expect(settled).toHaveBeenCalledOnce(); + } finally { release(); await manager.stopAll(); } + }, 25_000); + + it.each([false, true])("retains a received acquisition reply through failed recording and retirement (worker exits=%s)", async workerExits => { + const manager = await startWorker(), settled = vi.fn(); + let release!: () => void, recording!: () => void; + const gate = new Promise(resolve => { release = resolve; }), started = new Promise(resolve => { recording = resolve; }); + let unavailable = true; + const observe = vi.fn(async (result: unknown) => { + expect(result).toMatchObject({ providerLeaseId: "late-task" }); + if (unavailable) throw new Error("Database unavailable"); + recording(); await gate; + }); + try { + const acquire = manager.call(manifest.id, "environmentAcquireLease", { companyId: "company", environmentId: "environment", + driverKey: "tasks", config: {}, runId: "run", agentId: "agent", adapterType: "paperclip_runner" }, 10, undefined, settled, "late-acquire", observe); + await expect(acquire).rejects.toThrow("timed out"); + await manager.call(manifest.id, "configChanged", { config: { release: true, duplicate: true } }); + await vi.waitFor(() => expect(observe).toHaveBeenCalledOnce()); + await new Promise(resolve => setImmediate(resolve)); + expect(settled).not.toHaveBeenCalled(); + if (workerExits) await manager.stopWorker(manifest.id); + unavailable = false; + vi.useFakeTimers(); + const failed = manager.retireStartup!("late-acquire").catch(error => error.message); + await started; + expect(observe).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(15_000); + expect(await failed).toBe("Task cleanup is still pending"); + expect(settled).not.toHaveBeenCalled(); + vi.useRealTimers(); + let retired = false; + const retiring = manager.retireStartup!("late-acquire").then(() => { retired = true; }); + await new Promise(resolve => setImmediate(resolve)); + expect(retired).toBe(false); + release(); await retiring; + expect(settled).toHaveBeenCalledOnce(); + await manager.retireStartup!("late-acquire"); + expect(observe).toHaveBeenCalledTimes(2); + } finally { vi.useRealTimers(); release(); await manager.stopAll(); } + }); + it.each(["stop", "replace", "stopAll"])("keeps stopped startup work reachable through %s until verified retirement", async mode => { let release!: () => void, entered!: () => void; const gate = new Promise(resolve => { release = resolve; }), started = new Promise(resolve => { entered = resolve; }); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts index 1c969758ac7..d698855a97f 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.test.ts @@ -4,7 +4,7 @@ import { join } from "node:path"; import { expect, it } from "vitest"; import { removeNativeHarnessBackup } from "./native-harness-backup-cleanup.js"; -it("removes retired backups containing immutable skills without modifying live files or symlink targets", () => { +it("removes retired backups containing immutable skills without modifying live files or symlink targets", async () => { const base = mkdtempSync(join(tmpdir(), "paperclip-backup-cleanup-")); const retired = join(base, "previous"); const nested = join(retired, "claude", "skills", "first-task"); @@ -19,12 +19,12 @@ it("removes retired backups containing immutable skills without modifying live f chmodSync(retired, 0o555); chmodSync(live, 0o555); try { - removeNativeHarnessBackup(retired); + await removeNativeHarnessBackup(retired); expect(existsSync(retired)).toBe(false); expect(readFileSync(join(live, "SKILL.md"), "utf8")).toBe("live instructions"); expect(lstatSync(live).mode & 0o777).toBe(0o555); expect(lstatSync(join(live, "SKILL.md")).mode & 0o777).toBe(0o444); - expect(() => removeNativeHarnessBackup(retired)).not.toThrow(); + await expect(removeNativeHarnessBackup(retired)).resolves.toBeUndefined(); } finally { for (const directory of [retired, join(retired, "claude", "skills"), nested, live]) { if (existsSync(directory)) chmodSync(directory, 0o700); diff --git a/server/src/services/native-runtime/native-harness-backup-cleanup.ts b/server/src/services/native-runtime/native-harness-backup-cleanup.ts index d52e50ba5c4..7a803704ea7 100644 --- a/server/src/services/native-runtime/native-harness-backup-cleanup.ts +++ b/server/src/services/native-runtime/native-harness-backup-cleanup.ts @@ -1,18 +1,23 @@ -import { chmodSync, lstatSync, readdirSync, rmSync } from "node:fs"; +import { chmod, lstat, readdir, rm } from "node:fs/promises"; import { join } from "node:path"; /** Remove a retired controller-owned backup, never a live harness directory. */ -export function removeNativeHarnessBackup(root: string): void { +export async function removeNativeHarnessBackup(root: string): Promise { // Archives preserve immutable skill directory modes. Deleting their children // requires write permission on those directories, even with rm's force flag. // Only change directories in the backup being discarded. lstat deliberately // avoids following provider-created symlinks into live or unrelated trees. - const prepareDirectory = (path: string): void => { - const metadata = lstatSync(path, { throwIfNoEntry: false }); + const prepareDirectory = async (path: string): Promise => { + const metadata = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); if (!metadata?.isDirectory()) return; - chmodSync(path, (metadata.mode & 0o777) | 0o700); - for (const name of readdirSync(path)) prepareDirectory(join(path, name)); + await chmod(path, (metadata.mode & 0o777) | 0o700); + for (const entry of await readdir(path, { withFileTypes: true })) { + if (entry.isDirectory()) await prepareDirectory(join(path, entry.name)); + } }; - prepareDirectory(root); - rmSync(root, { recursive: true, force: true }); + await prepareDirectory(root); + await rm(root, { recursive: true, force: true }); } diff --git a/server/src/services/native-runtime/native-harness-backup-publication.test.ts b/server/src/services/native-runtime/native-harness-backup-publication.test.ts index dd9d940d9df..df1029f55f9 100644 --- a/server/src/services/native-runtime/native-harness-backup-publication.test.ts +++ b/server/src/services/native-runtime/native-harness-backup-publication.test.ts @@ -1,4 +1,6 @@ -import { promises as fs } from "node:fs"; +import { existsSync, promises as fs } from "node:fs"; +import { spawn } from "node:child_process"; +import { createServer, type ServerResponse } from "node:http"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -32,7 +34,17 @@ describe("native harness backup publication", () => { }); it("publishes the completed replacement once and removes its predecessor", async () => { const input = await fixture(); const changes: unknown[] = []; - unsubscribe = onNativeHarnessBackupChange(input.db, change => changes.push(change)); + const previous = path.join(backupRoot, "previous"); + for (const root of [input.current, previous]) { + const skill = path.join(root, "skills", "immutable"); + await fs.mkdir(skill, { recursive: true }); + await fs.writeFile(path.join(skill, "SKILL.md"), "saved instructions", { mode: 0o444 }); + await fs.chmod(skill, 0o555); + } + unsubscribe = onNativeHarnessBackupChange(input.db, change => { + expect(existsSync(previous)).toBe(false); + changes.push(change); + }); await publishNativeHarnessBackup({ ...input, stamp: async current => { expect(changes).toEqual([]); expect(await fs.readFile(path.join(current, "manifest.json"), "utf8")).toBe("replacement"); @@ -41,4 +53,85 @@ describe("native harness backup publication", () => { expect(await fs.readdir(backupRoot)).toEqual(["current"]); expect(changes).toEqual([{ companyId: input.companyId, scopeDigest: input.scopeDigest }]); }); + it("serves an unrelated request while a retired backup is being removed", async () => { + const input = await fixture(); + for (let index = 0; index < 40; index++) { + const directory = path.join(input.current, `skill-${index}`); + await fs.mkdir(directory); + await Promise.all(Array.from({ length: 50 }, (_, file) => + fs.writeFile(path.join(directory, `file-${file}`), "retired instructions", { mode: 0o444 }))); + await fs.chmod(directory, 0o555); + } + let publishing = false; + let probeReceived: (response: ServerResponse | null) => void; + const probeReady = new Promise(resolve => { probeReceived = resolve; }); + const server = createServer((request, response) => { + if (request.url === "/probe") { probeReceived(response); return; } + response.end("ok"); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("HTTP probe did not start"); + const client = spawn(process.execPath, ["--input-type=module", "-e", ` + import { Agent, get } from "node:http"; + const agent = new Agent({ keepAlive: true }); + const request = path => new Promise((resolve, reject) => { + get({ hostname: "127.0.0.1", port: Number(process.argv[1]), path, agent }, response => { + response.resume(); + response.on("end", () => resolve(response.headers["x-publication-active"])); + }).on("error", reject); + }); + await request("/"); + process.send({ kind: "ready" }); + process.once("message", async () => { + const active = await request("/probe"); + process.send({ kind: "result", active }, () => { agent.destroy(); process.disconnect(); }); + }); + `, String(address.port)], { stdio: ["ignore", "ignore", "inherit", "ipc"] }); + const message = (kind: string) => new Promise<{ active?: string }>((resolve, reject) => { + const timer = setTimeout(() => { cleanup(); reject(new Error(`HTTP probe ${kind} timed out`)); }, 10_000); + const received = (value: { kind?: string; active?: string }) => { + if (value.kind === kind) { cleanup(); resolve(value); } + }; + const failed = () => { cleanup(); reject(new Error(`HTTP probe exited before ${kind}`)); }; + const cleanup = () => { + clearTimeout(timer); client.off("message", received); client.off("exit", failed); client.off("error", failed); + }; + client.on("message", received); client.once("exit", failed); client.once("error", failed); + }); + let publication: Promise | undefined; + try { + await message("ready"); + const response = message("result"); + publishing = true; + publication = publishNativeHarnessBackup({ + ...input, + stamp: async () => { + client.send({ kind: "probe" }); + const response = await probeReady; + if (!response) return; + setImmediate(() => { + response.setHeader("x-publication-active", publishing ? "1" : "0"); + response.end("ok"); + }); + }, + }).finally(() => { publishing = false; }); + const result = await response; + await publication; + expect(result.active).toBe("1"); + expect(await fs.readdir(backupRoot)).toEqual(["current"]); + expect(await fs.readFile(path.join(input.current, "manifest.json"), "utf8")).toBe("replacement"); + } finally { + probeReceived!(null); + try { await publication; } + finally { + client.kill(); + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + } + } + }, 15_000); }); diff --git a/server/src/services/native-runtime/native-harness-backup-publication.ts b/server/src/services/native-runtime/native-harness-backup-publication.ts index 820303584b0..714dafb5f20 100644 --- a/server/src/services/native-runtime/native-harness-backup-publication.ts +++ b/server/src/services/native-runtime/native-harness-backup-publication.ts @@ -9,7 +9,7 @@ export async function publishNativeHarnessBackup(input: { }) { const currentRoot = resolve(input.backupRoot, "current"); const previousRoot = resolve(input.backupRoot, "previous"); - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); let movedCurrent = false; if (existsSync(currentRoot)) { renameSync(currentRoot, previousRoot); @@ -19,10 +19,10 @@ export async function publishNativeHarnessBackup(input: { renameSync(input.pendingRoot, currentRoot); await input.stamp?.(currentRoot); } catch (error) { - if (existsSync(currentRoot)) removeNativeHarnessBackup(currentRoot); + if (existsSync(currentRoot)) await removeNativeHarnessBackup(currentRoot); if (movedCurrent && existsSync(previousRoot) && !existsSync(currentRoot)) renameSync(previousRoot, currentRoot); throw error; } - removeNativeHarnessBackup(previousRoot); + await removeNativeHarnessBackup(previousRoot); noteNativeHarnessBackupChange(input.db, { companyId: input.companyId, scopeDigest: input.scopeDigest }); } diff --git a/server/src/services/native-runtime/native-response-presentation-query.ts b/server/src/services/native-runtime/native-response-presentation-query.ts index f3a476a287a..d181e3dc9c2 100644 --- a/server/src/services/native-runtime/native-response-presentation-query.ts +++ b/server/src/services/native-runtime/native-response-presentation-query.ts @@ -48,8 +48,13 @@ export function nativeResponsePresentationCandidates(scope?: NativeResponsePrese sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,channel}' = 'final'`, sql`${heartbeatRunEvents.payload} #>> '{prpEvent,payload,text}' ~ '[^[:space:]]'`, ))))), - sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`, - sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`), + or( + and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'done'`, + sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'done'`), + and(sql`(coalesce(${issues.conversationAgentId}::text, '') = '' or coalesce(${issues.conversationUserId}, '') = '')`, + sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'needs_review'`, + sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'needs_review'`), + )), and(sql`${nativeRunResults.resultJson} #>> '{result,reportedWorkDisposition}' = 'yielded'`, sql`${nativeRunResults.resultJson} #>> '{terminal,reportedWorkDisposition}' = 'yielded'`, sql`jsonb_typeof(${nativeRunResults.resultJson} #> '{result,summary}') = 'string' diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index a27af96ac3b..635808df5ef 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -38,6 +38,7 @@ import { isNewTerminalRunFailure, reportRunFailure } from "../run-failure-report import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; import { nativeRestartInterruptedTurnId, type PersistedNativeSession } from "../../vendor/paperclip-runner/index.js"; import { retireSupersededNativeRetry } from "./native-superseded-retry.js"; +import { requestNativeStartupCleanup, readNativeStartupContext, promoteNativeStartupCleanup } from "./native-startup-cleanup.js"; export type NativeControllerIdentity = { bootId: string; @@ -623,22 +624,41 @@ export async function claimNativeRestartRecoveries(input: { } as const; } + const startupContext = readNativeStartupContext(row.run, row.coordinator); + const startupRetryAt = startupContext ? new Date(Math.max(now.getTime() + 30_000, + startupContext.deadlineAtMs, row.coordinator.leaseExpiresAt?.getTime() ?? 0)) : undefined; if (input.liveRunExecutions?.has(row.run.id)) { return { kind: "awaiting_evidence", runId: row.run.id, reason: "current_controller_preparing", + ...(startupRetryAt ? { retryAt: startupRetryAt } : {}), } as const; } + if (startupContext) { + if (await promoteNativeStartupCleanup(tx as unknown as Db, row.run, row.coordinator, now)) { + return { kind: "blocked", runId: row.run.id, reason: "native_startup_cleanup_pending" } as const; + } + return { kind: "awaiting_evidence", runId: row.run.id, reason: "native_startup_cleanup_pending", retryAt: startupRetryAt } as const; + } const profile = row.run.runnerProfileJson ?? {}; const storedRemoteWorkspace = remoteWorkspaceRunReferenceSchema.safeParse(profile.remoteWorkspace); const remoteWorkspace = readNativeWorkspaceSyncReference(profile.nativeWorkspaceSync) ?? (storedRemoteWorkspace.success ? storedRemoteWorkspace.data : null); - const [pluginTask] = await tx.select({ id: environmentLeases.id }).from(environmentLeases).where(and( + const [pluginTask] = await tx.select({ id: environmentLeases.id, metadata: environmentLeases.metadata }).from(environmentLeases).where(and( eq(environmentLeases.companyId, row.run.companyId), eq(environmentLeases.heartbeatRunId, row.run.id), sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`, )).orderBy(desc(environmentLeases.createdAt)).limit(1); + const startup = pluginTask?.metadata?.runnerTaskStartup as { contextId?: unknown; deadlineAtMs?: unknown; runId?: unknown } | undefined; + if (startup?.runId === row.run.id && typeof startup.contextId === "string" && + Number.isSafeInteger(startup.deadlineAtMs) && Number(startup.deadlineAtMs) <= now.getTime() && + !row.run.sessionIdAfter && row.coordinator.leaseOwner && row.coordinator.leaseExpiresAt && row.coordinator.leaseExpiresAt <= now) { + await requestNativeStartupCleanup(tx as unknown as Db, { companyId: row.run.companyId, agentId: row.run.agentId, + runId: row.run.id, contextId: startup.contextId, deadlineAtMs: Number(startup.deadlineAtMs), + leaseOwner: row.coordinator.leaseOwner, requiresPreparationStop: true, leaseId: pluginTask!.id }, { publish: false }); + return { kind: "blocked", runId: row.run.id, reason: "native_startup_cleanup_pending" } as const; + } const remoteSandbox = profile.nativeWorkspaceSync != null || profile.remoteWorkspace != null; const remoteProcess = remoteSandbox || !!pluginTask; const durableAuthority = (remoteWorkspace || pluginTask) && await hasNativeRunnerCheckpoint(tx, { diff --git a/server/src/services/native-runtime/native-run-finalizer.ts b/server/src/services/native-runtime/native-run-finalizer.ts index d39a3b682bb..50b626d9adb 100644 --- a/server/src/services/native-runtime/native-run-finalizer.ts +++ b/server/src/services/native-runtime/native-run-finalizer.ts @@ -921,15 +921,17 @@ async function recoverCommittedNativeChatResponseInScope( && run.contextSnapshot?.issueId === issue.id && typeof run.contextSnapshot.conversationSessionGeneration === "number" && run.contextSnapshot.conversationSessionGeneration === issue.conversationSessionGeneration; + // A deferred ordinary-task review still needs its recorded final reply. + // Publishing that reply must not resolve the interaction or the task. if ( (!conversation || currentConversation) && run.status === "succeeded" && record(run.contextSnapshot).skipIssueComment !== true && !isExternalChatPresentationContext(run.contextSnapshot) && - result.reportedWorkDisposition === "done" && terminal.runTerminalState === "succeeded" && terminal.turnTerminalState === "completed" && - terminal.reportedWorkDisposition === "done" && + ((result.reportedWorkDisposition === "done" && terminal.reportedWorkDisposition === "done") || + (!conversation && result.reportedWorkDisposition === "needs_review" && terminal.reportedWorkDisposition === "needs_review")) && (await resolveChatRunPresentationAuthorizationReason(tx, input)) === "internal_agent_write" ) { const rows = await tx.select({ seq: heartbeatRunEvents.seq, payload: heartbeatRunEvents.payload }) diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f3565441c39..93992b4870d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -13734,7 +13734,7 @@ async function createRunnerdBackendWithinSessionClaim( } : undefined, }); } finally { - removeNativeHarnessBackup(pendingRoot); + await removeNativeHarnessBackup(pendingRoot); } }, { diff --git a/server/src/services/native-runtime/native-startup-cleanup.ts b/server/src/services/native-runtime/native-startup-cleanup.ts new file mode 100644 index 00000000000..e6b80230c2d --- /dev/null +++ b/server/src/services/native-runtime/native-startup-cleanup.ts @@ -0,0 +1,208 @@ +import { and, eq, isNull, sql } from "drizzle-orm"; +import { environmentLeases, heartbeatRuns, issues, nativeRunFinalizations, sandboxCreationOperations, type Db } from "@paperclipai/db"; +import { conflict } from "../../errors.js"; +import { persistActivity, publishActivity } from "../activity-log.js"; +import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; +import { hasRemoteTerminationReceipt } from "../remote-execution-termination.js"; +import { isNativeRunnerOwnershipHeld } from "./native-runner-ownership.js"; + +export const NATIVE_STARTUP_CLEANUP_PENDING = "native_startup_cleanup_pending"; + +export class NativeStartupCleanupPendingError extends Error { + constructor(readonly contextId: string, readonly deadlineAtMs: number, readonly leaseOwner: string, + readonly requiresPreparationStop: boolean, readonly leaseId?: string) { + super(NATIVE_STARTUP_CLEANUP_PENDING); + this.name = "NativeStartupCleanupPendingError"; + } +} + +type Run = typeof heartbeatRuns.$inferSelect; +export interface NativeStartupCleanup { + schema: "paperclip.native-startup-cleanup.v1"; + companyId: string; agentId: string; runId: string; + contextId: string; deadlineAtMs: number; leaseOwner: string; + controllerGeneration: number; attempt: number; leaseId: string; taskId: string; + requiresPreparationStop: boolean; state: "pending" | "complete"; auditId: string; +} + +export function readNativeStartupCleanup(run: Pick): NativeStartupCleanup | null { + const marker = run.resultJson?.nativeStartupCleanup as NativeStartupCleanup | undefined; + return marker?.schema === "paperclip.native-startup-cleanup.v1" && marker.companyId === run.companyId && + marker.runId === run.id && marker.agentId === run.agentId && ["contextId", "leaseOwner", "leaseId", "taskId", "auditId"].every(key => + typeof marker[key as keyof NativeStartupCleanup] === "string" && String(marker[key as keyof NativeStartupCleanup]).length > 0) && + Number.isSafeInteger(marker.deadlineAtMs) && Number.isSafeInteger(marker.controllerGeneration) && marker.controllerGeneration > 0 && + Number.isSafeInteger(marker.attempt) && marker.attempt > 0 && typeof marker.requiresPreparationStop === "boolean" && + ["pending", "complete"].includes(marker.state) ? marker : null; +} + +export function readNativeStartupContext(run: Run, + owner: { leaseOwner: string | null; controllerGeneration: number; attempt: number; resultId: string | null }) { + const value = run.runnerProfileJson?.nativeStartupContext as { schema?: unknown; companyId?: unknown; agentId?: unknown; runId?: unknown; + contextId?: unknown; deadlineAtMs?: unknown; leaseId?: unknown; leaseOwner?: unknown; controllerGeneration?: unknown; attempt?: unknown } | undefined; + if (run.runtimeMode !== "native" || run.status !== "running" || run.sessionIdAfter || owner.resultId || isNativeRunnerOwnershipHeld(run) || + !owner.leaseOwner || owner.controllerGeneration < 1 || owner.attempt < 1 || value?.schema !== "paperclip.native-startup-context.v1" || + value.companyId !== run.companyId || value.agentId !== run.agentId || value.runId !== run.id || value.leaseOwner !== owner.leaseOwner || + value.controllerGeneration !== owner.controllerGeneration || value.attempt !== owner.attempt || + typeof value.contextId !== "string" || !value.contextId || typeof value.leaseId !== "string" || !value.leaseId || + !Number.isSafeInteger(value.deadlineAtMs)) return null; + return { contextId: value.contextId, leaseId: value.leaseId, deadlineAtMs: Number(value.deadlineAtMs) }; +} + +export async function clearNativeStartupContext(db: Db, input: { + companyId: string; runId: string; contextId: string; leaseOwner: string; controllerGeneration: number; +}): Promise { + const cleared = await db.update(heartbeatRuns).set({ + runnerProfileJson: sql`${heartbeatRuns.runnerProfileJson} - 'nativeStartupContext'`, + }).where(and(eq(heartbeatRuns.companyId, input.companyId), eq(heartbeatRuns.id, input.runId), + eq(heartbeatRuns.status, "running"), + sql`coalesce(${heartbeatRuns.resultJson}->'nativeStartupCleanup'->>'state', '') != 'pending'`, + sql`${heartbeatRuns.runnerProfileJson}->'nativeStartupContext'->>'contextId' = ${input.contextId}`, + sql`${heartbeatRuns.runnerProfileJson}->'nativeStartupContext'->>'leaseOwner' = ${input.leaseOwner}`, + sql`${heartbeatRuns.runnerProfileJson}->'nativeStartupContext'->>'controllerGeneration' = ${String(input.controllerGeneration)}`, + sql`exists (select 1 from ${nativeRunFinalizations} where ${nativeRunFinalizations.runId} = ${input.runId} + and ${nativeRunFinalizations.leaseOwner} = ${input.leaseOwner} + and ${nativeRunFinalizations.controllerGeneration} = ${input.controllerGeneration})`, + )).returning({ id: heartbeatRuns.id }); + if (cleared.length !== 1) throw new Error("Native startup fallback lost authority"); +} + +export async function promoteNativeStartupCleanup(db: Db, run: Run, + owner: { leaseOwner: string | null; leaseExpiresAt: Date | null; controllerGeneration: number; attempt: number; resultId: string | null }, + now: Date): Promise { + const startup = readNativeStartupContext(run, owner); + if (!startup || startup.deadlineAtMs > now.getTime() || !owner.leaseOwner || !owner.leaseExpiresAt || owner.leaseExpiresAt > now || isNativeRunnerOwnershipHeld(run)) return false; + const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.id, startup.leaseId), + eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.heartbeatRunId, run.id), + sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`)); + if (!lease) return false; + const prepared = lease.metadata?.runnerTaskStartup as { contextId?: unknown; runId?: unknown } | undefined; + return requestNativeStartupCleanup(db, { companyId: run.companyId, agentId: run.agentId, runId: run.id, + ...startup, leaseOwner: owner.leaseOwner, + requiresPreparationStop: prepared?.contextId === startup.contextId && prepared.runId === run.id }, { publish: false }); +} + +export async function requestNativeStartupCleanup(db: Db, input: { + companyId: string; agentId: string; runId: string; contextId: string; deadlineAtMs: number; leaseOwner: string; + requiresPreparationStop: boolean; + leaseId?: string; +}, options: { publish?: boolean } = {}): Promise { + let publication: Parameters[0] | null = null; + const [binding] = await db.select({ issueId: nativeRunFinalizations.issueId }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId))); + if (!binding) return false; + const requested = await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, binding.issueId))).for("update"); + const [coordinator] = await tx.select().from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId))).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, input.companyId), + eq(heartbeatRuns.id, input.runId), eq(heartbeatRuns.agentId, input.agentId))).for("update"); + if (!run || !coordinator || run.runtimeMode !== "native" || run.status !== "running" || run.sessionIdAfter || coordinator.resultId || + isNativeRunnerOwnershipHeld(run)) return false; + const existing = readNativeStartupCleanup(run); + if (existing) return existing.contextId === input.contextId; + if (coordinator.leaseOwner !== input.leaseOwner) return false; + const leases = await tx.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, input.companyId), + eq(environmentLeases.heartbeatRunId, input.runId), input.leaseId ? eq(environmentLeases.id, input.leaseId) : undefined, + sql`${environmentLeases.metadata}->>'runnerTaskLease' = 'true'`)).for("update"); + if (leases.length !== 1 || !leases[0]!.providerLeaseId) throw new Error("Native startup cleanup lease unavailable"); + const lease = leases[0]!; + const activity = await persistActivity(tx as unknown as Db, { companyId: input.companyId, actorType: "system", + actorId: "native-startup-cleanup", action: "native.startup_cleanup_requested", entityType: "heartbeat_run", entityId: run.id, + agentId: run.agentId, runId: run.id, issueId: run.nativeIssueId ?? undefined, + details: { contextId: input.contextId, deadlineAtMs: input.deadlineAtMs, leaseId: lease.id, + controllerGeneration: coordinator.controllerGeneration, reason: "startup_worker_unsettled" } }); + if (!activity.activity?.id) throw new Error("Native startup cleanup audit unavailable"); + publication = activity.publication; + const preparation = lease.metadata?.runnerTaskStartup as { contextId?: unknown; runId?: unknown } | undefined; + const marker: NativeStartupCleanup = { schema: "paperclip.native-startup-cleanup.v1", companyId: run.companyId, agentId: run.agentId, + runId: run.id, contextId: input.contextId, deadlineAtMs: input.deadlineAtMs, leaseOwner: input.leaseOwner, + controllerGeneration: coordinator.controllerGeneration, attempt: coordinator.attempt, leaseId: lease.id, + taskId: lease.providerLeaseId!, requiresPreparationStop: input.requiresPreparationStop || + (preparation?.contextId === input.contextId && preparation.runId === run.id), state: "pending", auditId: activity.activity.id }; + await notifyDeliveryWork(tx, DELIVERY_QUEUES.executionFinalization); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.environmentCleanup); + await tx.update(heartbeatRuns).set({ errorCode: NATIVE_STARTUP_CLEANUP_PENDING, + error: "Native startup timed out. Cancellation and verified cleanup are pending.", + resultJson: sql`coalesce(${heartbeatRuns.resultJson}, '{}'::jsonb) || ${JSON.stringify({ nativeStartupCleanup: marker, + startupCancellation: { ...run.resultJson?.startupCancellation as object, requestedAt: new Date().toISOString() } })}::jsonb`, + updatedAt: new Date() }).where(eq(heartbeatRuns.id, run.id)); + await tx.update(nativeRunFinalizations).set({ phase: "terminal_failure", failureCode: NATIVE_STARTUP_CLEANUP_PENDING, + nextAttemptAt: null, controlDeadlineAt: new Date(), recoveryState: "blocked", updatedAt: new Date() }) + .where(eq(nativeRunFinalizations.runId, run.id)); + await tx.update(environmentLeases).set({ metadata: sql`coalesce(${environmentLeases.metadata}, '{}'::jsonb) || ${JSON.stringify({ + runnerTaskRetiring: true, runnerTaskStartupCleanup: { contextId: marker.contextId, requiresPreparationStop: marker.requiresPreparationStop }, + })}::jsonb`, updatedAt: new Date() }).where(eq(environmentLeases.id, lease.id)); + return true; + }); + if (publication && options.publish !== false) publishActivity(publication); + return requested; +} + +async function assertStartupCreationsStopped(db: Db, marker: NativeStartupCleanup): Promise { + const creations = await db.select({ creation: sandboxCreationOperations, lease: environmentLeases }).from(sandboxCreationOperations) + .leftJoin(environmentLeases, eq(environmentLeases.id, sandboxCreationOperations.leaseId)).where(and( + eq(sandboxCreationOperations.companyId, marker.companyId), eq(sandboxCreationOperations.runId, marker.runId), + sql`${sandboxCreationOperations.metadata}->>'runnerTaskLease' = 'true'`)); + if (creations.some(({ creation, lease }) => creation.state !== "attached" && + (!["cleanup_pending", "resolved"].includes(creation.state) || !lease || lease.companyId !== marker.companyId || lease.heartbeatRunId !== marker.runId || + lease.providerLeaseId !== creation.metadata.creationProviderLeaseId || lease.cleanupStatus !== "success" || !hasRemoteTerminationReceipt(lease)))) { + throw conflict("Task cleanup is still pending", { code: NATIVE_STARTUP_CLEANUP_PENDING }); + } +} + +export async function verifyNativeStartupCleanup(db: Db, run: Run, release: () => Promise): Promise { + const marker = readNativeStartupCleanup(run); + if (!marker || isNativeRunnerOwnershipHeld(run) || !["running", "cancelled"].includes(run.status)) throw conflict("Native startup cleanup intent unavailable"); + const read = async () => { + const [lease] = await db.select().from(environmentLeases).where(and(eq(environmentLeases.companyId, run.companyId), eq(environmentLeases.id, marker.leaseId))); + if (!lease || lease.heartbeatRunId !== run.id || lease.providerLeaseId !== marker.taskId) throw conflict("Native startup cleanup target changed"); + return lease; + }; + const [owner] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)); + if (!owner || owner.controllerGeneration !== marker.controllerGeneration || owner.attempt !== marker.attempt || + owner.leaseOwner !== marker.leaseOwner || owner.resultId) throw conflict("Native startup cleanup ownership changed"); + let lease = await read(); + const stopped = (value: typeof lease) => hasRemoteTerminationReceipt(value) && value.cleanupStatus === "success" && + (!marker.requiresPreparationStop || value.metadata?.runnerTaskPreparationStopped === true); + if (!stopped(lease)) { await release(); lease = await read(); } + if (!stopped(lease)) throw conflict("Task cleanup is still pending", { code: NATIVE_STARTUP_CLEANUP_PENDING }); + await assertStartupCreationsStopped(db, marker); + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, run.companyId), eq(issues.id, run.nativeIssueId!))).for("update"); + const [coordinator] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, run.id)).for("update"); + const [current] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)).for("update"); + const observed = current && readNativeStartupCleanup(current); + const [receipt] = await tx.select().from(environmentLeases).where(eq(environmentLeases.id, marker.leaseId)).for("update"); + if (!coordinator || !current || isNativeRunnerOwnershipHeld(current) || coordinator.resultId || coordinator.controllerGeneration !== marker.controllerGeneration || + coordinator.attempt !== marker.attempt || coordinator.leaseOwner !== marker.leaseOwner || observed?.contextId !== marker.contextId || + !receipt || receipt.providerLeaseId !== marker.taskId || receipt.heartbeatRunId !== run.id || !stopped(receipt)) { + throw conflict("Native startup cleanup ownership changed"); + } + await assertStartupCreationsStopped(tx as unknown as Db, marker); + await tx.update(heartbeatRuns).set({ resultJson: sql`jsonb_set(${heartbeatRuns.resultJson}, '{nativeStartupCleanup,state}', '"complete"'::jsonb)`, updatedAt: new Date() }) + .where(eq(heartbeatRuns.id, run.id)); + }); +} + +export async function acknowledgeNativeStartupCleanup(db: Db, input: { companyId: string; runId: string }): Promise { + const [binding] = await db.select({ issueId: nativeRunFinalizations.issueId }).from(nativeRunFinalizations).where(and( + eq(nativeRunFinalizations.companyId, input.companyId), eq(nativeRunFinalizations.runId, input.runId))); + if (!binding) return; + await db.transaction(async tx => { + await tx.select({ id: issues.id }).from(issues).where(and(eq(issues.companyId, input.companyId), eq(issues.id, binding.issueId))).for("update"); + const [owner] = await tx.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, input.runId)).for("update"); + const [run] = await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId)).for("update"); + const marker = run && readNativeStartupCleanup(run); + if (!owner || !run || run.status !== "cancelled" || !marker || marker.state !== "complete" || owner.resultId || + owner.failureCode !== NATIVE_STARTUP_CLEANUP_PENDING || owner.controllerGeneration !== marker.controllerGeneration || + owner.attempt !== marker.attempt || owner.leaseOwner !== marker.leaseOwner) return; + const [lease] = await tx.select().from(environmentLeases).where(eq(environmentLeases.id, marker.leaseId)).for("update"); + if (!lease || lease.heartbeatRunId !== run.id || lease.providerLeaseId !== marker.taskId || !hasRemoteTerminationReceipt(lease) || + (marker.requiresPreparationStop && lease.metadata?.runnerTaskPreparationStopped !== true)) return; + await assertStartupCreationsStopped(tx as unknown as Db, marker); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.agentLifecycle); + await notifyDeliveryWork(tx, DELIVERY_QUEUES.environmentCleanup); + await tx.update(nativeRunFinalizations).set({ leaseOwner: null, leaseExpiresAt: null, controlDeadlineAt: null, recoveryState: null, + failureCode: "native_retry_cancelled", updatedAt: new Date() }).where(and(eq(nativeRunFinalizations.runId, input.runId), isNull(nativeRunFinalizations.resultId))); + }); +} diff --git a/server/src/services/native-runtime/native-workspace-reconstruction.ts b/server/src/services/native-runtime/native-workspace-reconstruction.ts index 2daf462972f..aa20b852e14 100644 --- a/server/src/services/native-runtime/native-workspace-reconstruction.ts +++ b/server/src/services/native-runtime/native-workspace-reconstruction.ts @@ -36,7 +36,8 @@ export function unreconciledNativeExportPredicate() { and ${heartbeatRuns.nativeIssueId} = ${issueRecoveryActions.sourceIssueId} and ${heartbeatRuns.id}::text = ${issueRecoveryActions.evidence}->>'runId' and ${heartbeatRuns.runtimeMode} = 'native' and ${heartbeatRuns.status} = 'failed' - and ${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared')`, + and (${heartbeatRuns.runnerProfileJson}->'nativeWorkspaceSync'->>'state' = 'prepared' + or ${heartbeatRuns.runnerProfileJson}->'remoteWorkspace'->>'state' = 'prepared'))`, ); } diff --git a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts index bf4f53dd371..5fab72c4c2c 100644 --- a/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts +++ b/server/src/services/native-runtime/paperclip-runner-tool-authority.test.ts @@ -19,7 +19,7 @@ import { issueThreadInteractions, issues, } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { startRecoveryPostgres } from "../../__tests__/helpers/recovery-postgres.js"; import { initializeRunIdentity, reserveSteeredIdentity, reconcileSteeredIdentity } from "../run-identity.js"; import { documentService } from "../documents.js"; @@ -248,7 +248,7 @@ describe("PaperclipRunnerToolAuthority", () => { }).where(eq(heartbeatRuns.id, runId)); const [expanded] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const observed = createDb(temporary!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues, agents, companies }, capture); try { const reader = new PaperclipRunnerToolAuthority(observed, binding); @@ -291,7 +291,7 @@ describe("PaperclipRunnerToolAuthority", () => { await db.update(heartbeatRuns).set({ contextSnapshot: context }).where(eq(heartbeatRuns.id, runId)); const [expanded] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId)); const observed = createDb(temporary!.connectionString), events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(observed, { heartbeatRuns, issues, agents, companies }, capture); try { const reader = new PaperclipRunnerToolAuthority(observed, binding); diff --git a/server/src/services/plugin-host-services.ts b/server/src/services/plugin-host-services.ts index 2a9a845d01a..eeff010abbe 100644 --- a/server/src/services/plugin-host-services.ts +++ b/server/src/services/plugin-host-services.ts @@ -537,6 +537,9 @@ const KNOWN_PROVIDER_SPAN_NAMES: ReadonlySet = new Set([ "postUploadCommand", "session.open", "session.close", + "lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup", + "lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", + "lease.shell", "lease.create", "lease.expiry", "lease.cleanup", "plugin.worker", "plugin.host.identity.read", "plugin.host.config.read", "plugin.host.progress.read", "plugin.host.progress.write", "plugin.workspace.prepare", "plugin.storage.queue", "plugin.storage.execute", "plugin.runtime.queue", "plugin.runtime.prepare", "plugin.runtime.reconcile", "plugin.runtime.verify", @@ -564,6 +567,7 @@ const PROVIDER_SPAN_ATTR_ALLOWLIST: ReadonlySet = new Set([ SPAN_ATTRS.transferWallMs, SPAN_ATTRS.transferGuardCount, SPAN_ATTRS.transferDirection, + SPAN_ATTRS.state, "plugin.operation", "plugin.reused", "plugin.item_count", "plugin.bytes", "plugin.call_count", ]); @@ -595,6 +599,10 @@ export function clampProviderSpanAttributes( if (!raw) return clamped; for (const [key, value] of Object.entries(raw)) { if (!PROVIDER_SPAN_ATTR_ALLOWLIST.has(key)) continue; + if (key === SPAN_ATTRS.state) { + if (typeof value === "string" && ["started", "stopped", "archived", "error", "other"].includes(value)) clamped[key] = value; + continue; + } if (key === "plugin.operation") { if (typeof value === "string" && ["request", "queue.wait", "launch.wait", "mount.refresh", "guest.setup", "process.execute", "script.workspace", "script.git", "script.settings", "script.publication", "cleanup", "process.start", "hello", "checkpoint.load"].includes(value)) clamped[key] = value; continue; diff --git a/server/src/services/plugin-worker-manager.ts b/server/src/services/plugin-worker-manager.ts index 363d15c065c..2fa59b62573 100644 --- a/server/src/services/plugin-worker-manager.ts +++ b/server/src/services/plugin-worker-manager.ts @@ -725,6 +725,7 @@ export interface PluginWorkerHandle { executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise; /** @@ -865,6 +866,7 @@ export interface PluginWorkerManager { executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise; /** @@ -914,6 +916,10 @@ export function createPluginWorkerHandle( // Unlike RPC promises, these survive timeouts. A late response is the // completion receipt. A worker crash cannot manufacture one. const unsettledCalls = new Map void>(); + const observedResults = new Map; + observe: (result: unknown) => Promise; finish: () => void; + }>(); const startupCalls = new Map void>>(); const startupRetirements = new Map>(); const startupHostHandlers = new Map>>(); @@ -1156,24 +1162,50 @@ export function createPluginWorkerHandle( /** * Handle a JSON-RPC response from the worker (matching a pending request). */ + function settleResponse(response: JsonRpcResponse): void { + const id = response.id!; + unsettledCalls.get(id)?.(); + unsettledCalls.delete(id); + const pending = pendingRequests.get(id); + if (!pending) { log.warn({ id }, "received response for unknown request id"); return; } + clearTimeout(pending.timer); + pendingRequests.delete(id); + pending.resolve(response); + } + + function processObservedResult(id: string | number): Promise { + const observer = observedResults.get(id); + if (!observer?.response) return Promise.resolve(); + if (observer.running) return observer.running; + const response = observer.response; + const processing = (async () => { + if (isJsonRpcSuccessResponse(response)) await observer.observe(response.result); + observedResults.delete(id); + observer.finish(); + settleResponse(response); + })(); + observer.running = processing; + void processing.catch(() => { + const pending = pendingRequests.get(id); + pending?.resolve(createErrorResponse(id, JSONRPC_ERROR_CODES.INTERNAL_ERROR, "Plugin result could not be recorded")); + }).finally(() => { if (observer.running === processing) observer.running = undefined; }); + return processing; + } + function handleResponse(response: JsonRpcResponse): void { const id = response.id; if (id === null || id === undefined) { log.warn("received response with null/undefined id"); return; } - - unsettledCalls.get(id)?.(); - unsettledCalls.delete(id); - const pending = pendingRequests.get(id); - if (!pending) { - log.warn({ id }, "received response for unknown request id"); + const observer = observedResults.get(id); + if (observer) { + if (observer.response) return; + observer.response = response; + void processObservedResult(id).catch(() => log.warn({ id }, "plugin result recording will retry during cleanup")); return; } - - clearTimeout(pending.timer); - pendingRequests.delete(id); - pending.resolve(response); + settleResponse(response); } function readNonEmptyString(value: unknown): string | null { @@ -3087,6 +3119,11 @@ export function createPluginWorkerHandle( // Read-only requests cannot continue in an exited worker. Do not infer // completion for execute/cleanup calls, whose provider work may outlive it. for (const finish of [...unsettledInspections]) finish(); + for (const [id, observer] of observedResults) { + if (observer.response) continue; + observedResults.delete(id); + observer.finish(); + } // Close the one login pseudo-terminal route with a fixed non-secret exit and // clear the route one time. The pending pseudo-terminal calls @@ -3423,8 +3460,9 @@ export function createPluginWorkerHandle( executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise { - const dispatched = measureRunPhase("execution.plugin_dispatch", () => callInternalRaw(method, params, timeoutMs, executeLogSink, onWorkerSettled, startupContextId)); + const dispatched = measureRunPhase("execution.plugin_dispatch", () => callInternalRaw(method, params, timeoutMs, executeLogSink, onWorkerSettled, startupContextId, observeResult)); // Tracing wraps the RPC in a new promise. Guard the promise handed to the // caller as well, without swallowing its eventual rejection for that caller. void dispatched.catch(() => undefined); @@ -3438,6 +3476,7 @@ export function createPluginWorkerHandle( executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise { const rpcPromise = new Promise((resolve, reject) => { if (startupContextId && startupRetirements.has(startupContextId)) { @@ -3484,6 +3523,19 @@ export function createPluginWorkerHandle( } if (method === "environmentInspectLease") unsettledInspections.add(finish); } + if (observeResult && startupContextId) { + let finish!: () => void; + const processing = new Promise(resolve => { finish = resolve; }); + let handlers = startupHostHandlers.get(startupContextId); + if (!handlers) { handlers = new Set(); startupHostHandlers.set(startupContextId, handlers); } + handlers.add(processing); + observedResults.set(id, { contextId: startupContextId, observe: observeResult as (result: unknown) => Promise, finish: () => { + startupHostHandlers.get(startupContextId)?.delete(processing); + if (startupHostHandlers.get(startupContextId)?.size === 0) startupHostHandlers.delete(startupContextId); + finish(); + onStartupWorkChanged?.(); + } }); + } const timeout = resolveRpcCallTimeoutMs(timeoutMs, rpcTimeoutMs); const invocationScope = deriveInvocationScope(method, params); const invocation = invocationScope ? registerInvocation(invocationScope) : null; @@ -3550,6 +3602,8 @@ export function createPluginWorkerHandle( } catch (err) { clearTimeout(timer); pendingRequests.delete(id); + observedResults.get(id)?.finish(); + observedResults.delete(id); unsettledCalls.get(id)?.(); unsettledCalls.delete(id); clearInvocation(invocation); @@ -3637,6 +3691,7 @@ export function createPluginWorkerHandle( hasStartupWork() { return startupCalls.size > 0 || startupHostHandlers.size > 0 || startupRetirements.size > 0; }, async drainStartup(contextId: string) { revokeStartupInvocations(contextId); + for (const [id, observer] of observedResults) if (observer.contextId === contextId) void processObservedResult(id).catch(() => undefined); await waitForStartupHandlers(contextId); }, async retireStartup(contextId: string) { @@ -3644,6 +3699,7 @@ export function createPluginWorkerHandle( if (current) return current; const retiring = (async () => { revokeStartupInvocations(contextId); + for (const [id, observer] of observedResults) if (observer.contextId === contextId) void processObservedResult(id).catch(() => undefined); await waitForStartupHandlers(contextId); for (const finish of [...startupCalls.get(contextId) ?? []]) finish(); })(); @@ -3662,6 +3718,7 @@ export function createPluginWorkerHandle( executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise { if (status !== "running" && status !== "starting") { onWorkerSettled?.(); @@ -3671,7 +3728,7 @@ export function createPluginWorkerHandle( ), ); } - return callInternal(method, params, timeoutMs, executeLogSink, onWorkerSettled, startupContextId); + return callInternal(method, params, timeoutMs, executeLogSink, onWorkerSettled, startupContextId, observeResult); }, openLoginPtySession(input: LoginPtyOpenInput) { @@ -4032,6 +4089,7 @@ export function createPluginWorkerManager( executeLogSink?: ExecuteLogSink, onWorkerSettled?: () => void, startupContextId?: string, + observeResult?: (result: HostToWorkerMethods[M][1]) => Promise, ): Promise { const handle = workers.get(pluginId); if (!startupContextId && ["environmentReleaseLease", "environmentDestroyLease"].includes(method)) { @@ -4057,7 +4115,7 @@ export function createPluginWorkerManager( // A caller timeout is not worker completion. Inspection capacity is // released only by its actual reply, send failure, or worker exit. const finish = onWorkerSettled ? () => { finishInspection?.(); onWorkerSettled(); } : finishInspection; - const result = handle.call(method, params, timeoutMs, executeLogSink, finish, startupContextId); + const result = handle.call(method, params, timeoutMs, executeLogSink, finish, startupContextId, observeResult); if (method === "configChanged") void result.then(() => { for (const listener of configApplied) { try { listener(); } diff --git a/server/src/services/recovery/issue-state-read.test.ts b/server/src/services/recovery/issue-state-read.test.ts index 5a8ad5acf28..b1b4a39a0b5 100644 --- a/server/src/services/recovery/issue-state-read.test.ts +++ b/server/src/services/recovery/issue-state-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, issueRecoveryActions, issues } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -60,7 +60,7 @@ describe.skipIf(!support.supported)("recovery issue state reads", () => { const service = recoveryService(db, { enqueueWakeup }); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { issues, issueRecoveryActions }, capture); try { await service.reconcileStrandedAssignedIssues(); @@ -100,7 +100,7 @@ describe.skipIf(!support.supported)("recovery issue state reads", () => { const service = recoveryService(db, { enqueueWakeup }); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { agents, issues, issueRecoveryActions }, capture); try { expect(await service.reconcileStrandedAssignedIssues()).toMatchObject({ skipped: 1 }); diff --git a/server/src/services/recovery/legacy-disposition-read.test.ts b/server/src/services/recovery/legacy-disposition-read.test.ts index 749a7991879..21ae049d472 100644 --- a/server/src/services/recovery/legacy-disposition-read.test.ts +++ b/server/src/services/recovery/legacy-disposition-read.test.ts @@ -1,6 +1,6 @@ import { randomBytes, randomUUID } from "node:crypto"; import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db"; -import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture"; +import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js"; import { eq } from "drizzle-orm"; import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js"; @@ -37,7 +37,7 @@ describe.skipIf(!support.supported)("legacy disposition source reads", () => { errorCode: "adapter_failed", resultJson: { executionRecovery: { providerWorkStarted: true }, stdout: bulk } }).returning(); const events: Array> = []; const now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await readLegacyDispositionSource(db, { companyId, issueId, runId: before.id })) @@ -77,7 +77,7 @@ describe.skipIf(!support.supported)("legacy disposition source reads", () => { const enqueueWakeup = vi.fn(async () => { throw new Error("native work must not enter legacy recovery"); }); const service = recoveryService(db, { enqueueWakeup }); const events: Array> = [], now = Date.now(); - const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); + const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event)); installDatabaseResponseCapture(db, { heartbeatRuns }, capture); try { expect(await service.reconcileLegacyContinuation(before.id)).toBe("skipped"); } finally { capture.flush(); } diff --git a/server/src/services/remote-execution-termination.ts b/server/src/services/remote-execution-termination.ts index f9fae854c69..d30b60ae697 100644 --- a/server/src/services/remote-execution-termination.ts +++ b/server/src/services/remote-execution-termination.ts @@ -1,5 +1,8 @@ import { and, eq } from "drizzle-orm"; import { environmentLeases, type Db } from "@paperclipai/db"; +import { retiredReusableLeasePredecessors } from "./environment-lease-handoff.js"; + +type Lease = typeof environmentLeases.$inferSelect; type LeaseIdentity = { id: string; companyId: string; heartbeatRunId: string | null; @@ -39,12 +42,36 @@ export function remoteLeaseCleanupScope(lease: Pick { + if (hasRemoteTerminationReceipt(lease) || lease.status !== "expired" || + lease.leasePolicy !== "reuse_by_environment" || !lease.provider || !lease.providerLeaseId) return lease; + const history = await db.select().from(environmentLeases).where(and( + eq(environmentLeases.provider, lease.provider), eq(environmentLeases.providerLeaseId, lease.providerLeaseId), + )); + const current = history.find(row => row.id === lease.id); + if (!current || current.companyId !== lease.companyId || current.heartbeatRunId !== lease.heartbeatRunId) return lease; + const tips = history.filter(row => !Object.hasOwn(row.metadata ?? {}, "reusableLeaseSuccessor")); + if (tips.length !== 1) return current; + const tip = tips[0]!; + const receipt = tip.metadata?.remoteExecutionTermination as Record | undefined; + if (!hasRemoteTerminationReceipt(tip) || receipt?.state !== "destroyed") return current; + const predecessors = retiredReusableLeasePredecessors(tip, history.filter(row => row.id !== tip.id)); + if (!predecessors?.some(row => row.id === current.id)) return current; + const termination = remoteTerminationReceipt(current, { providerLeaseId: current.providerLeaseId, state: "destroyed" }); + if (!termination) return current; + return { ...current, metadata: { ...current.metadata, remoteExecutionTermination: { + ...termination, confirmedAt: receipt.confirmedAt, + } } }; +} + export async function stoppedRemoteCleanupScopes(db: Db, companyId: string, runId: string) { const leases = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, companyId), eq(environmentLeases.heartbeatRunId, runId), )); - if (leases.length === 0 || !leases.every(hasRemoteTerminationReceipt)) return null; - return [...new Set(leases.map(lease => remoteLeaseCleanupScope(lease)!))]; + if (leases.length === 0) return null; + const resolved = await Promise.all(leases.map(lease => resolveRemoteTerminationLease(db, lease))); + if (!resolved.every(hasRemoteTerminationReceipt)) return null; + return [...new Set(resolved.map(lease => remoteLeaseCleanupScope(lease)!))]; } export async function remoteExecutionHasStopped(db: Db, companyId: string, runId: string) { diff --git a/server/src/services/remote-workspace-files.ts b/server/src/services/remote-workspace-files.ts index 5031b5ece9f..fd3d5ae3d42 100644 --- a/server/src/services/remote-workspace-files.ts +++ b/server/src/services/remote-workspace-files.ts @@ -64,6 +64,7 @@ export interface SavedWorkspaceFiles { export interface RemoteWorkspaceFileOptions { environmentRuntime?: Pick; + /** Verify the exact company-owned, committed snapshot before exposing files. */ loadSavedWorkspace?(input: { companyId: string; workspaceId: string; @@ -290,7 +291,11 @@ export function createRemoteWorkspaceFiles(db: Db, options: RemoteWorkspaceFileO ...(input.signal ? { signal: input.signal } : {}) }); return { root: saved.root, source: { kind: "saved", savedAt: saved.savedAt, snapshotId: authority.snapshotId } as WorkspaceFileSource }; }; - if (authority.location) { + // A final save clears dirty state and writer authority. Its accepted + // bytes do not depend on an allocation that can already be closing. + if (authority.snapshotId && !authority.dirty && !authority.writerRunId && options.loadSavedWorkspace) { + ({ root, source } = await loadSaved()); + } else if (authority.location) { const [leaseRow] = await db.select().from(environmentLeases).where(and( eq(environmentLeases.companyId, input.companyId), eq(environmentLeases.id, authority.location.leaseId), )).limit(1); diff --git a/server/src/services/remote-workspace-snapshot-archive.test.ts b/server/src/services/remote-workspace-snapshot-archive.test.ts index 07fa20609f1..ef9664d05df 100644 --- a/server/src/services/remote-workspace-snapshot-archive.test.ts +++ b/server/src/services/remote-workspace-snapshot-archive.test.ts @@ -7,10 +7,21 @@ import { promisify } from "node:util"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { captureRemoteWorkspaceDirectory, remoteWorkspaceArchiveScript, restoreRemoteWorkspaceArchive } from "./remote-workspace-snapshot-archive.js"; -const archiveIo = vi.hoisted(() => ({ afterChmod: undefined as ((target: string, mode: number) => void) | undefined })); +const archiveIo = vi.hoisted(() => ({ + afterChmod: undefined as ((target: string, mode: number) => void) | undefined, + beforeOpen: undefined as ((target: string) => Promise) | undefined, + captures: [] as string[], +})); vi.mock("node:fs/promises", async importOriginal => { const actual = await importOriginal(); - return { ...actual, async chmod(target: string, mode: number) { + return { ...actual, async open(...args: Parameters) { + await archiveIo.beforeOpen?.(String(args[0])); + return actual.open(...args); + }, async mkdtemp(...args: Parameters) { + const directory = await actual.mkdtemp(...args); + if (String(args[0]).includes("pc-workspace-capture-")) archiveIo.captures.push(String(directory)); + return directory; + }, async chmod(target: string, mode: number) { await actual.chmod(target, mode); archiveIo.afterChmod?.(target, mode); } }; }); @@ -22,7 +33,10 @@ const git = async (cwd: string, ...args: string[]) => (await execute("git", ["-C describe("portable remote workspace archives", () => { let root: string; beforeEach(async () => { root = await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-archive-test-")); }); - afterEach(async () => { archiveIo.afterChmod = undefined; await fs.rm(root, { recursive: true, force: true }); }); + afterEach(async () => { + archiveIo.afterChmod = undefined; archiveIo.beforeOpen = undefined; archiveIo.captures = []; + await fs.rm(root, { recursive: true, force: true }); + }); async function source() { const cwd = path.join(root, "source"); await fs.mkdir(cwd); return cwd; } async function capture(cwd: string) { const archivePath = path.join(root, "snapshot.pcws"); return { archivePath, ...await captureRemoteWorkspaceDirectory({ cwd, archivePath }) }; } async function restore(snapshot: Awaited>) { @@ -57,6 +71,76 @@ describe("portable remote workspace archives", () => { const snapshot = await capture(await source()); const restored = await restore(snapshot); expect(await fs.readdir(restored)).toEqual([]); }); + it("captures with room for one archive and bounded metadata without another copy of working files", async () => { + const cwd = await source(), payload = Buffer.alloc(4 * 1024 * 1024, 123); + await fs.writeFile(path.join(cwd, "work.bin"), payload); + const archivePath = path.join(root, "snapshot.pcws"); + async function bytes(directory: string): Promise { + let total = 0; + for (const name of await fs.readdir(directory)) { + const file = path.join(directory, name), stat = await fs.lstat(file); + total += stat.isDirectory() ? await bytes(file) : stat.size; + } + return total; + } + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + let staged = 0; + for (const directory of archiveIo.captures) staged += await bytes(directory); + if (staged + payload.length > payload.length + 128 * 1024) { + throw Object.assign(new Error("No space left for the archive"), { code: "ENOSPC" }); + } + }; + const snapshot = await capture(cwd); + archiveIo.beforeOpen = undefined; + const digest = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex"); + expect(digest(await fs.readFile(path.join(await restore(snapshot), "work.bin")))).toBe(digest(payload)); + expect(digest(await fs.readFile(path.join(cwd, "work.bin")))).toBe(digest(payload)); + }); + it.each(["content", "symlink", "added-file"])("rejects a concurrent %s change and removes its incomplete archive", async change => { + const cwd = await source(), file = path.join(cwd, "work.txt"), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(file, "original\n"); + const outside = path.join(root, "outside.txt"); await fs.writeFile(outside, "must stay outside\n"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + if (change === "content") await fs.writeFile(file, "changed\n"); + if (change === "added-file") await fs.writeFile(path.join(cwd, "new.txt"), "new file\n"); + if (change === "symlink") { await fs.unlink(file); await fs.symlink(outside, file); } + }; + await expect(capture(cwd)).rejects.toThrow(); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await fs.readFile(outside, "utf8")).toBe("must stay outside\n"); + for (const directory of archiveIo.captures) await expect(fs.access(directory)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("cleans a cancelled capture without deleting a pre-existing archive", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + const controller = new AbortController(), reason = new Error("capture cancelled"); + archiveIo.beforeOpen = async target => { if (target === archivePath) controller.abort(reason); }; + await expect(captureRemoteWorkspaceDirectory({ cwd, archivePath, signal: controller.signal })).rejects.toBe(reason); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + archiveIo.beforeOpen = undefined; + await fs.writeFile(archivePath, "existing archive\n"); + await expect(capture(cwd)).rejects.toMatchObject({ code: "EEXIST" }); + expect(await fs.readFile(archivePath, "utf8")).toBe("existing archive\n"); + expect(await fs.readFile(path.join(cwd, "file"), "utf8")).toBe("retained\n"); + }); + it("rejects Git reference changes during archive output and keeps the new reference", async () => { + const cwd = await source(), archivePath = path.join(root, "snapshot.pcws"); + await git(cwd, "init", "-q", "-b", "main"); + await fs.writeFile(path.join(cwd, "file"), "retained\n"); + await git(cwd, "add", "."); await git(cwd, "commit", "-qm", "base"); + const head = await git(cwd, "rev-parse", "HEAD"); + archiveIo.beforeOpen = async target => { + if (target !== archivePath) return; + archiveIo.beforeOpen = undefined; + await git(cwd, "update-ref", "refs/heads/later-work", head); + }; + await expect(capture(cwd)).rejects.toThrow("remote_workspace_archive_changed"); + await expect(fs.access(archivePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(await git(cwd, "rev-parse", "refs/heads/later-work")).toBe(head); + }); it("rejects a Files entry limit before extraction while ordinary restores keep their full contents", async () => { const cwd = await source(); await fs.writeFile(path.join(cwd, "first.txt"), "first\n"); await fs.writeFile(path.join(cwd, "second.txt"), "second\n"); const snapshot = await capture(cwd), targetDir = path.join(root, "limited"); diff --git a/server/src/services/remote-workspace-snapshot-archive.ts b/server/src/services/remote-workspace-snapshot-archive.ts index 6d4b1552053..6c910a83ed9 100644 --- a/server/src/services/remote-workspace-snapshot-archive.ts +++ b/server/src/services/remote-workspace-snapshot-archive.ts @@ -120,9 +120,28 @@ async function workspaceArchive(input: { if (state) fail("git_operation_in_progress: finish or abort the Git operation before saving the workspace"); } }; + const readStableFile = async (source: string, consume?: (bytes: Buffer) => Promise) => { + check(); + const original = await fs.lstat(source, { bigint: true }); + const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); + try { + const before = await handle.stat({ bigint: true }); + if (!before.isFile() || original.dev !== before.dev || original.ino !== before.ino || await fs.realpath(source) !== source) fail("changed"); + const digest = crypto.createHash("sha256"); let size = 0; + for await (const bytes of handle.createReadStream({ autoClose: false, signal: input.signal })) { + check(); digest.update(bytes); size += bytes.length; + await consume?.(bytes); + } + const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); + if (size !== Number(before.size) || await fs.realpath(source) !== source + || ["dev", "ino", "size", "mode", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] + || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); + return { sha256: digest.digest("hex"), size, mode: Number(before.mode & 0o777n) }; + } finally { await handle.close(); } + }; const directories: string[] = []; const excludedPaths = new Set(); - const walk = async (root: string, copyTo?: string, skipGit = false) => { + const walk = async (root: string, skipGit = false) => { const entries: Entry[] = []; const visit = async (relative: string, parentIsRepository = false) => { const directory = relative ? path.join(root, relative) : root; @@ -137,7 +156,7 @@ async function workspaceArchive(input: { if (input.exclude?.some(excluded => next === excluded || next.startsWith(`${excluded}/`)) || next === ".paperclip-runtime" || next.startsWith(".paperclip-runtime/")) { excludedPaths.add(next); continue; } - const source = path.join(root, next), target = copyTo ? path.join(copyTo, next) : null; + const source = path.join(root, next); const stat = await fs.lstat(source); if (skipGit && name === ".git") { if (stat.isSymbolicLink()) fail("unsafe_git_link"); directories.push(relative); continue; } if (stat.isDirectory() && name === "node_modules") { @@ -150,30 +169,13 @@ async function workspaceArchive(input: { } if (stat.isDirectory()) { entries.push({ path: next, type: "directory", mode: stat.mode & 0o777 }); - if (target) await fs.mkdir(target, { mode: 0o700 }); await visit(next, inRepository); } else if (stat.isSymbolicLink()) { const link = await fs.readlink(source); if (!safeLink(next, link)) fail("unsafe_symlink"); entries.push({ path: next, type: "symlink", target: link }); - if (target) await fs.symlink(link, target); } else if (stat.isFile()) { - const handle = await fs.open(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW | fs.constants.O_NONBLOCK); - let output: Awaited> | undefined; - try { - const before = await handle.stat({ bigint: true }); - if (!before.isFile() || BigInt(stat.dev) !== before.dev || BigInt(stat.ino) !== before.ino || await fs.realpath(source) !== source) fail("changed"); - if (target) output = await fs.open(target, "wx", 0o600); - const digest = crypto.createHash("sha256"); let bytes = 0; - for await (const part of handle.createReadStream({ autoClose: false, signal: input.signal })) { - check(); digest.update(part); bytes += part.length; - await output?.writeFile(part); - } - const after = await handle.stat({ bigint: true }), current = await fs.lstat(source, { bigint: true }); - if (bytes !== Number(before.size) || await fs.realpath(source) !== source || ["dev", "ino", "size", "mtimeNs", "ctimeNs"].some(key => before[key as keyof typeof before] !== after[key as keyof typeof after] || after[key as keyof typeof after] !== current[key as keyof typeof current])) fail("changed"); - entries.push({ path: next, type: "file", mode: stat.mode & 0o777, size: bytes, sha256: digest.digest("hex") }); - if (target) await fs.chmod(target, stat.mode & 0o777); - } finally { await output?.close(); await handle.close(); } + entries.push({ path: next, type: "file", ...await readStableFile(source) }); } else fail("unsupported_entry"); } }; @@ -188,9 +190,10 @@ async function workspaceArchive(input: { const root = await fs.realpath(input.cwd!); if (!(await fs.lstat(root)).isDirectory()) fail("source_invalid"); const scratch = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "pc-workspace-capture-"))); + let archiveCreated = false, completed = false; try { const tree = path.join(scratch, "tree"); await fs.mkdir(tree, { mode: 0o700 }); - const before = await walk(root, tree, true); + const before = await walk(root, true); const repositories = [...new Set(directories)]; const gitSignatures = new Map(); const signature = async (repo: string, settings?: Awaited>) => { @@ -207,6 +210,7 @@ async function workspaceArchive(input: { // private repository. The selected worktree's index and HEAD are separate. await git(root, ["clone", "--mirror", "--no-hardlinks", "--dissociate", "--", repo, normalized]); const destination = path.join(tree, relative, ".git"); + await fs.mkdir(path.dirname(destination), { recursive: true, mode: 0o700 }); await fs.rename(normalized, destination); const sourceGit = (await git(repo, ["rev-parse", "--absolute-git-dir"])).trim(); const sourceCommon = (await git(repo, ["rev-parse", "--path-format=absolute", "--git-common-dir"])).trim(); @@ -232,35 +236,38 @@ async function workspaceArchive(input: { const [object, ref] = row.split(" "); return `update ${ref} ${object}\n`; }).join(""); if (updates) await git(restoredRepo, ["update-ref", "--stdin"], updates); - await git(restoredRepo, ["repack", "-a", "-d"]); await fs.rm(path.join(destination, "objects", "info", "alternates"), { force: true }); await git(restoredRepo, ["fsck", "--connectivity-only", "--no-dangling"]); if (await signature(repo) !== gitSignatures.get(relative)) fail("changed"); } - directories.length = 0; - const after = await walk(root, undefined, true); - if (JSON.stringify(before) !== JSON.stringify(after)) fail("changed"); - for (const relative of repositories) await assertGitOperationComplete(path.join(root, relative)); - const entries = await walk(tree); - // Keep private staging writable for cleanup; the manifest preserves the - // source modes and restoration applies them only after writing children. const originals = new Map(before.map(entry => [entry.path, entry])); - for (const entry of entries) if (entry.type === "directory") { - const original = originals.get(entry.path); if (original) entry.mode = original.mode; - } + const entries = [...before, ...(await walk(tree)).filter(entry => !originals.has(entry.path))] + .sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); const manifest = { schema, exclusionPolicy: "paperclip.remote-workspace-exclusions.v1", excluded: [...excludedPaths].sort(), entries }, body = Buffer.from(JSON.stringify(manifest)); if (body.length > 64 * 1024 * 1024) fail("manifest_too_large"); const digest = manifestHash(manifest), length = Buffer.alloc(4); length.writeUInt32BE(body.length); const output = await fs.open(input.archivePath, "wx", 0o600); + archiveCreated = true; try { await output.writeFile(Buffer.concat([magic, length, body])); for (const entry of entries) if (entry.type === "file") { - for await (const bytes of createReadStream(path.join(tree, entry.path), { signal: input.signal })) { check(); await output.writeFile(bytes); } + const observed = await readStableFile(path.join(originals.has(entry.path) ? root : tree, entry.path), bytes => output.writeFile(bytes)); + if (observed.size !== entry.size || observed.sha256 !== entry.sha256 || observed.mode !== entry.mode) fail("changed"); } } finally { await output.close(); } + directories.length = 0; + if (JSON.stringify(before) !== JSON.stringify(await walk(root, true))) fail("changed"); + for (const relative of repositories) { + if (await signature(path.join(root, relative)) !== gitSignatures.get(relative)) fail("changed"); + } if (input.manifestPath) await fs.writeFile(input.manifestPath, body, { flag: "wx", mode: 0o600 }); - return { schema, manifestDigest: digest, ...await hashFile(input.archivePath), entries: entries.length }; - } finally { await fs.rm(scratch, { recursive: true, force: true }); } + const content = await hashFile(input.archivePath); + completed = true; + return { schema, manifestDigest: digest, ...content, entries: entries.length }; + } finally { + if (archiveCreated && !completed) await fs.rm(input.archivePath, { force: true }); + await fs.rm(scratch, { recursive: true, force: true }); + } } if (!input.targetDir || !/^[a-f0-9]{64}$/.test(input.expectedManifestDigest ?? "")) fail("restore_binding_invalid"); if (input.maxEntries !== undefined && (!Number.isSafeInteger(input.maxEntries) || input.maxEntries < 0)) fail("restore_binding_invalid"); diff --git a/server/src/services/remote-workspace-snapshot-storage.test.ts b/server/src/services/remote-workspace-snapshot-storage.test.ts index 8700af9346d..e1c0511da9c 100644 --- a/server/src/services/remote-workspace-snapshot-storage.test.ts +++ b/server/src/services/remote-workspace-snapshot-storage.test.ts @@ -86,7 +86,12 @@ suite("remote workspace snapshot storage", () => { it("rejects object corruption and leaves the requested plaintext path unpublished", async () => { const f = await fixture(), ref = await uploadWorkspaceSnapshot(f.upload); const objectPath = path.join(root, "objects", ref.objectKey), object = await fs.open(objectPath, "r+"); - try { await object.write(Buffer.from("X"), 0, 1, 15); } finally { await object.close(); } + try { + const byte = Buffer.alloc(1); + expect((await object.read(byte, 0, 1, 15)).bytesRead).toBe(1); + byte[0] = byte[0]! ^ 1; + await object.write(byte, 0, 1, 15); + } finally { await object.close(); } const targetPath = path.join(root, "corrupted.pcws"); await expect(loadWorkspaceSnapshotArchive({ storage, reference: ref, targetPath })).rejects.toThrow("object_hash_mismatch"); await expect(fs.access(targetPath)).rejects.toMatchObject({ code: "ENOENT" }); diff --git a/server/src/services/sandbox-creation-operations.ts b/server/src/services/sandbox-creation-operations.ts index a56c6615284..7fab2432fe6 100644 --- a/server/src/services/sandbox-creation-operations.ts +++ b/server/src/services/sandbox-creation-operations.ts @@ -4,9 +4,9 @@ import { and, eq, lte, or, sql } from "drizzle-orm"; import { companies, environments, sandboxCreationOperations, type Db } from "@paperclipai/db"; import { readEnvironmentCreationCleanupError, type PluginEnvironmentLease, type PluginEnvironmentLeaseObservation } from "@paperclipai/plugin-sdk"; import { assertEnvironmentCompanyBinding, environmentService } from "./environments.js"; -import { ENVIRONMENT_CLEANUP_RETRY_MS } from "./environment-cleanup-state.js"; +import { ENVIRONMENT_CLEANUP_RETRY_MS, recoverableSandboxCreationCondition } from "./environment-cleanup-state.js"; import { logger } from "../middleware/logger.js"; -import { notifyEnvironmentCleanup } from "./environment-cleanup-notifications.js"; +import { notifyEnvironmentCleanup, writeEnvironmentCleanup } from "./environment-cleanup-notifications.js"; export type SandboxCreationOperation = typeof sandboxCreationOperations.$inferSelect; @@ -78,26 +78,69 @@ export async function createOwnedSandbox( * This enriches the existing owner; it never transfers or reopens ownership. */ export async function recordSandboxCreationResource(db: Db, input: { id: string; companyId: string; environmentId: string; providerLeaseId: string; + providerMetadata?: Record; }): Promise { - const [recorded] = await db.update(sandboxCreationOperations).set({ - metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ creationProviderLeaseId: input.providerLeaseId })}::jsonb`, - updatedAt: new Date(), - }).where(and(eq(sandboxCreationOperations.id, input.id), eq(sandboxCreationOperations.companyId, input.companyId), - eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId), - or(sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' is null`, - sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' = ${input.providerLeaseId}`), - )).returning({ id: sandboxCreationOperations.id }); - if (!recorded) throw new Error("Sandbox creation resource identity changed"); + await writeEnvironmentCleanup(db, async tx => { + const [recorded] = await tx.update(sandboxCreationOperations).set({ + nextAttemptAt: sql`case when ${sandboxCreationOperations.state} = 'needs_reconciliation' then clock_timestamp() else ${sandboxCreationOperations.nextAttemptAt} end`, + metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || ${JSON.stringify({ creationProviderLeaseId: input.providerLeaseId, ...(input.providerMetadata ? { creationProviderMetadata: input.providerMetadata } : {}) })}::jsonb`, + updatedAt: new Date(), + }).where(and(eq(sandboxCreationOperations.id, input.id), eq(sandboxCreationOperations.companyId, input.companyId), + eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId), + or(sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' is null`, + sql`${sandboxCreationOperations.metadata}->>'creationProviderLeaseId' = ${input.providerLeaseId}`), + )).returning({ id: sandboxCreationOperations.id }); + if (!recorded) throw new Error("Sandbox creation resource identity changed"); + }, { companyId: input.companyId }); +} + +export async function queueSandboxCreationCleanup(db: Db, input: { + id: string; companyId: string; environmentId: string; providerLeaseId: string; + providerMetadata?: Record; +}): Promise { + await db.transaction(async tx => { + await tx.select({ id: environments.id }).from(environments).where(eq(environments.id, input.environmentId)).for("update"); + const [operation] = await tx.select().from(sandboxCreationOperations).where(and(eq(sandboxCreationOperations.id, input.id), + eq(sandboxCreationOperations.companyId, input.companyId), eq(sandboxCreationOperations.originalEnvironmentId, input.environmentId))).for("update"); + if (!operation || (operation.metadata.creationProviderLeaseId && operation.metadata.creationProviderLeaseId !== input.providerLeaseId)) { + throw new Error("Sandbox creation cleanup ownership changed"); + } + if (["attached", "cleanup_pending", "resolved"].includes(operation.state)) return; + const contextId = operation.metadata.runnerTaskLease === true && typeof operation.metadata.runnerTaskCreationContextId === "string" + ? operation.metadata.runnerTaskCreationContextId : null; + if (operation.state !== "pending" && !(operation.state === "needs_reconciliation" && contextId && + operation.metadata.creationProviderLeaseId === input.providerLeaseId)) throw new Error("Sandbox creation cleanup ownership changed"); + await notifyEnvironmentCleanup(tx, { companyId: operation.companyId, runId: operation.runId }); + const lease = await environmentService(tx as unknown as Db).insertPendingCleanupLease({ + companyId: operation.companyId, environmentId: operation.originalEnvironmentId, + heartbeatRunId: operation.runId, issueId: operation.issueId, executionWorkspaceId: operation.executionWorkspaceId, + provider: operation.provider, providerLeaseId: input.providerLeaseId, + metadata: { ...operation.metadata, creationOperationId: operation.id, environmentId: operation.originalEnvironmentId, + providerMetadata: input.providerMetadata ?? operation.metadata.creationProviderMetadata ?? {}, runnerTaskRetiring: true, + ...(contextId ? { runnerTaskStartupCleanup: { contextId, requiresPreparationStop: false } } : {}) }, + failureReason: "interrupted_task_creation", + }); + await tx.update(sandboxCreationOperations).set({ state: "cleanup_pending", leaseId: lease.id, updatedAt: new Date(), + metadata: { ...operation.metadata, creationProviderLeaseId: input.providerLeaseId, + creationProviderMetadata: input.providerMetadata ?? operation.metadata.creationProviderMetadata ?? {} }, + }).where(eq(sandboxCreationOperations.id, operation.id)); + }); } export async function recoverSandboxCreations(db: Db, inspect: (operation: SandboxCreationOperation) => Promise, ): Promise { const due = await db.select().from(sandboxCreationOperations).where(and( - eq(sandboxCreationOperations.state, "pending"), lte(sandboxCreationOperations.nextAttemptAt, new Date()), + recoverableSandboxCreationCondition(), lte(sandboxCreationOperations.nextAttemptAt, new Date()), )).limit(100); for (const candidate of due) { try { + if (candidate.metadata.runnerTaskLease === true && typeof candidate.metadata.runnerTaskCreationContextId === "string" && + typeof candidate.metadata.creationProviderLeaseId === "string") { + await queueSandboxCreationCleanup(db, { id: candidate.id, companyId: candidate.companyId, + environmentId: candidate.originalEnvironmentId, providerLeaseId: candidate.metadata.creationProviderLeaseId }); + continue; + } const [existingCompany] = await db.select({ id: companies.id }).from(companies).where(eq(companies.id, candidate.companyId)).limit(1); const observation = existingCompany ? await inspect(candidate) : null; const cleanup = observation?.creationCleanup ? readEnvironmentCreationCleanupError({ data: { @@ -140,7 +183,7 @@ export async function recoverSandboxCreations(db: Db, ? { nextAttemptAt: new Date(Date.now() + ENVIRONMENT_CLEANUP_RETRY_MS), updatedAt: new Date() } : { state: "needs_reconciliation", updatedAt: new Date(), metadata: sql`coalesce(${sandboxCreationOperations.metadata}, '{}'::jsonb) || '{"creationReconciliationReason":"company_deleted"}'::jsonb` }) - .where(and(eq(sandboxCreationOperations.id, candidate.id), eq(sandboxCreationOperations.state, "pending"))); + .where(and(eq(sandboxCreationOperations.id, candidate.id), recoverableSandboxCreationCondition())); return Boolean(company); }); logger.warn({ errorKind: retry ? "sandbox_creation_recovery_failed" : "sandbox_creation_needs_reconciliation", operationId: candidate.id, companyId: candidate.companyId }, diff --git a/server/src/services/status-cards.ts b/server/src/services/status-cards.ts index d4fe9d7ce80..669e097c532 100644 --- a/server/src/services/status-cards.ts +++ b/server/src/services/status-cards.ts @@ -1,4 +1,5 @@ import { notifyStatusCardEvaluation } from "./status-card-notifications.js"; +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { createHash } from "node:crypto"; import { and, desc, eq, gt, gte, inArray, isNotNull, isNull, lte, ne, or, sql, type SQL } from "drizzle-orm"; import { @@ -552,6 +553,7 @@ export function statusCardService( identifier: issues.identifier, title: issues.title, status: issues.status, + displayStatus: issueDisplayStatusSql(), priority: issues.priority, assigneeAgentId: issues.assigneeAgentId, assigneeUserId: issues.assigneeUserId, diff --git a/server/src/services/task-retry.ts b/server/src/services/task-retry.ts index a116aeba7fe..71b6b12c292 100644 --- a/server/src/services/task-retry.ts +++ b/server/src/services/task-retry.ts @@ -6,7 +6,7 @@ import { z } from "zod"; import { randomUUID } from "node:crypto"; import { and, desc, eq, inArray, or, sql } from "drizzle-orm"; import { agentWakeupRequests, chatActions, chatConversations, chatEndpoints, heartbeatRuns, issues, issueComments, runIdentityContexts, issueRecoveryActions, type Db } from "@paperclipai/db"; -import { conflict, forbidden, notFound } from "../errors.js"; +import { conflict, forbidden, notFound, HttpError } from "../errors.js"; import { persistActivity } from "./activity-log.js"; import { createSystemRecovery, readSystemRecovery } from "./system-recovery-state.js"; import { adoptSystemRecovery, updateSystemRecovery } from "./system-recovery.js"; @@ -17,6 +17,10 @@ import { preserveNativeFailedWorkspace } from "./native-runtime/native-failed-wo import { NativeWorkspaceFinalizationBusyError } from "./native-runtime/native-workspace-finalization-ownership.js"; import { adapterExecutionControls } from "./adapter-execution-control.js"; import { hasRequiredWorkspaceRecovery } from "./workspace-restore-recovery-state.js"; +import { deferredChatInputPredicate, resumeDeferredChatWake } from "./durable-chat-wakeup.js"; +import { notifyDeferredExecutionWork } from "./deferred-execution-notifications.js"; +import { hasPendingRunCleanup } from "./execution-blocker.js"; +import { PersistedExecutionWaitRefusal } from "./deferred-execution-wait.js"; export async function taskRetryMutationAllowed(db: Db, input: { companyId: string; issueId: string; actorId: string; actorSource: TaskRetryIntent["actorSource"]; @@ -45,6 +49,7 @@ const taskRetryIntentSchema = z.object({ schema: z.literal("paperclip.task-retry.v1"), generationId: z.string().uuid(), sourceRunId: z.string().uuid(), actorId: z.string().min(1), actorSource: z.enum(["local_implicit", "session", "board_key", "cloud_tenant"]).default("session"), strategy: z.enum(["evaluate", "continue", "fresh_isolated_workspace", "finalize_result"]), chatActionId: z.string().uuid().optional(), canonicalIntentId: z.string().uuid().optional(), + queuedChatRequestId: z.string().uuid().optional(), boardMessageOrigin: z.object({ commentId: z.string().uuid(), identityContextId: z.string().uuid() }).optional(), }); export type TaskRetryIntent = z.infer; @@ -159,7 +164,7 @@ async function readBoardMessageRetrySource(db: Db, input: { sql`${agentWakeupRequests.payload}->>'issueId' = ${input.issueId}`)); const intent = readTaskRetryIntent(receipt?.payload); if (!intent?.boardMessageOrigin || intent.generationId !== receipt!.id || intent.actorId !== input.actorId || - intent.chatActionId || intent.canonicalIntentId || intent.sourceRunId !== successor.previousRunId) return null; + intent.chatActionId || intent.queuedChatRequestId || intent.canonicalIntentId || intent.sourceRunId !== successor.previousRunId) return null; const [identity] = await db.select({ id: runIdentityContexts.id }).from(runIdentityContexts).where(and( eq(runIdentityContexts.id, successor.identityContextId), eq(runIdentityContexts.companyId, input.companyId), eq(runIdentityContexts.runId, input.sourceRunId), eq(runIdentityContexts.responsibleUserId, input.actorId), @@ -279,6 +284,7 @@ export async function requestTaskRetry(input: { sourceRunId: latest.id, actorId: input.actorId, actorSource: taskRetryIntentSchema.shape.actorSource.parse(input.actorSource ?? "session"), strategy: "evaluate", ...(active && activeIntent?.chatActionId ? { chatActionId: activeIntent.chatActionId } : {}), + ...(active && activeIntent?.queuedChatRequestId ? { queuedChatRequestId: activeIntent.queuedChatRequestId } : {}), ...(active ? { canonicalIntentId: active.id } : {}) }; // Preserve external chat authority through its existing admission owner. // Generic board Retry must never manufacture a chat-inbound request. @@ -290,7 +296,20 @@ export async function requestTaskRetry(input: { ? await readTaskRetryBoardMessageOrigin(db, { companyId: issue.companyId, issueId: issue.id, agentId: issue.assigneeAgentId, sourceRunId: latest.id, actorId: input.actorId }) : null; if (boardMessageOrigin) intent.boardMessageOrigin = boardMessageOrigin; - if (chat && !active && !busy && !boardMessageOrigin && (latest.contextSnapshot?.source === "issue.comment.reopen" || latest.contextSnapshot?.boardMessageRetry === true) && + if (chat && !active && latest.status === "succeeded") { + const [savedInput] = await tx.select({ id: agentWakeupRequests.id }).from(agentWakeupRequests).where(and( + eq(agentWakeupRequests.companyId, issue.companyId), eq(agentWakeupRequests.agentId, issue.assigneeAgentId), + eq(agentWakeupRequests.status, "deferred_issue_execution"), sql`${agentWakeupRequests.runId} is null`, + sql`${agentWakeupRequests.payload}->>'issueId' = ${issue.id}`, deferredChatInputPredicate(db), + )).orderBy(desc(agentWakeupRequests.requestedAt), desc(agentWakeupRequests.id)).limit(1).for("update"); + // A completed turn stays history. Only independently admitted input can + // continue, with its original identity and current inbound authorization. + if (savedInput) { + intent.queuedChatRequestId = savedInput.id; + await notifyDeferredExecutionWork(tx, { companyId: issue.companyId, requestId: id }); + } + } + if (chat && !active && !busy && !boardMessageOrigin && !intent.queuedChatRequestId && (latest.contextSnapshot?.source === "issue.comment.reopen" || latest.contextSnapshot?.boardMessageRetry === true) && latest.responsibleUserId === input.actorId) { throw conflict("This saved message cannot be retried safely. Send a new message on the task to continue from saved work.", { code: "chat_failed_run_retry_not_authorized", reason: "saved_board_message_not_replayable" }); @@ -301,7 +320,7 @@ export async function requestTaskRetry(input: { status: active || busy ? "coalesced" : "deferred_issue_execution", runId: active?.runId ?? (busy ? latest.id : null), payload: { issueId: issue.id, taskRetry: intent }, finishedAt: active || busy ? new Date() : null }); - if (chat && !boardMessageOrigin && !active && !busy) { + if (chat && !boardMessageOrigin && !intent.queuedChatRequestId && !active && !busy) { if (!input.prepareChatRetry) throw conflict("Retry is unavailable for this connected conversation."); intent.chatActionId = (await input.prepareChatRetry(db, { companyId: issue.companyId, issueId: issue.id, agentId: issue.assigneeAgentId, failedRunId: latest.id, initiatedByUserId: input.actorId, retryIntentId: id })).actionId; @@ -351,21 +370,58 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId? scope.companyId ? eq(agentWakeupRequests.companyId, scope.companyId) : undefined, scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined, eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"), - sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, + or(sql`${agentWakeupRequests.payload}#>>'{taskRetry,chatActionId}' is not null`, + sql`${agentWakeupRequests.payload}#>>'{taskRetry,queuedChatRequestId}' is not null`), )).orderBy(agentWakeupRequests.updatedAt).limit(50); for (const wake of pending) { const intent = readTaskRetryIntent(wake.payload); - if (!intent?.chatActionId) continue; + if (!intent || (!intent.chatActionId && !intent.queuedChatRequestId)) continue; + const actionId = intent.queuedChatRequestId ?? intent.chatActionId!; const [chat] = await db.select().from(chatActions).where(and(eq(chatActions.companyId, wake.companyId), - eq(chatActions.id, intent.chatActionId), eq(chatActions.kind, "failed_run_retry"), + eq(chatActions.id, actionId), eq(chatActions.kind, intent.queuedChatRequestId ? "inbound_wakeup" : "failed_run_retry"), sql`${chatActions.payload}->>'issueId' = ${String(wake.payload?.issueId)}`)); - const [receipt] = chat ? await db.select().from(agentWakeupRequests).where(and( + let [receipt] = chat ? await db.select().from(agentWakeupRequests).where(and( eq(agentWakeupRequests.companyId, wake.companyId), eq(agentWakeupRequests.id, chat.id), eq(agentWakeupRequests.agentId, wake.agentId))) : []; + let inputDenied = false; + if (intent.queuedChatRequestId && !scope.settleOnly && !receipt?.runId) { + const issueId = String(wake.payload?.issueId); + const admission = await db.transaction(async tx => { + if (intent.generationId !== wake.id || intent.canonicalIntentId || + wake.requestedByActorType !== "user" || wake.requestedByActorId !== intent.actorId || + !wake.idempotencyKey?.startsWith("task-retry:")) return "denied"; + const [issue] = await tx.select().from(issues).where(and(eq(issues.id, issueId), + eq(issues.companyId, wake.companyId))).for("update"); + if (!issue || issue.assigneeAgentId !== wake.agentId || ["done", "cancelled"].includes(issue.status)) return "denied"; + if (await retireSupersededTaskRetry(tx as unknown as Db, { companyId: wake.companyId, issueId, + requestId: wake.id, sourceRunId: intent.sourceRunId })) return "denied"; + if (!await taskRetryMutationAllowed(tx as unknown as Db, { companyId: wake.companyId, issueId, + actorId: intent.actorId, actorSource: intent.actorSource })) return "denied"; + if (issue.conversationAgentId && (issue.conversationUserId !== intent.actorId || + !(await instanceSettingsService(tx as unknown as Db).getExperimental()).enableAgentChat)) return "denied"; + if (adapterExecutionControls.has(intent.sourceRunId) || + await hasPendingRunCleanup(tx as unknown as Db, wake.companyId, intent.sourceRunId)) return "waiting"; + return "ready"; + }); + inputDenied = admission === "denied" || !chat || !receipt; + if (!inputDenied && admission === "ready" && receipt?.status === "deferred_issue_execution") { + try { + await resumeDeferredChatWake(db, { companyId: wake.companyId, agentId: wake.agentId, issueId, requestId: actionId }); + } catch (error) { + if (!(error instanceof PersistedExecutionWaitRefusal)) { + if (!(error instanceof HttpError) || ![403, 404, 409].includes(error.status)) throw error; + inputDenied = true; + } + } + [receipt] = await db.select().from(agentWakeupRequests).where(and(eq(agentWakeupRequests.companyId, wake.companyId), + eq(agentWakeupRequests.id, actionId), eq(agentWakeupRequests.agentId, wake.agentId))); + } + inputDenied ||= Boolean(receipt && ["failed", "skipped", "cancelled"].includes(receipt.status)); + } // A deferred chat receipt is admitted input, not a completed dispatch. // Re-stage that exact action through chat's owner; it rechecks authority // and normal queue policy before updating the existing receipt. - if (!scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && + if (!intent.queuedChatRequestId && !scope.settleOnly && chat?.status === "processed" && chat.payload.taskRetryIntentId === wake.id && chat.payload.failedRunId === intent.sourceRunId && receipt?.status === "deferred_issue_execution" && !receipt.runId && receipt.payload?.taskRetryRequestId === wake.id) { await db.transaction(async tx => { @@ -375,7 +431,7 @@ export async function reconcileTaskChatRetryReceipts(db: Db, scope: { companyId? eq(chatActions.status, "processed"), eq(chatActions.updatedAt, chat.updatedAt))); }); } - const failed = chat && ["failed", "cancelled"].includes(chat.status); + const failed = inputDenied || (chat && ["failed", "cancelled"].includes(chat.status)); const write = (writer: Pick) => writer.update(agentWakeupRequests).set({ ...(receipt?.runId ? { status: "coalesced", runId: receipt.runId, finishedAt: new Date() } : failed ? { status: "failed", finishedAt: new Date() } : {}), diff --git a/server/src/services/task-search.ts b/server/src/services/task-search.ts index 600e355217e..9f56b5a7996 100644 --- a/server/src/services/task-search.ts +++ b/server/src/services/task-search.ts @@ -1,3 +1,4 @@ +import { issueDisplayStatusSql } from "./issue-display-status.js"; import { sql, type SQL } from "drizzle-orm"; import { COMPANY_SEARCH_MAX_QUERY_LENGTH, COMPANY_SEARCH_MAX_TOKENS } from "@paperclipai/shared"; import { visibleIssueCondition } from "./issue-visibility.js"; @@ -134,7 +135,7 @@ export function taskSearchCtes(companyId: string, search: TaskSearch, includeCon // The search page fetches descriptions only for its result window. const flags = (fuzzyMatch: SQL) => sql` SELECT issues.id, issues.identifier, issues.title, - issues.status, issues.priority, issues.assignee_agent_id, issues.assignee_user_id, + issues.status, ${issueDisplayStatusSql()} AS display_status, issues.priority, issues.assignee_agent_id, issues.assignee_user_id, issues.project_id, issues.created_at, issues.updated_at, ${identExact} AS ident_exact, ${identStarts} AS ident_starts, ${phraseMatch(sql`issues.identifier`)} AS ident_phrase, diff --git a/server/src/services/tool-access/profiles/projection.ts b/server/src/services/tool-access/profiles/projection.ts index 53b92ef182f..877c2606fd4 100644 --- a/server/src/services/tool-access/profiles/projection.ts +++ b/server/src/services/tool-access/profiles/projection.ts @@ -103,6 +103,24 @@ export function profileEntryMatchesCatalog( return false; } +export function profileCatalogMatcher(entries: ProfileEntryMatch[]) { + const applications = new Set(); + const connections = new Set(); + const catalogIds = new Set(); + const toolNames = new Set(); + const risks = new Set(); + for (const entry of entries) { + if (entry.selectorType === "application") applications.add(entry.applicationId); + else if (entry.selectorType === "connection") connections.add(entry.connectionId); + else if (entry.selectorType === "catalog_entry") catalogIds.add(entry.catalogEntryId); + else if (entry.selectorType === "tool_name") toolNames.add(entry.toolName); + else if (entry.selectorType === "risk_level") risks.add(entry.riskLevel); + } + return (entry: ProfileCatalogMatch) => applications.has(entry.applicationId) + || connections.has(entry.connectionId) || catalogIds.has(entry.id) + || toolNames.has(entry.toolName) || risks.has(entry.riskLevel); +} + export function summarizeProfile(input: { profile: typeof toolProfiles.$inferSelect; entries: Array; @@ -110,22 +128,18 @@ export function summarizeProfile(input: { catalog: ProfileCatalogEntry[]; agentIds: string[]; }): ToolProfileSummary { - const includes = input.entries.filter((entry) => entry.effect === "include"); - const excludes = input.entries.filter((entry) => entry.effect === "exclude"); + const included = profileCatalogMatcher(input.entries.filter((entry) => entry.effect === "include")); + const excluded = profileCatalogMatcher(input.entries.filter((entry) => entry.effect === "exclude")); const allowedCatalogIds = new Set(); const allowedApplicationIds = new Set(); const excludedCatalogIds = new Set(); for (const catalogEntry of input.catalog) { - const excluded = excludes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ); - if (excluded) excludedCatalogIds.add(catalogEntry.id); - if (excluded) continue; - const included = includes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ); - if (input.profile.defaultAction === "allow" || included) { + if (excluded(catalogEntry)) { + excludedCatalogIds.add(catalogEntry.id); + continue; + } + if (input.profile.defaultAction === "allow" || included(catalogEntry)) { allowedCatalogIds.add(catalogEntry.id); if (catalogEntry.applicationId) allowedApplicationIds.add(catalogEntry.applicationId); @@ -203,32 +217,28 @@ export function pendingNewToolsForProfile(input: { return []; const watermark = input.profile.newToolsReviewedAt ?? input.profile.createdAt; const catalogById = new Map(input.catalog.map((entry) => [entry.id, entry])); - const scopedIncludes = input.entries.filter( - (entry) => - entry.effect === "include" && - (entry.selectorType === "application" || - entry.selectorType === "connection" || - entry.selectorType === "catalog_entry"), - ); - if (scopedIncludes.length === 0) return []; + const coveredApplications = new Set(); + const coveredConnections = new Set(); + for (const entry of input.entries) { + if (entry.effect !== "include") continue; + if (entry.selectorType === "application") coveredApplications.add(entry.applicationId); + else if (entry.selectorType === "connection") coveredConnections.add(entry.connectionId); + else if (entry.selectorType === "catalog_entry" && entry.catalogEntryId) { + const scopedEntry = catalogById.get(entry.catalogEntryId); + if (!scopedEntry) continue; + coveredConnections.add(scopedEntry.connectionId); + if (scopedEntry.applicationId) coveredApplications.add(scopedEntry.applicationId); + } + } + if (coveredApplications.size === 0 && coveredConnections.size === 0) return []; + const explicitlyReviewed = profileCatalogMatcher(input.entries); return input.catalog - .filter( - (catalogEntry) => - catalogEntry.status === "active" || - catalogEntry.status === "quarantined", - ) - .filter((catalogEntry) => catalogEntry.firstSeenAt > watermark) .filter((catalogEntry) => - scopedIncludes.some((entry) => - profileCoversCatalogScope({ entry, catalogEntry, catalogById }), - ), - ) - .filter( - (catalogEntry) => - !input.entries.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ), + (catalogEntry.status === "active" || catalogEntry.status === "quarantined") + && catalogEntry.firstSeenAt > watermark + && (coveredApplications.has(catalogEntry.applicationId) || coveredConnections.has(catalogEntry.connectionId)) + && !explicitlyReviewed(catalogEntry), ) .map((catalogEntry) => ({ catalogEntryId: catalogEntry.id, diff --git a/server/src/services/tool-access/profiles/resolution.ts b/server/src/services/tool-access/profiles/resolution.ts index c237636eb2b..b06683b8203 100644 --- a/server/src/services/tool-access/profiles/resolution.ts +++ b/server/src/services/tool-access/profiles/resolution.ts @@ -3,7 +3,7 @@ import type { Db } from "@paperclipai/db"; import { agents, toolCatalogEntries, toolProfileBindings, toolProfileEntries, toolProfiles } from "@paperclipai/db"; import type { ToolCatalogEntry, ToolConnection, ToolProfileEffectiveSummary, ToolProfileWithDetails } from "@paperclipai/shared"; import { effectiveToolProfileBindings, profileIdsInBindingOrder } from "../../tool-profile-binding-precedence.js"; -import { buildProfileDetails, profileCatalogColumns, profileEntryMatchesCatalog, toProfileBinding, toProfileEntry, type ProfileCatalogEntry, type ProfileEntryMatch, type ProfileCatalogMatch } from "./projection.js"; +import { buildProfileDetails, profileCatalogColumns, profileCatalogMatcher, toProfileBinding, toProfileEntry, type ProfileCatalogEntry, type ProfileEntryMatch, type ProfileCatalogMatch } from "./projection.js"; type ProfileAssignmentDependencies = { assertOptionalAgent(companyId: string, agentId: string | null | undefined, label: string): Promise; resolveInstalledConnectionsForAgent(companyId: string, agentId: string): Promise; @@ -37,18 +37,13 @@ function resolveCatalogAccess( const excludes = profileEntries.filter( (entry) => entry.effect === "exclude", ); + const included = profileCatalogMatcher(includes); + const excluded = profileCatalogMatcher(excludes); for (const catalogEntry of catalog) { - if ( - excludes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ) - ) - continue; + if (excluded(catalogEntry)) continue; if ( profile.defaultAction === "allow" || - includes.some((entry) => - profileEntryMatchesCatalog(entry, catalogEntry), - ) + included(catalogEntry) ) { allowedCatalogIds.add(catalogEntry.id); allowedToolNames.add(catalogEntry.toolName); diff --git a/server/src/services/workspace-recovery-source.ts b/server/src/services/workspace-recovery-source.ts index 3cf14003e49..ad65d438d2b 100644 --- a/server/src/services/workspace-recovery-source.ts +++ b/server/src/services/workspace-recovery-source.ts @@ -25,6 +25,10 @@ export class WorkspaceSourceInspectionLimit extends Error {} export function workspaceRecoverySourceEligible(includeCompleted = false) { const actionEligible = sql`(${issueRecoveryActions.status} in ('active', 'escalated') or + (${issueRecoveryActions.status} = 'cancelled' + and ${inArray(issueRecoveryActions.cause, workspaceExportCauses)} + and jsonb_typeof(${issueRecoveryActions.evidence}#>'{workspaceRecovery,requestIds}') = 'array' + and ${issueRecoveryActions.evidence}#>'{workspaceRecovery,requestIds}' <> '[]'::jsonb) or (${issueRecoveryActions.status} = 'resolved' and ( (${issueRecoveryActions.evidence}#>>'{automaticRecovery,policy}' = 'preserve_without_replay_v1' and ${issueRecoveryActions.evidence}#>>'{automaticRecovery,replay}' = 'blocked' diff --git a/server/src/services/workspace-recovery.test.ts b/server/src/services/workspace-recovery.test.ts index 8648acad550..5c732d4f922 100644 --- a/server/src/services/workspace-recovery.test.ts +++ b/server/src/services/workspace-recovery.test.ts @@ -49,6 +49,10 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { agents, approvals, budgetPolicies, costEvents, completionContracts, issueApprovals, issueTreeHolds, issueTreeHoldMembers, companies, createDb, applyPendingMigrations, environmentLeases, environments, heartbeatRuns, issueComments, issueRecoveryActions, issues, nativeRunFinalizations, nativeRunResults, agentWakeupRequests, workspaceOperations } from "@paperclipai/db"; import { resumePreservedResultFinalization } from "./preserved-result-finalization.js"; +import { finalizeNativeRun } from "./native-runtime/native-run-finalizer.js"; +import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js"; +import { issueThreadInteractions } from "@paperclipai/db"; +import { issueRecoveryActionService } from "./issue-recovery-actions.js"; import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target"; import type { EnvironmentLease } from "@paperclipai/shared"; import { startEmbeddedPostgresTestDatabase } from "../__tests__/helpers/embedded-postgres.js"; @@ -190,6 +194,149 @@ describe("automatic retained workspace recovery", () => { const action = async (id: string) => (await db.select().from(issueRecoveryActions).where(eq(issueRecoveryActions.id, id)))[0]!; const run = async (id: string) => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, id)))[0]!; + function boardRecoveryApp() { + const app = express(); + app.use(express.json()); + app.use((req, _res, next) => { req.actor = { type: "board", source: "local_implicit", userId: "local-board" }; next(); }); + app.use("/api", issueRoutes(db, {} as Parameters[1])); + app.use(errorHandler); + return app; + } + + async function acceptedRemoteExport() { + const f = await seed("native", false); + const source = await run(f.runId), workspaceId = randomUUID(); + const remoteCwd = await realpath(f.remote); + const git = (args: string[], cwd = remoteCwd) => new Promise((resolve, reject) => + execFile("git", args, { cwd }, (error, stdout) => error ? reject(error) : resolve(stdout.trim()))); + await git(["init"]); + await git(["add", "."]); + await git(["-c", "user.name=Recovery fixture", "-c", "user.email=recovery@example.invalid", "commit", "-m", "Completed work"]); + const commit = await git(["rev-parse", "HEAD"]); + targets.get(f.lease.id)!.remoteCwd = remoteCwd; + f.runtime.resumeRunLease.mockImplementation(async () => ({ providerLeaseId: f.providerLeaseId, metadata: { remoteCwd } })); + const authority = { schema: "paperclip.remote-workspace.v1", epoch: 1, snapshotId: null, + source: { kind: "empty" }, writerRunId: f.runId, dirty: true, + location: { environmentId: f.lease.environmentId!, leaseId: f.lease.id, providerLeaseId: f.providerLeaseId, remoteCwd } }; + await db.insert(executionWorkspaces).values({ id: workspaceId, companyId: f.companyId, sourceIssueId: f.issueId, + name: "Completed remote work", cwd: remoteCwd, providerType: "cloud_sandbox", mode: "isolated_workspace", + strategyType: "project_primary", status: "active", metadata: { remoteWorkspace: authority } }); + await db.update(issues).set({ executionWorkspaceId: workspaceId }).where(eq(issues.id, f.issueId)); + const [lease] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, f.lease.id)); + await db.update(environmentLeases).set({ executionWorkspaceId: workspaceId, leasePolicy: "reuse_by_environment", + metadata: { ...lease.metadata, remoteCwd, agentId: source.agentId } }).where(eq(environmentLeases.id, lease.id)); + const contractId = randomUUID(), resultId = randomUUID(); + await db.insert(completionContracts).values({ id: contractId, companyId: f.companyId, issueId: f.issueId, + revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "low", + completionAuthority: "agent_claim_policy", incompleteCriteriaPolicy: "preserve_non_terminal", + contractJson: { revision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision, + objective: "Complete the work", criteria: [{ id: "objective", requirement: "Complete the task" }] }, + canonicalSha256: "contract", createdByActorType: "system", createdByActorId: "test" }); + await db.update(heartbeatRuns).set({ completionContractId: contractId, resultJson: {}, runnerProfileJson: { + adapterDispatch: { adapterType: "paperclip_runner" }, + remoteWorkspace: { schema: "paperclip.remote-workspace-run.v1", workspaceId, leaseId: lease.id, + providerLeaseId: f.providerLeaseId, remoteCwd, epoch: 1, state: "prepared", snapshotId: null }, + } }).where(eq(heartbeatRuns.id, f.runId)); + await db.insert(nativeRunResults).values({ id: resultId, companyId: f.companyId, issueId: f.issueId, runId: f.runId, + completionContractId: contractId, schemaStatus: "accepted", serverFingerprint: "result", canonicalSha256: "result", + resultJson: { terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, result: CONTROL_PLANE_CONFORMANCE_RESULT } }); + await db.update(nativeRunFinalizations).set({ resultId, failureCode: "native_workspace_sync_out_retry_exhausted" }) + .where(eq(nativeRunFinalizations.runId, f.runId)); + await db.update(issueRecoveryActions).set({ cause: "native_workspace_sync_out_retry_exhausted", evidence: { runId: f.runId } }) + .where(eq(issueRecoveryActions.id, f.actionId)); + await adoptSystemRecovery(db, f); + return { ...f, source, workspaceId, resultId, git, commit }; + } + + it.each([false, true])("keeps Retry behind an accepted remote export and resumes only its saved result (review pending: %s)", async reviewPending => { + const f = await acceptedRemoteExport(); + const { source, workspaceId, resultId, git, commit } = f; + const interactionId = randomUUID(); + if (reviewPending) await db.insert(issueThreadInteractions).values({ id: interactionId, companyId: f.companyId, + issueId: f.issueId, sourceRunId: f.runId, createdByAgentId: source.agentId, kind: "request_confirmation", + title: "Review saved work", effectiveResolverPolicy: "human_only", + payload: { version: 1, prompt: "Review before publishing", rejectRequiresReason: true, + target: { type: "custom", key: "layout_review", label: "Review saved work" } } }); + await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: source.agentId, status: "running" }); + const app = boardRecoveryApp(); + const intent = { observedRunId: f.runId, idempotencyKey: randomUUID() }; + const retry = (await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body; + expect((await request(app).post(`/api/issues/${f.issueId}/retry`).send(intent).expect(202)).body.requestId).toBe(retry.requestId); + await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); + expect(await db.select().from(heartbeatRuns).where(and(eq(heartbeatRuns.companyId, f.companyId), + sql`coalesce(${heartbeatRuns.nativeIssueId}::text, ${heartbeatRuns.issueId}::text, + ${heartbeatRuns.contextSnapshot}->>'issueId') = ${f.issueId}`))).toHaveLength(1); + expect(await getExecutionBlocker(db, f.companyId, f.issueId)).toMatchObject({ runId: f.runId, workspaceRepairRequired: true }); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => { + await heartbeatService(db).resumeTaskRetry(f.companyId, retry.requestId!); + }) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, workspaceId)); + const saved = readRemoteWorkspaceAuthority(workspace.metadata)!; + const restored = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId, snapshotId: saved.snapshotId! }); + try { + expect(await readFile(path.join(restored.root, "saved.txt"), "utf8")).toBe("finished work\n"); + expect(await git(["rev-parse", "HEAD"], restored.root)).toBe(commit); + } + finally { await restored.cleanup(); } + const [coordinator] = await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)); + expect(coordinator).toMatchObject({ phase: "workspace_finalizing", resultId }); + expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, retry.requestId!)))[0]) + .toMatchObject({ status: "coalesced", runId: f.runId, payload: { taskRetry: { strategy: "finalize_result" } } }); + expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.companyId))).toHaveLength(2); + await finalizeNativeRun({ db, runId: f.runId, workspaceFinalizeStatus: "succeeded", projectRunStatus: true }); + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) + .toMatchObject({ status: reviewPending ? "in_review" : "done", executionRunId: null, checkoutRunId: null }); + expect((await run(f.runId)).status).toBe("succeeded"); + if (reviewPending) expect((await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interactionId)))[0]) + .toMatchObject({ status: "pending", result: null, resolvedAt: null }); + }, 30_000); + + it("preserves an explicitly requested superseded remote export without applying its result over newer work", async () => { + const f = await acceptedRemoteExport(); + const [later] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.source.agentId, + issueId: f.issueId, status: "cancelled", runtimeMode: "legacy", finishedAt: new Date(), + contextSnapshot: { issueId: f.issueId }, createdAt: new Date(f.source.createdAt.getTime() + 1_000) }).returning(); + const [currentWorkspace] = await db.insert(executionWorkspaces).values({ companyId: f.companyId, + sourceIssueId: f.issueId, name: "Newer workspace", cwd: path.join(root, later.id), providerType: "cloud_sandbox", + mode: "isolated_workspace", strategyType: "project_primary", status: "active" }).returning(); + await db.update(issues).set({ executionWorkspaceId: currentWorkspace.id }).where(eq(issues.id, f.issueId)); + const laterAction = await issueRecoveryActionService(db).upsertSourceScoped({ companyId: f.companyId, + sourceIssueId: f.issueId, kind: "active_run_watchdog", cause: "legacy_execution_requires_reconciliation", + fingerprint: later.id, evidence: { runId: later.id }, nextAction: "Inspect the stopped work", + ownerType: "board", supersedeOnIdentityChange: true }); + expect((await action(f.actionId)).status).toBe("cancelled"); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + const app = boardRecoveryApp(); + const recoveryRequest = { actionId: f.actionId, runId: f.runId, idempotencyKey: randomUUID() }; + await request(app).post(`/api/issues/${f.issueId}/recovery-actions/recover-workspace`).send(recoveryRequest).expect(202); + const admitted = await action(f.actionId); + await request(app).post(`/api/issues/${f.issueId}/recovery-actions/recover-workspace`).send(recoveryRequest).expect(202); + expect(await action(f.actionId)).toEqual(admitted); + await db.update(heartbeatRuns).set({ status: "running", finishedAt: null }).where(eq(heartbeatRuns.id, later.id)); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.failure?.code).toBe("workspace_writer_active"); + expect(f.runtime.resumeRunLease).not.toHaveBeenCalled(); + await db.update(heartbeatRuns).set({ status: "cancelled", finishedAt: new Date() }).where(eq(heartbeatRuns.id, later.id)); + await makeRetryDue(f.actionId); + await Promise.all(await reconcileRetainedWorkspaces(db, f.environmentRuntime, async () => {}) ?? []); + expect(readSystemRecovery((await action(f.actionId)).evidence)?.state).toBe("complete"); + const [workspace] = await db.select().from(executionWorkspaces).where(eq(executionWorkspaces.id, f.workspaceId)); + const saved = readRemoteWorkspaceAuthority(workspace.metadata)!; + const restored = await createSavedWorkspaceLoader(db)({ companyId: f.companyId, workspaceId: f.workspaceId, snapshotId: saved.snapshotId! }); + try { + expect(await f.git(["rev-parse", "HEAD"], restored.root)).toBe(f.commit); + expect(await readFile(path.join(restored.root, "saved.txt"), "utf8")).toBe("finished work\n"); + } finally { await restored.cleanup(); } + expect((await db.select().from(issues).where(eq(issues.id, f.issueId)))[0]) + .toMatchObject({ status: "blocked", executionWorkspaceId: currentWorkspace.id }); + expect(await action(laterAction.id)).toEqual(expect.objectContaining({ status: "active", evidence: { runId: later.id } })); + expect((await db.select().from(nativeRunFinalizations).where(eq(nativeRunFinalizations.runId, f.runId)))[0]) + .toMatchObject({ phase: "terminal_failure", resultId: f.resultId }); + expect(await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.companyId, f.companyId))).toEqual([]); + }, 30_000); + it.each([ { status: "failed", runtimeMode: "native", uploadFailure: "none" }, { status: "cancelled", runtimeMode: "native", uploadFailure: "none" }, diff --git a/server/src/storage/s3-multipart.ts b/server/src/storage/s3-multipart.ts index 67db0d7c330..a8744eb8588 100644 --- a/server/src/storage/s3-multipart.ts +++ b/server/src/storage/s3-multipart.ts @@ -1,5 +1,6 @@ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, UploadPartCommand, type S3Client } from "@aws-sdk/client-s3"; -import { Readable } from "node:stream"; +import { once } from "node:events"; +import { PassThrough, Readable } from "node:stream"; import type { PutObjectInput } from "./types.js"; import { withS3FailureDiagnostics } from "./s3-failure.js"; import { cleanupStorageFailure } from "./failure-cleanup.js"; @@ -9,45 +10,86 @@ import { cleanupStorageFailure } from "./failure-cleanup.js"; export async function putS3Multipart(client: S3Client, bucket: string, key: string, input: PutObjectInput) { const source = input.body instanceof Readable ? input.body : Readable.from([input.body]); const iterator = source[Symbol.asyncIterator](); + const controller = new AbortController(); + const active = new Set>(); + const bodies = new Set(); + let failed = false; + let failure: unknown; + const stop = (error: unknown) => { + if (failed) return; + failed = true; + failure = error; + controller.abort(error); + source.destroy(); + for (const body of bodies) body.destroy(); + }; + const check = () => { if (failed) throw failure; }; + const onAbort = () => stop(input.signal!.reason); + input.signal?.addEventListener("abort", onAbort, { once: true }); + if (input.signal?.aborted) onAbort(); + const readNext = async () => { + controller.signal.throwIfAborted(); + let abort!: () => void; + const interrupted = new Promise((_resolve, reject) => { + abort = () => reject(controller.signal.reason); + controller.signal.addEventListener("abort", abort, { once: true }); + }); + try { return await Promise.race([iterator.next(), interrupted]); } + finally { controller.signal.removeEventListener("abort", abort); } + }; let pending = Buffer.alloc(0); let uploadId: string | undefined; // Scale part sizes to the S3 part-count limit while retaining stream backpressure. const partSize = Math.max(8 * 1024 * 1024, Math.ceil(input.contentLength / 10_000)); try { - uploadId = (await withS3FailureDiagnostics("create_multipart_upload", () => client.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: input.contentType }), { abortSignal: input.signal }))).UploadId; + check(); + uploadId = (await withS3FailureDiagnostics("create_multipart_upload", () => client.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key, ContentType: input.contentType }), { abortSignal: controller.signal }))).UploadId; if (!uploadId) throw new Error("S3 did not return a multipart upload id"); const parts: { ETag: string; PartNumber: number }[] = []; + let partNumber = 0; for (let offset = 0; offset < input.contentLength;) { + while (active.size >= 4) { await Promise.race(active); check(); } + check(); const length = Math.min(partSize, input.contentLength - offset); + const number = ++partNumber; let remaining = length; - const body = Readable.from((async function* () { - while (remaining > 0) { - if (!pending.length) { - const next = await iterator.next(); - if (next.done) throw new Error("S3 upload source was truncated"); - pending = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value); - if (!pending.length) continue; - } - const count = Math.min(remaining, pending.length); - const chunk = pending.subarray(0, count); - pending = pending.subarray(count); - remaining -= count; - yield chunk; + const body = new PassThrough({ highWaterMark: 64 * 1024 }); + bodies.add(body); + const uploading = (async () => { + const result = await withS3FailureDiagnostics("upload_part", () => client.send(new UploadPartCommand({ Bucket: bucket, Key: key, UploadId: uploadId, PartNumber: number, Body: body, ContentLength: length }), { abortSignal: controller.signal })); + if (remaining || !body.readableEnded || !result.ETag) throw new Error("S3 upload part is incomplete"); + parts[number - 1] = { ETag: result.ETag, PartNumber: number }; + })().catch(stop).finally(() => { active.delete(uploading); bodies.delete(body); body.destroy(); }); + active.add(uploading); + while (remaining > 0) { + check(); + if (!pending.length) { + const next = await readNext(); + check(); + if (next.done) throw new Error("S3 upload source was truncated"); + pending = Buffer.isBuffer(next.value) ? next.value : Buffer.from(next.value); + if (!pending.length) continue; } - })()); - try { - const result = await withS3FailureDiagnostics("upload_part", () => client.send(new UploadPartCommand({ Bucket: bucket, Key: key, UploadId: uploadId, PartNumber: parts.length + 1, Body: body, ContentLength: length }), { abortSignal: input.signal })); - if (remaining || !result.ETag) throw new Error("S3 upload part is incomplete"); - parts.push({ ETag: result.ETag, PartNumber: parts.length + 1 }); - } finally { body.destroy(); } + const count = Math.min(remaining, pending.length, 64 * 1024); + const chunk = pending.subarray(0, count); + pending = pending.subarray(count); + remaining -= count; + if (!body.write(chunk)) await once(body, "drain", { signal: controller.signal }); + } + body.end(); offset += length; } - if (pending.length || !(await iterator.next()).done) throw new Error("S3 upload source exceeds its declared size"); - await withS3FailureDiagnostics("complete_multipart_upload", () => client.send(new CompleteMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId, MultipartUpload: { Parts: parts } }), { abortSignal: input.signal })); + if (pending.length || !(await readNext()).done) throw new Error("S3 upload source exceeds its declared size"); + await Promise.all(active); + check(); + await withS3FailureDiagnostics("complete_multipart_upload", () => client.send(new CompleteMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId, MultipartUpload: { Parts: parts } }), { abortSignal: controller.signal })); } catch (error) { + stop(error); + await Promise.all(active); if (uploadId) await cleanupStorageFailure(signal => client.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId }), { abortSignal: signal })); - throw error; + throw failure; } finally { + input.signal?.removeEventListener("abort", onAbort); source.destroy(); await cleanupStorageFailure(async () => iterator.return?.()); } diff --git a/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts b/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts index fcb8588483e..efba0a8996a 100644 --- a/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts +++ b/tests/e2e/execution-recovery/failed-attempt-notice.spec.ts @@ -24,10 +24,12 @@ test.describe("failed-attempt notice", () => { return { background: style.backgroundColor, padding: [style.paddingTop, style.paddingRight, style.paddingBottom, style.paddingLeft].map(parseFloat), + topPadding: parseFloat(getComputedStyle(element.parentElement!).paddingTop), }; }); expect(surface.background).not.toBe("rgba(0, 0, 0, 0)"); expect(surface.padding.every(value => value > 0)).toBe(true); + expect(surface.topPadding).toBeCloseTo(16); const iconBounds = (await icon.boundingBox())!; const textBounds = (await message.boundingBox())!; diff --git a/tests/perf/collections/README.md b/tests/perf/collections/README.md index eda8de20f4c..c88a05532ac 100644 --- a/tests/perf/collections/README.md +++ b/tests/perf/collections/README.md @@ -23,6 +23,31 @@ Request counts and row budgets are acceptance checks. They do not establish an end-to-end latency improvement. Measure cold, warm, and Back timings separately with the same fixture, assets, cache, viewport, and CPU/network settings. +`parent-context.spec.ts` also covers a delayed missing-parent index response, +reuse of an already indexed parent, and row geometry through deferred visible +diagnostics. Parent captions retain the initial reveal boundary. Failed-run +context and parent-to-failure cache transitions are covered by `Inbox.test.tsx`. + +To collect timings, serve pinned baseline and candidate production assets from +two loopback origins that proxy the same disposable API. Then run: + +```sh +INBOX_BASELINE_URL=http://127.0.0.1: \ +INBOX_CANDIDATE_URL=http://127.0.0.1: \ + node --import ./server/node_modules/tsx/dist/loader.mjs \ + tests/perf/collections/measure-mine-context.ts +``` + +The script retains a four-parent, 30-child fixture, archives three completed +parents, and enables the personal Coordinator preference. It runs five alternating +pairs by default. `INBOX_PAIRS`, `INBOX_FIXTURE_FILE`, and `INBOX_OUTPUT` override +the count and evidence paths. Each cold sample starts with empty HTTP and query +caches. Warm Mine navigation and task-to-Mine Back retain those caches. The report +keeps every sample, errors, row membership/order, geometry, assets and API timings. +Keep upstream identity/data timings as controls: a faster cold total alone does +not attribute the gain to parent context. Use the same immutable asset cache policy +on both origins and stop builds and suites during timing runs. + Startup cases also check that shared collection controls do not require the task-detail bundle before reveal, even when a different organization is remembered. Code warms after the collection paints, or on task-link intent. A successful diff --git a/tests/perf/collections/measure-mine-context.ts b/tests/perf/collections/measure-mine-context.ts new file mode 100644 index 00000000000..36108383dee --- /dev/null +++ b/tests/perf/collections/measure-mine-context.ts @@ -0,0 +1,170 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { arch, loadavg, platform } from "node:os"; +import { chromium, expect, request, type APIRequestContext } from "@playwright/test"; + +type Fixture = { + company: { id: string; issuePrefix: string }; + parents: Array<{ id: string }>; + children: Array<{ id: string }>; +}; +type Arm = "baseline" | "candidate"; +declare global { + interface Window { __mineContextMeasure: { first: number | null } } +} + +function loopbackOrigin(value: string | undefined, name: string) { + assert.ok(value, `Set ${name} to a disposable local production UI origin.`); + const url = new URL(value); + assert.ok(["http:", "https:"].includes(url.protocol) + && ["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) + && url.pathname === "/" && !url.username && !url.password && !url.search && !url.hash, + `${name} must be an exact loopback origin.`); + return url.origin; +} +const origins = { + baseline: loopbackOrigin(process.env.INBOX_BASELINE_URL, "INBOX_BASELINE_URL"), + candidate: loopbackOrigin(process.env.INBOX_CANDIDATE_URL, "INBOX_CANDIDATE_URL"), +}; +assert.notEqual(origins.baseline, origins.candidate, "Use distinct baseline and candidate UI origins."); +const pairs = Number(process.env.INBOX_PAIRS ?? 5); +assert.ok(Number.isSafeInteger(pairs) && pairs > 0, "INBOX_PAIRS must be a positive integer."); +const fixtureFile = process.env.INBOX_FIXTURE_FILE ?? "test-results/collections/mine-context-fixture.json"; +const output = process.env.INBOX_OUTPUT ?? "test-results/collections/mine-context-timings.json"; +assert.notEqual(path.resolve(fixtureFile), path.resolve(output), "Fixture and report paths must differ."); + +async function api(client: APIRequestContext, method: string, url: string, data?: unknown) { + const response = await client.fetch(`/api${url}`, { method, data }); + assert.ok(response.ok(), `${method} ${url} returned HTTP ${response.status()}.`); + return response.status() === 204 ? null : response.json(); +} +async function retainFixture(client: APIRequestContext): Promise { + const saved = await fs.readFile(fixtureFile, "utf8").then(JSON.parse).catch(error => { + if (error.code !== "ENOENT") throw error; + return null; + }) as Fixture | null; + if (saved) { + assert.ok(saved.company?.id && saved.company.issuePrefix + && saved.parents.length === 4 && saved.children.length === 30, "Invalid retained fixture."); + return saved; + } + const company = await api(client, "POST", "/companies", { name: "Mine parent context performance" }); + const fixture: Fixture = { company: { id: company.id, issuePrefix: company.issuePrefix }, parents: [], children: [] }; + const create = async (title: string, status: string, parentId?: string) => { + const row = await api(client, "POST", `/companies/${company.id}/issues`, { title, status, parentId }); + return { id: row.id as string }; + }; + for (let index = 0; index < 4; index++) { + fixture.parents.push(await create(`Parent ${index}`, index === 0 ? "blocked" : "done")); + } + for (let index = 0; index < 30; index++) { + fixture.children.push(await create(`Inbox child ${index}`, "todo", fixture.parents[index % 4]!.id)); + } + for (const parent of fixture.parents.slice(1)) await api(client, "POST", `/issues/${parent.id}/inbox-archive`, {}); + await fs.mkdir(path.dirname(fixtureFile), { recursive: true }); + await fs.writeFile(fixtureFile, JSON.stringify(fixture, null, 2)); + return fixture; +} + +const clients = { + baseline: await request.newContext({ baseURL: origins.baseline }), + candidate: await request.newContext({ baseURL: origins.candidate }), +}; +const samples: Array> = []; +let expectedRowIds: Array | undefined; +let fixture: Fixture | undefined; +let browser: Awaited> | undefined; +const save = async () => { + await fs.mkdir(path.dirname(output), { recursive: true }); + await fs.writeFile(output, JSON.stringify({ + pairs, fixture, origins, rowSequenceReference: expectedRowIds, + settings: { viewport: "1440x900", httpCache: "fresh context", serviceWorkers: "blocked", + queryCache: "empty for cold, retained for warm and Back", cpu: "native", network: "loopback", + readiness: "cold: browser clock to first Mine row; warm/Back: host clock around browser actions", + node: process.version, platform: platform(), arch: arch(), browser: browser?.version() }, + samples, + }, null, 2)); +}; +try { + for (const arm of ["baseline", "candidate"] as const) { + const response = await clients[arm].get("/"); + assert.ok(response.ok(), `${arm} UI is unavailable.`); + const html = await response.text(); + assert.ok(/src=["']\/assets\/[^"']+\.js["']/.test(html) && !html.includes("/@vite/client"), + `${arm} must serve a production UI build.`); + } + fixture = await retainFixture(clients.candidate); + for (const arm of ["baseline", "candidate"] as const) await api(clients[arm], "GET", `/companies/${fixture.company.id}`); + await api(clients.candidate, "PUT", `/companies/${fixture.company.id}/workspace-preferences/me`, { coordinatorWorkspaceEnabled: true }); + browser = await chromium.launch(); + for (let pair = 0; pair < pairs; pair++) { + const order: Arm[] = pair % 2 ? ["candidate", "baseline"] : ["baseline", "candidate"]; + for (const arm of order) { + const context = await browser.newContext({ viewport: { width: 1440, height: 900 }, serviceWorkers: "block" }); + const page = await context.newPage(); + const errors: string[] = []; + const reads: Array<{ view: string | null; ids: number }> = []; + let rowIds: Array | undefined; + page.on("pageerror", error => errors.push(error.message)); + page.on("request", request => { + const url = new URL(request.url()); + if (url.pathname.endsWith("/issues")) reads.push({ view: url.searchParams.get("view"), ids: url.searchParams.get("ids")?.split(",").length ?? 0 }); + }); + try { + await page.addInitScript(() => { + window.__mineContextMeasure = { first: null }; + const observer = new MutationObserver(() => { + if (window.__mineContextMeasure.first === null && location.pathname.endsWith("/inbox/mine") && document.querySelector("[data-inbox-item]")) { + window.__mineContextMeasure.first = performance.now(); + observer.disconnect(); + } + }); + observer.observe(document, { subtree: true, childList: true }); + }); + await page.goto(`${origins[arm]}/${fixture.company.issuePrefix}/inbox/mine`); + const rows = page.locator("[data-inbox-item]"); + await expect(rows.first()).toBeVisible(); + const cold = await page.evaluate(() => window.__mineContextMeasure.first); + assert.ok(cold !== null, "Cold readiness observer did not see a row."); + rowIds = await page.locator("[data-inbox-issue-link]").evaluateAll(nodes => nodes.map(node => node.getAttribute("href"))); + const firstBox = await rows.first().boundingBox(); + await page.getByRole("tab", { name: "Recent", exact: true }).click(); + await expect(page.getByRole("tab", { name: "Recent", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const start = performance.now(); + await page.getByRole("tab", { name: "Mine", exact: true }).click(); + await expect(page.getByRole("tab", { name: "Mine", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const warm = performance.now() - start; + await page.locator("[data-inbox-issue-link]").first().click(); + await expect(page.getByTestId("issue-detail-header")).toBeVisible(); + const backStart = performance.now(); + await page.goBack(); + await expect(page.getByRole("tab", { name: "Mine", exact: true })).toHaveAttribute("aria-selected", "true"); + await expect(rows.first()).toBeVisible(); + const back = performance.now() - backStart; + const resources = await page.evaluate(() => performance.getEntriesByType("resource").map(entry => { + const resource = entry as PerformanceResourceTiming; + const url = new URL(resource.name); + return { path: url.pathname, start: resource.startTime, ms: resource.duration, end: resource.responseEnd }; + }).filter(resource => resource.path.startsWith("/api/") || resource.path.startsWith("/assets/"))); + assert.deepEqual(errors, [], "The page reported JavaScript errors."); + expectedRowIds ??= rowIds; + assert.deepEqual(rowIds, expectedRowIds, "Mine row order or membership changed between samples."); + samples.push({ pair, arm, status: "passed", cold, warm, back, rowIds, rowSequenceEquivalent: true, firstBox, reads, resources, errors, load: loadavg() }); + console.log(JSON.stringify({ pair, arm, cold, warm, back, rows: rowIds.length, errors })); + } catch (error) { + samples.push({ pair, arm, status: "failed", error: error instanceof Error ? error.message : String(error), rowIds, errors, reads, load: loadavg() }); + throw error; + } finally { + await save(); + await context.close(); + } + } + } +} finally { + await save(); + await browser?.close(); + await Promise.all(Object.values(clients).map(client => client.dispose())); +} diff --git a/tests/perf/collections/parent-context.spec.ts b/tests/perf/collections/parent-context.spec.ts new file mode 100644 index 00000000000..a7065d340da --- /dev/null +++ b/tests/perf/collections/parent-context.spec.ts @@ -0,0 +1,78 @@ +import { expect, test } from "@playwright/test"; + +let companyId: string | undefined; +test.afterEach(async ({ request }) => { + if (!companyId) return; + const response = await request.delete(`/api/companies/${companyId}`); + expect(response.ok(), await response.text()).toBeTruthy(); + companyId = undefined; +}); + +test("Mine waits for missing parent metadata without loading parent diagnostics", async ({ page, request }) => { + const created = await request.post("/api/companies", { data: { name: "Parent context browser check" } }); + expect(created.ok(), await created.text()).toBeTruthy(); + const company = await created.json() as { id: string; issuePrefix: string }; + companyId = company.id; + const preference = await request.put(`/api/companies/${company.id}/workspace-preferences/me`, { + data: { coordinatorWorkspaceEnabled: true }, + }); + expect(preference.ok(), await preference.text()).toBeTruthy(); + const create = async (title: string, status: string, parentId?: string) => { + const response = await request.post(`/api/companies/${company.id}/issues`, { data: { title, status, parentId } }); + expect(response.ok(), await response.text()).toBeTruthy(); + return response.json() as Promise<{ id: string; identifier: string; title: string }>; + }; + const loaded = await create("Indexed parent context", "blocked"); + const missing = await create("Archived parent context", "done"); + const first = await create("Child of indexed parent", "todo", loaded.id); + const second = await create("Child of archived parent", "todo", missing.id); + const archived = await request.post(`/api/issues/${missing.id}/inbox-archive`, { data: {} }); + expect(archived.ok(), await archived.text()).toBeTruthy(); + + const reads: Array<{ projection: string | null; ids: string[] }> = []; + const contextRead = (url: URL) => url.pathname === `/api/companies/${company.id}/issues` + && url.searchParams.get("view") === "index" + && url.searchParams.get("ids")?.split(",").includes(missing.id) === true; + page.on("request", request => { + const url = new URL(request.url()); + if (url.pathname === `/api/companies/${company.id}/issues` && url.searchParams.has("ids")) { + reads.push({ projection: url.searchParams.get("view"), ids: url.searchParams.get("ids")!.split(",") }); + } + }); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let releaseDetails!: () => void; + const detailsHeld = new Promise(resolve => { releaseDetails = resolve; }); + const detailRead = (url: URL) => url.pathname === `/api/companies/${company.id}/issues` + && url.searchParams.get("view") === "rows" + && url.searchParams.get("ids")?.split(",").includes(second.id) === true; + await page.route(detailRead, async route => { await detailsHeld; await route.continue(); }); + await page.route(contextRead, async route => { await held; await route.continue(); }); + const parentRequest = page.waitForRequest(request => contextRead(new URL(request.url()))); + try { + await page.goto(`/${company.issuePrefix}/inbox/mine`); + await parentRequest; + expect(await page.locator("[data-inbox-item]").count()).toBe(0); + expect(reads.some(read => read.projection === "rows")).toBe(false); + expect(reads.some(read => read.ids.includes(loaded.id))).toBe(false); + } finally { release(); } + + const child = page.locator(`[data-inbox-issue-link][href$="/${second.identifier}"]`); + await expect(child).toBeVisible(); + const node = await child.elementHandle(); + const row = child.locator(".."); + // Visible task diagnostics remain deferred, and their completion must retain + // the title's node, position and row height. + await expect.poll(() => reads.some(read => read.projection === "rows" && read.ids.includes(second.id))).toBe(true); + const before = await row.boundingBox(); + const detailsResponse = page.waitForResponse(response => detailRead(new URL(response.url()))); + releaseDetails(); + await (await detailsResponse).finished(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await expect(page.locator(`[data-inbox-issue-link][href$="/${first.identifier}"]`)).toBeVisible(); + expect(reads.filter(read => read.ids.includes(missing.id)).every(read => read.projection === "index")).toBe(true); + expect(await node!.evaluate(element => element.isConnected)).toBe(true); + const after = await row.boundingBox(); + expect(Math.abs(before!.height - after!.height)).toBeLessThan(1); + expect(Math.abs(before!.y - after!.y)).toBeLessThan(1); +}); diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index a66cf38a374..a2267a42242 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -181,82 +181,31 @@ backlog is 5/5 and delegation is 5/6. The local `results/chat-feature-recheck/verification.md` records the follow-up evidence; its standard dashboard is explicitly scoped to the latest campaign. -## TestOps completion pilot +## Retired TestOps plans -The separate `Chat completion TestOps` workflow connects only these preserved -catalog cases to the private Paperclip Evals project: +The one-case **Chat feature evals** plan and three-case +**Chat completion pilot — OpenCode 1017** plan are retired. Their scenarios are +included in the [full catalog](#full-catalog-in-testops). Use that plan or select +cases from it; do not recreate the smoke or pilot plans. -- `projects.project-work` -- `projects.project-defaults` -- `instance-operations.restart-service` +The former `chat-feature-testops.yml` and `chat-completion-testops.yml` workflows +are removed. Their bootstrap functions reject registration before any API call +or mutation marker. Shared transport, evidence, export and verification helpers +remain for the full-catalog workflow and retained historical evidence. Retiring a +plan does not delete its cases, results, launches or private evidence. -It evaluates the immutable catalog source in -`chat-completion-testops-contract.mjs`. This keeps the broader catalog on its -integration branch. The source includes the bounded TestOps host prerequisite -from #18515. Changing the pin requires source review. The original DeepSeek -1017 model, case clocks, independent checks, and zero retries remain fixed. -Source registration does not establish a new live result. - -Run the named plan **Chat completion pilot — OpenCode 1017** in TestOps. The -adapter validates the actual project, plan, job, launch and selected cases before -build or provider access. A selected subset runs only those cases. The AWS matrix -has concurrency one and does not cancel sibling cases after a failure. - -The main-only reporter uses the existing protected `ALLURE_TOKEN`; the evaluated -app and worker never receive it. Original evidence stays in the existing private -AWS storage with the unchanged validators, scanners, byte bounds and exact-key -policies. A separate trusted projection carries verdicts, checks, accounting and -private-evidence receipts. The final PNGs are verified in protected storage; -only their checksums and availability are recorded in TestOps. GitHub artifacts -contain metadata only. Raw logs, screenshots, hidden -reasoning, API snapshots, credentials and provider sessions are not attachments. -Original failed grades and cleanup failures remain failed. Same-run readback -requires matching results and complete result-owned attachment metadata within -the existing bounded processing window; it does not claim remote byte hashes. - -Initial setup uses the existing workflow operations: - -1. Dispatch reviewed `master` once with `operation=bootstrap`. This creates or - reconciles only the three exact automated cases and the dedicated plan. It - registers the GitHub job with an empty Allure results directory. It reuses - only the existing enabled GitHub integration from - `CHAT_TESTOPS_INTEGRATION_ID`. No model or fabricated result is created. - The observation receipt captures a closed parameter-shape inspection before - binding. Only the observed single `Branch/defaultValue` definition for - `refs/heads/master` is accepted. Bootstrap enables the job and assigns cases - while preserving that definition; the plan uses its default branch. - Assignment inspection records unassigned and existing-job counts before - binding. A single unassigned bucket must cover the exact selected cases; - foreign jobs and unknown or incomplete buckets stop before writes. -2. Read the closed registration receipt. Store its observed `binding` JSON as - `CHAT_COMPLETION_TESTOPS_BINDING` in `runner-e2e-testops-reporting`. -3. Use TestOps **Run**. For the initial operator launch, set an exact next - workflow run number in `CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER`, set a new - 32-hex `CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE`, then dispatch `launch-plan` - once. Its request produces the TestOps-owned `evaluate` dispatch. Do not - manually dispatch an additional evaluation after an uncertain launch. -4. Verify the fresh TestOps launch, all selected case results, source links, - independent checks, private-evidence receipts and cleanup. Inspect screenshots - through authenticated private-history retrieval. Retain earlier failed attempts. - -Launch preflight searches use TestOps' typed name filter, encoded as Base64 of -UTF-8 JSON in the `search` query parameter. Plain name text returns HTTP 400. -The backlog and completion launchers share this encoding; the full-catalog -launcher uses the completion adapter. Existing nonce and once-only guards still -apply before a plan-run request. - -The runtime accepts only `operation`, `ALLURE_JOB_RUN_ID` and reserved username -metadata. It rejects arbitrary source/model/budget selectors, branch workflows, -replays and foreign cases. All secret-bearing jobs recheck actual GitHub run and -actor identities. The existing one-case backlog plan remains independent. +Retained launch helpers encode TestOps name filters as Base64 of UTF-8 JSON in +`search`. Plain name text returns HTTP 400. Their nonce and once-only guards +remain unchanged; this transport contract does not restore the retired plans. ## Full catalog in TestOps The [Chat catalog TestOps workflow](../../.github/workflows/chat-catalog-testops.yml) registers all 244 scenarios from the 35 YAML areas in a separate **Chat feature catalog — OpenCode 1017 (candidate)** plan. Its source-derived -manifest is `chat-full-catalog-testops-manifest.json`. The existing backlog plan -and three-case completion pilot keep their own jobs, selections and history. +manifest is `chat-full-catalog-testops-manifest.json`. This is the active chat +plan. The former backlog and three-case pilot plans are retired; their cases +and measured history are preserved. Legacy aliases reuse existing TestOps case identities without renaming them. The catalog contains 236 Agent Chat routes, five browser boundaries, one external @@ -301,19 +250,32 @@ Initial setup uses the existing reporting environment: evaluation after an uncertain request. Initial route verification uses a small selection and does not qualify all 244 cases. -Each worker executes one case. Selections of up to three cases run serially. -Larger selections allow eight independent cases and one external-input case at -once. External-input cases remain serial because they can share QA resources. -A full run can take hours. Its elapsed time depends on setup and fleet capacity; -the declared case clocks are not a completion-time guarantee. Each worker and -report reader has a singleton private-evidence approval and exact-key AWS -session; the existing eight-cell -authority limit is unchanged. Reports wait for all selected workers. A -credential-free job then validates and merges their metadata projections. The -protected reporter uploads original outcomes and verifies every selected result, -including paginated selections over 100 cases. Failed originals and failed -cleanup remain failed beside passed results. Missing originals cannot become -fabricated measurements or a full-selection pass. +Each worker executes one case. Selections of up to three cases allow two cases +at once. Larger selections allow eight independent cases and one external-input +case at once. External-input cases remain serial because they can share QA +resources. A full run can take hours. Its elapsed time depends on setup and fleet +capacity; the declared case clocks are not a completion-time guarantee. + +Each finished worker starts its own protected reader and safe exporter. A +separate reporting job uploads that case and verifies its result and two JSON +attachments while other workers continue. The reader has only an exact-key AWS +session. The worker has only its declared model keys. Only the uploader has the +TestOps key. The existing eight-cell private authority limit is unchanged. + +The exporter freezes each UUID-bearing Allure bundle once. An uncertain upload +response triggers readback, never an automatic resend. The final job verifies +complete selected-case coverage and the original statuses without uploading the +results again. Failed originals and failed cleanup remain failed beside passed +results. Missing originals cannot become fabricated measurements or a +full-selection pass. Backend ingestion does not guarantee that an open TestOps +page refreshes without user action. + +The caller and reusable case workflow must both be admitted at their exact +`refs/heads/master` paths in the existing runner group. Called jobs use their own +protected environments; the caller passes empty named secret placeholders and +never inherits or forwards actual model or reporting keys. Streaming runs use +the fixed `chat-catalog-testops-streaming-v1` serialization group. Runs started +before this change retain their original batch-reporting flow. Raw evidence and final PNGs remain in protected storage. Public GitHub artifacts and TestOps attachments contain only allowlisted provenance, checks, accounting @@ -322,77 +284,3 @@ and credentials do not enter the public artifact path. Additional credentials reach only cases that declare them. External inputs use the existing selected-case private-input owner, disposable-resource checks and oracle commitments. Unconfigured inputs remain setup blockers. - -## TestOps-backed AWS backlog cell - -The dedicated `Chat feature TestOps` workflow accepts only the shipped default -branch and the fixed OpenCode backlog cell. `evaluate` is the default operation -and requires TestOps' `ALLURE_JOB_RUN_ID`. A protected reporting job verifies the -project, singleton plan, assigned public workflow, selected case and current -GitHub run before any build or provider access. The evaluated process receives -no TestOps token. It retains the original one-query, one-work-run, 120-second -turn, 420-second attempt, 300-second settled-state and zero-retry limits. - -Two metadata operations prepare that route. `bootstrap` registers the actual -public CI job through pinned `allurectl` with an empty results directory and -reconciles only the dedicated case and singleton plan. It produces no passing -test result. After the registered job and protected case/plan/job identifiers -are verified, `launch-plan` sends one TestOps plan-run request and exits. -The TestOps job then dispatches `evaluate`; an operator must not dispatch a -second evaluation manually. Missing or ambiguous registration and selection -metadata stops before inference. - -The workflow reuses the existing AWS fleet and strict private evidence storage. -Its shipped-main authority is separate from manual candidate approval. Original -logs, provider records and failure evidence remain private. The reporting adapter -exports only a source/run-bound normalized result, JUnit and screenshots already -marked by the owned fixture screenshot producer with matching file hashes. -A separate protected reporting job converts that projection to the exact TestOps -case. Failed grades and cleanup failures remain failed. - -Reporting setup uses the master-only `runner-e2e-testops-reporting` environment, -with the existing reporting token delivered through the reviewed sealed-box -bootstrap. `CHAT_TESTOPS_CASE_ID`, `CHAT_TESTOPS_PLAN_ID` and -`CHAT_TESTOPS_JOB_ID` must name the observed dedicated mapping; the numeric -`CHAT_TESTOPS_GITHUB_WORKFLOW_ID` supports TestOps' registered workflow URL. -No arbitrary source, model, case, retry or budget input is accepted. - -The initial `launch-plan` administrator operation also requires a protected -`CHAT_TESTOPS_LAUNCH_RUN_NUMBER`. After registration, the trusted operator creates -this variable once with the reviewed next workflow run number and reads it back. -The launcher compares GitHub's actual run number, attempt, workflow, source and -actors before any TestOps mutation. A later dispatch or replay cannot reuse that -approved number. Missing or uncertain metadata stops; an uncertain request needs -reconciliation rather than a new send. The nonce and launch search provide -additional evidence. Normal TestOps **Run** uses `evaluate` and is independent of -this one-time administrator fence. The launch nonce is protected configuration, -not a dispatch input. Job parameter IDs are opaque. The launcher preserves the -API's exact ID/value pair for `refs/heads/master` and uses the fixed workflow's -`evaluate` default. An operation parameter is used only when the protected -`CHAT_TESTOPS_OPERATION_PARAMETER_ID` matches a returned ID with a `bootstrap` -or `evaluate` value; the launcher then sets it to `evaluate`. Other values, -duplicate IDs and conflicting configuration are rejected. No relationship to -global environment-variable IDs or names is inferred. - -`CHAT_TESTOPS_INTEGRATION_ID` names the reviewed existing project GitHub -integration. The launcher verifies its project, type and enabled state, then binds -only the newly registered public job to it. It never changes integration settings -or credentials. Since the job response does not expose that association, only the -real TestOps-to-GitHub dispatch can qualify it. - -After `allurectl watch` finishes, the same protected reporter checks the actual -TestOps job-run, launch, selected case, result source/status and attachment -metadata. Processing reads are bounded to 12 polls, five-second gaps and a -120-second processing deadline. Unknown or missing results remain unverified. -The check downloads no attachment content and claims no server-side attachment -byte hash. Its closed receipt is retained separately. An upload success alone -is not a passing feature result, and verifying an original failure cannot turn -that failure into a pass. - -Build and worker installs use the existing reviewed report-lock policy. Only its -committed input or output lock is accepted; its authenticated correction is the -only allowed normalization. Dependency manifests, patch inputs and policy must -remain unchanged. Installs are frozen with lifecycle scripts disabled. The -credential-free build rechecks that boundary before packaging, and the worker -matches the source/lock receipt before installing. Arbitrary resolver drift is -rejected before provider access. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 9b76f43b535..65cfaa34ea0 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -847,8 +847,16 @@ and asserts `created`, `resumed`, `resumed` lease acquisition on one sandbox. Runner Codex additionally proves stable native session, provider session, runner instance, PID, and process-start identity. The ordinary warm cell uses managed instructions and edits AGENT_HOME on every turn: a growing memory file, -an unchanged 8 MiB binary, and a deletion. Public API reads independently verify -the canonical bytes after every turn. Native checkpoint receipts must show only +an unchanged 8 MiB binary, and a deletion. The temporary file contains exactly +the nine ASCII bytes `temporary` without a trailing newline, so its transfer +count is explicit in the prompt. Public API reads independently verify +the canonical bytes after every turn. Save receipts come from the latest +`instruction_save` event in the complete, paginated run log. Missing receipts, +later failed saves, or events from another run fail the check; a successful run +summary cannot replace this evidence. Definition version 5 preserves earlier +results under their original receipt lookup. Review actions target the actual +interaction card by its test ID and interaction ID, so the request navigation +button cannot make the locator ambiguous. Native checkpoint receipts must show only the changed memory file transferred on turns 2 and 3; the PID oracle remains strict. The journal stress cell retains fixed external instructions as a control. Each warm turn is bounded to ten minutes, the cell to thirty minutes, and cleanup explicitly deletes the diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 1520f034b4c..d6e41627457 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -408,6 +408,10 @@ describe("runner E2E catalog", () => { const suite = runnerSuites.find( (candidate) => candidate.id === "daytona-warm-continuity", )!; + expect(suite.definitionMetadata).toMatchObject({ version: 5, + managedSaveReceipt: "latest-paginated-run-log-instruction-save" }); + expect(suiteDefinitionHash(suite)).not.toBe(suiteDefinitionHash({ ...suite, + definitionMetadata: { version: 4, nativeInstructions: "managed-incremental", managedFileBytes: 8 * 1024 * 1024 } })); expect( suiteDefinitionHash({ ...suite, diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index c05ef65fd52..d6b65064441 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -990,7 +990,7 @@ function managedWarmTurnInstructions(turn: 1 | 2 | 3, nonce: string) { return [warmTurnInstructions(turn, nonce), "Also update your personal AGENT_HOME with ordinary filesystem tools. Do not edit the loaded AGENTS.md instructions and do not use an API to save these files.", turn === 1 - ? `Create notes/warm-memory.txt containing exactly T1-${nonce} followed by a newline. Create notes/unchanged.bin with exactly 8388608 bytes, each byte equal to 93. Create notes/delete-me.txt containing temporary.` + ? `Create notes/warm-memory.txt containing exactly T1-${nonce} followed by a newline. Create notes/unchanged.bin with exactly 8388608 bytes, each byte equal to 93. Create notes/delete-me.txt with exactly the nine ASCII bytes temporary and no trailing newline.` : `Read notes/warm-memory.txt under AGENT_HOME and verify it contains exactly the prior turn lines ${Array.from({ length: turn - 1 }, (_, i) => `T${i + 1}-${nonce}`).join(" | ")}, each followed by a newline. Append exactly T${turn}-${nonce} and a newline. Verify notes/unchanged.bin still has 8388608 bytes, each equal to 93, and leave it unchanged. ${turn === 2 ? "Delete notes/delete-me.txt." : "Verify notes/delete-me.txt is absent."}`, "Perform these personal-file edits and verification before calling paperclip_finish. Paperclip saves them at the turn boundary.", ].join("\n"); @@ -1852,7 +1852,8 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environments: [daytonaWarmEnvironment], tasks: [daytonaWarmContinuityTask], expectedMatrixSize: 2, - definitionMetadata: { version: 4, nativeInstructions: "managed-incremental", managedFileBytes: 8 * 1024 * 1024 }, + definitionMetadata: { version: 5, nativeInstructions: "managed-incremental", managedFileBytes: 8 * 1024 * 1024, + managedSaveReceipt: "latest-paginated-run-log-instruction-save" }, }, { id: "daytona-journal-continuity", diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.mjs new file mode 100644 index 00000000000..6d27e7d30ae --- /dev/null +++ b/tests/runner-e2e/chat-catalog-testops-streaming.mjs @@ -0,0 +1,134 @@ +import { lstat, readFile, readdir, realpath } from 'node:fs/promises'; +import path from 'node:path'; +import { hash, validateSelection, reject } from './chat-completion-testops-contract.mjs'; +import { readPilotExport, exportPilotAllure } from './chat-completion-testops-results.mjs'; +import { inspectPilotCaseUpload, verifyPilotCaseUpload } from './chat-completion-testops-api.mjs'; + +export function caseSelection(selection, context, executionId) { + if (context.catalog !== 'full') reject('stream_scope'); + const authorized = validateSelection(selection, context, selection.allureJobRunId); + const row = authorized.find(value => value.executionId === executionId); + if (!row) reject('stream_scope'); + return { ...selection, cases: [row] }; +} +const identity = (selection, context, executionId, selectionSha256) => { + if (!/^[a-f0-9]{64}$/.test(selectionSha256)) reject('stream_binding'); + const scoped = caseSelection(selection, context, executionId), row = scoped.cases[0]; + return { schema: 'paperclip.chat-catalog-testops-case-bundle/v1', ...context, + projectId: 1, planId: selection.planId, jobId: selection.jobId, launchId: selection.launchId, + allureJobRunId: selection.allureJobRunId, executionId, caseId: row.caseId, + fullSelectionSha256: selectionSha256, fullSelectionCount: selection.cases.length }; +}; +async function filesAt(root, wanted) { + if (await realpath(root) !== path.resolve(root)) reject('stream_bundle'); + const names = (await readdir(root)).sort(); + if (names.length !== 3 || names.some(name => !/^[a-f0-9-]{36}-(?:attachment|result)\.json$/.test(name)) + || names.filter(name => name.endsWith('-result.json')).length !== 1) reject('stream_bundle'); + const files = []; let total = 0; + for (const name of names) { + const file = path.join(root, name), before = await lstat(file); + if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > 12 * 1024 * 1024) reject('stream_bundle'); + const bytes = await readFile(file), after = await lstat(file); total += bytes.length; + if (total > 32 * 1024 * 1024 || before.ino !== after.ino || before.dev !== after.dev || before.size !== bytes.length + || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs) reject('stream_bundle'); + files.push({ name, bytes, sha256: hash(bytes) }); + } + const metadata = files.map(file => ({ name: file.name, bytes: file.bytes.length, sha256: file.sha256 })); + if (wanted && JSON.stringify(metadata) !== JSON.stringify(wanted)) reject('stream_bundle'); + return { files, metadata }; +} + +// The existing exporter owns grades and the public allowlist. Materialize once; +// preserve these exact UUID-bearing bytes through every later readback. +export async function prepareCaseBundle(root, output, context, selection, executionId, selectionSha256) { + const bound = identity(selection, context, executionId, selectionSha256); + const scoped = caseSelection(selection, context, executionId); + const exported = await exportPilotAllure(root, output, context, scoped); + const { metadata: files } = await filesAt(output); + const prepared = { ...bound, + projectionBindingSha256: hash(await readFile(path.join(root, 'binding.json'))), exported, files }; + return { ...prepared, payloadSha256: hash(JSON.stringify(prepared)) }; +} + +export async function readCaseBundle(root, output, prepared, context, selection, executionId, selectionSha256) { + const expected = identity(selection, context, executionId, selectionSha256); + if (!prepared || Object.entries(expected).some(([key, value]) => prepared[key] !== value)) reject('stream_binding'); + const { payloadSha256, ...unsigned } = prepared; + if (!/^[a-f0-9]{64}$/.test(payloadSha256) || hash(JSON.stringify(unsigned)) !== payloadSha256) reject('stream_bundle'); + const scoped = caseSelection(selection, context, executionId); + const validated = await readPilotExport(root, context, scoped), original = validated.campaign.results[0]; + if (hash(await readFile(path.join(root, 'binding.json'))) !== prepared.projectionBindingSha256 + || !Array.isArray(prepared.exported) || prepared.exported.length !== 1 + || prepared.exported[0].caseId !== expected.caseId || prepared.exported[0].executionId !== executionId + || prepared.exported[0].status !== (original.status === 'passed' && original.cleanup === 'passed' ? 'passed' : 'failed') + || prepared.exported[0].cleanup !== original.cleanup) reject('stream_binding'); + return filesAt(output, prepared.files); +} + +export async function publishCaseBundle(api, root, output, prepared, context, selection, executionId, selectionSha256, markOnce, options) { + const { files } = await readCaseBundle(root, output, prepared, context, selection, executionId, selectionSha256); + const before = await inspectPilotCaseUpload(api, selection, context, executionId, options); + let response, uncertain = null; + if (!before.present) { + // Once markers are retained before this call. An uncertain reply is reconciled + // by GET only. Neither this function nor the finalizer repeats the POST. + await markOnce({ executionId, caseId: prepared.caseId, payloadSha256: prepared.payloadSha256 }); + const body = new FormData(); + for (const file of files) body.append('file', new Blob([file.bytes], { type: 'application/json' }), file.name); + try { response = await api(`/api/jobrun/${selection.allureJobRunId}/upload`, 'POST', body); } + catch (error) { uncertain = { kind: ['testops_api_http', 'testops_response_invalid', 'testops_response_too_large'].includes(error?.code) + ? error.code : 'transport_or_response_failure', httpStatus: Number.isInteger(error?.status) ? error.status : null }; } + } + let verification; + try { verification = await verifyPilotCaseUpload(api, selection, context, executionId, prepared.exported, options); } + catch (error) { + if (uncertain) { const stopped = new Error('chat_completion_testops_stream_upload_uncertain'); stopped.uploadFailure = uncertain; throw stopped; } + throw error; + } + const after = await api(`/api/jobrun/${selection.allureJobRunId}`); + if (after.id !== Number(selection.allureJobRunId) || after.job?.id !== selection.jobId || after.launchId !== selection.launchId + || after.url !== `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}` || !['in_progress', 'finished'].includes(after.stage)) reject('jobrun_mismatch'); + return { ...identity(selection, context, executionId, selectionSha256), schema: 'paperclip.chat-catalog-testops-case-publication/v1', + payloadSha256: prepared.payloadSha256, projectionBindingSha256: prepared.projectionBindingSha256, exported: prepared.exported[0], + uploadSent: !before.present, uncertainUpload: uncertain, jobRunStageBefore: before.jobRunStage, jobRunStageAfter: after.stage, + responseFilesCount: Number.isSafeInteger(response?.filesCount) && response.filesCount >= 0 ? response.filesCount : null, + verification: verification.verification, results: verification.results }; +} + +// The finalizer authenticates the already-emitted bytes and their original +// grades. It never calls the random-UUID exporter or the upload endpoint. +export async function publicationSet(bundlesRoot, publicationsRoot, context, selection, selectionSha256) { + const rows = validateSelection(selection, context, selection.allureJobRunId); + const roots = [bundlesRoot, publicationsRoot]; + for (const [index, root] of roots.entries()) { + if (await realpath(root) !== path.resolve(root)) reject('stream_publications'); + const prefix = `chat-catalog-case-${index === 0 ? 'bundle' : 'publication'}-${context.runId}-1-`; + const names = await readdir(root); + if (names.length !== rows.length || names.some(name => !rows.some((_, i) => name === `${prefix}${i}`))) reject('stream_publications'); + } + const exported = []; + for (const [index, row] of rows.entries()) { + const bundle = path.join(bundlesRoot, `chat-catalog-case-bundle-${context.runId}-1-${index}`); + const publication = path.join(publicationsRoot, `chat-catalog-case-publication-${context.runId}-1-${index}`); + for (const directory of [bundle, publication]) { + const stat = await lstat(directory); + if (!stat.isDirectory() || stat.isSymbolicLink()) reject('stream_publications'); + } + const read = async (directory, name) => { + const stat = await lstat(path.join(directory, name)); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 65_536) reject('stream_publications'); + return JSON.parse(await readFile(path.join(directory, name), 'utf8')); + }; + const prepared = await read(bundle, 'prepared.json'); + await readCaseBundle(path.join(bundle, 'projection'), path.join(bundle, 'allure'), prepared, context, selection, row.executionId, selectionSha256); + const value = await read(publication, 'publication.json'); + const expected = { ...identity(selection, context, row.executionId, selectionSha256), schema: 'paperclip.chat-catalog-testops-case-publication/v1' }; + if (Object.entries(expected).some(([key, wanted]) => value[key] !== wanted) + || value.payloadSha256 !== prepared.payloadSha256 || value.projectionBindingSha256 !== prepared.projectionBindingSha256 + || JSON.stringify(value.exported) !== JSON.stringify(prepared.exported[0]) || value.verification !== 'VERIFIED' + || !Array.isArray(value.results) || value.results.length !== 1 || value.results[0].caseId !== row.caseId + || value.results[0].executionId !== row.executionId || value.results[0].attachmentsVerified !== true) reject('stream_publications'); + exported.push(value.exported); + } + return exported; +} diff --git a/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs new file mode 100644 index 00000000000..8e8a462c74c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-testops-streaming.test.mjs @@ -0,0 +1,318 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, readdir, writeFile, rm, realpath, cp } from 'node:fs/promises'; +import path from 'node:path'; +import { tmpdir } from 'node:os'; +import { fullCatalogCases, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs'; +import { hash, model } from './chat-completion-testops-contract.mjs'; +import { run } from './chat-completion-testops-cli.mjs'; +import { prepareCaseBundle, readCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs'; +const context = { catalog: 'full', runId: '123', attempt: '1', workflowSha: 'a'.repeat(40), targetSha: catalogTargetSha }; +const caseIds = { 'tasks.create-backlog': 388, 'documents-artifacts.write-plan': 26, 'projects.project-defaults': 100 }; +const cases = fullCatalogCases.filter(row => Object.hasOwn(caseIds, row.key)).map(row => ({ ...row, caseId: caseIds[row.key], + registeredFullName: row.aliases[0] ?? row.fullName })); +const selection = { schema: 'paperclip.chat-completion-testops-selection/v1', ...context, projectId: 1, + planId: 102, jobId: 100, launchId: 12, allureJobRunId: '77', cases }; +const digest = hash(JSON.stringify(selection)), current = cases.find(row => row.caseId === 388), runUrl = 'https://github.com/paperclipai/paperclip/actions/runs/123'; +async function fixture(t, failed = false, owned = current, runtimeContext = context) { + const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-test-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const root = path.join(base, 'projection'), output = path.join(base, 'allure'); await mkdir(root); + const original = { executionId: owned.executionId, suiteId: 'chat-features', caseId: owned.scenario, + profileId: 'runner-opencode', provider: 'opencode', environmentId: 'local', model, runtimeMode: 'native', attempt: 1, + source: { sha: catalogTargetSha, workflowRunUrl: runUrl }, status: failed ? 'failed' : 'passed', cleanup: failed ? 'failed' : 'passed', + runIds: failed ? [] : Array.from({ length: owned.minimumRuns }, (_, i) => `synthetic-owned-run-${i}`), matcherResults: failed ? [] : Array.from({ length: owned.checks ?? 2 }, (_, i) => ({ + matcher: { kind: 'json_path', path: `owned.check${i}`, expected: true }, passed: true })), screenshots: [] }; + const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2', + campaignId: 'gha-123-1', retries: 0, results: [original] }))], ['junit.xml', Buffer.from('')]]); + const binding = { schema: 'paperclip.chat-completion-testops-export/v1', ...context, campaignId: 'gha-123-1', + executionIds: [owned.executionId], files: [...files].map(([name, bytes]) => ({ path: name, bytes: bytes.length, sha256: hash(bytes) })), + privateEvidence: [{ executionId: owned.executionId, requiredEvidenceComplete: !failed, + finalScreenshot: failed ? null : { sha256: 'e'.repeat(64), bytes: 24 } }] }; + for (const [name, bytes] of files) await writeFile(path.join(root, name), bytes); + await writeFile(path.join(root, 'binding.json'), JSON.stringify(binding)); + const prepared = await prepareCaseBundle(root, output, runtimeContext, selection, owned.executionId, digest); + return { root, output, prepared }; +} +function server(prepared, { existing = false, uncertain = false, apply = true, scheduled = false, processingReads = 0 } = {}) { + let present = existing, posts = 0, pendingReads = processingReads; const calls = []; + const resultId = scheduled ? 3347 : 501; + const detail = () => { + const pending = !present || pendingReads-- > 0; + return { id: resultId, testCaseId: 388, projectId: 1, launchId: 12, + fullName: current.registeredFullName, jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], + ...(!pending ? { status: prepared.exported[0].status } : {}) }; + }; + const api = async (route, method = 'GET', body) => { + calls.push({ route, method }); + if (method === 'POST') { + posts++; assert.equal(route, '/api/jobrun/77/upload'); + assert.deepEqual([...body.keys()], ['file','file','file']); + const result = [...body.values()].find(value => value.name.endsWith('-result.json')); + const parsed = JSON.parse(await result.text()); + assert.equal(parsed.status, prepared.exported[0].status); assert.equal(parsed.labels[0].value, '388'); + if (apply) present = true; + if (uncertain) throw new Error('SYNTHETIC_PRIVATE_RESPONSE_MUST_NOT_ESCAPE'); + return { filesCount: 3 }; + } + if (route === '/api/jobrun/77') return { id: 77, job: { id: 100, type: 'github' }, stage: 'finished', launchId: 12, url: runUrl }; + if (route === '/api/launch/12') return { id: 12, projectId: 1, closed: false }; + if (route.startsWith('/api/testresult?')) return { content: scheduled ? cases.map(row => ({ + id: row.caseId === 388 ? resultId : row.caseId === 100 ? 3346 : 3345, testCaseId: row.caseId })) + : [...(present ? [{ id: resultId, testCaseId: 388 }] : []), { id: 502, testCaseId: 26 }], + totalElements: scheduled ? 3 : present ? 2 : 1, totalPages: 1, number: 0 }; + if (route === `/api/testresult/${resultId}`) return detail(); + if (route.startsWith('/api/testresult/attachment?')) return { content: prepared.exported[0].attachments.map((row, index) => ({ + ...row, id: 700 + index, missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 }; + throw new Error(`Unexpected test route ${route}`); + }; + return { api, calls, get posts() { return posts; } }; +} +const immediate = { now: () => 0, wait: async () => {} }; +async function publish(f, s, mark = async () => {}, options = immediate) { + return publishCaseBundle(s.api, f.root, f.output, f.prepared, context, selection, current.executionId, digest, mark, options); +} +test('one frozen singleton bundle publishes into the bound shared launch beside siblings', async t => { + const f = await fixture(t), s = server(f.prepared); let markers = 0; + const names = await readdir(f.output), value = await publish(f, s, async () => { markers++; }); + assert.equal(value.verification, 'VERIFIED'); assert.equal(markers, 1); assert.equal(s.posts, 1); + assert.equal(value.jobRunStageBefore, 'finished'); assert.equal(value.jobRunStageAfter, 'finished'); + assert.deepEqual(await readdir(f.output), names); + await assert.rejects(prepareCaseBundle(f.root, f.output, context, selection, current.executionId, digest), /allure_output/); +}); +test('an existing result is verified without another mutation or a regenerated UUID', async t => { + const f = await fixture(t), s = server(f.prepared, { existing: true }); + const value = await publish(f, s, async () => assert.fail('must not mark another upload')); + assert.equal(value.uploadSent, false); assert.equal(s.posts, 0); assert.equal(value.verification, 'VERIFIED'); +}); +test('an existing unknown outcome remains readback-only and never requests another upload', async t => { + const f = await fixture(t), s = server(f.prepared, { existing: true }), original = s.api; + s.api = async (...args) => { const row = await original(...args); if (args[0] === '/api/testresult/501') row.status = 'unknown'; return row; }; + await assert.rejects(publish(f, s, async () => assert.fail('must not mark another upload')), /upload_unverified/); + assert.equal(s.posts, 0); +}); +test('three scheduled launch rows permit one initial upload and remain unready until an outcome is processed', async t => { + const f = await fixture(t), s = server(f.prepared, { scheduled: true, processingReads: 1 }); + let markers = 0, waits = 0; + const value = await publish(f, s, async () => { markers++; }, { now: () => 0, wait: async () => { waits++; } }); + assert.equal(value.verification, 'VERIFIED'); assert.equal(value.uploadSent, true); + assert.equal(value.exported.status, 'passed'); assert.equal(value.results[0].resultId, 3347); + assert.equal(s.posts, 1); assert.equal(markers, 1); assert.equal(waits, 1); +}); +test('permanently scheduled readback stops at the original poll bound without another upload', async t => { + const f = await fixture(t), s = server(f.prepared, { scheduled: true, processingReads: Infinity }); + let waits = 0; + await assert.rejects(publish(f, s, async () => {}, { now: () => 0, wait: async () => { waits++; } }), /upload_unverified/); + assert.equal(s.posts, 1); assert.equal(waits, 11); +}); +test('scheduled details with a foreign jobrun, contradictory source or malformed status refuse before upload', async t => { + for (const change of [row => { row.jobRun.id = 78; }, row => { row.jobRun.url = runUrl + '0'; }, + row => { row.status = null; }]) { + const f = await fixture(t), s = server(f.prepared, { scheduled: true }), original = s.api; + s.api = async (...args) => { const row = await original(...args); if (args[0] === '/api/testresult/3347') change(row); return row; }; + await assert.rejects(publish(f, s, async () => assert.fail('must not mark an invalid result')), /result_mismatch/); + assert.equal(s.posts, 0); + } +}); +test('an uncertain applied upload reconciles by GET while an absent upload stops without resend', async t => { + for (const apply of [true, false]) { + const f = await fixture(t), s = server(f.prepared, { uncertain: true, apply, scheduled: true }); + if (apply) { + const value = await publish(f, s); assert.equal(value.verification, 'VERIFIED'); + assert.equal(value.uncertainUpload.kind, 'transport_or_response_failure'); + assert.ok(!JSON.stringify(value).includes('SYNTHETIC_PRIVATE')); + } else await assert.rejects(publish(f, s), error => error.message === 'chat_completion_testops_stream_upload_uncertain' + && !JSON.stringify(error).includes('SYNTHETIC_PRIVATE')); + assert.equal(s.posts, 1); + } +}); +test('failed originals remain failed, including zero-run setup or cleanup failure', async t => { + const f = await fixture(t, true), s = server(f.prepared), value = await publish(f, s); + assert.equal(value.exported.status, 'failed'); assert.equal(value.exported.cleanup, 'failed'); + assert.equal(value.verification, 'VERIFIED'); +}); +test('changed bundle bytes, source, selection digest and foreign case refuse before transport', async t => { + const f = await fixture(t); let calls = 0; + const noApi = async () => { calls++; assert.fail('untrusted bundle must not reach API'); }; + await assert.rejects(publishCaseBundle(noApi, f.root, f.output, f.prepared, { ...context, workflowSha: 'b'.repeat(40) }, selection, + current.executionId, digest, async () => {}), /selection_binding/); + await assert.rejects(readCaseBundle(f.root, f.output, f.prepared, context, selection, current.executionId, 'b'.repeat(64)), /stream_binding/); + await assert.rejects(readCaseBundle(f.root, f.output, f.prepared, context, selection, 'foreign', digest), /stream_scope/); + await writeFile(path.join(f.output, f.prepared.files[0].name), 'SYNTHETIC_PRIVATE_TAMPER'); + await assert.rejects(publishCaseBundle(noApi, f.root, f.output, f.prepared, context, selection, current.executionId, digest, async () => {}), /stream_bundle/); + assert.equal(calls, 0); +}); + + test('final coverage authenticates existing per-case bytes and keeps mixed original grades without re-export', async t => { + const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-final-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const bundles = path.join(base, 'bundles'), publications = path.join(base, 'publications'); + await mkdir(bundles); await mkdir(publications); + for (const [index, row] of cases.entries()) { + const f = await fixture(t, index === 1, row); + const bundle = path.join(bundles, `chat-catalog-case-bundle-123-1-${index}`); + const receipt = path.join(publications, `chat-catalog-case-publication-123-1-${index}`); + await mkdir(bundle); await mkdir(receipt); + await cp(f.root, path.join(bundle, 'projection'), { recursive: true }); + await cp(f.output, path.join(bundle, 'allure'), { recursive: true }); + await writeFile(path.join(bundle, 'prepared.json'), JSON.stringify(f.prepared)); + await writeFile(path.join(receipt, 'publication.json'), JSON.stringify({ ...f.prepared, + schema: 'paperclip.chat-catalog-testops-case-publication/v1', exported: f.prepared.exported[0], + verification: 'VERIFIED', results: [{ executionId: row.executionId, caseId: row.caseId, attachmentsVerified: true }] })); + } + const rows = await publicationSet(bundles, publications, context, selection, digest); + assert.deepEqual(rows.map(row => row.status), ['passed','failed','passed']); + const receiptPath = path.join(publications, 'chat-catalog-case-publication-123-1-1', 'publication.json'); + const bad = JSON.parse(await readFile(receiptPath)); bad.exported.status = 'passed'; + await writeFile(receiptPath, JSON.stringify(bad)); + await assert.rejects(publicationSet(bundles, publications, context, selection, digest), /stream_publications/); + await rm(path.dirname(receiptPath), { recursive: true }); + await assert.rejects(publicationSet(bundles, publications, context, selection, digest), /stream_publications/); +}); + +test('the real CLI prepares without TestOps credentials and retains closed failed or uncertain publication witnesses', async t => { + for (const uncertain of [false, true]) { + const f = await fixture(t, true), base = path.dirname(f.root); + await rm(f.output, { recursive: true }); + const selectionRoot = path.join(base, 'chat-completion-testops-selection'); await mkdir(selectionRoot); + await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selection)); + const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, + ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } }; + const eventPath = path.join(base, 'event.json'); await writeFile(eventPath, JSON.stringify(event)); + const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name, + GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master', + GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', + TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath, + GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: base, GITHUB_WORKSPACE: base, ALLURE_JOB_RUN_ID: '77', + ALLURE_RESULTS: f.output, CHAT_TESTOPS_EXECUTION_ID: current.executionId, EXPECTED_SELECTION_SHA256: digest }; + const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99, + head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml', + repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) }); + assert.equal(await run('prepare-case-upload', f.root, env, { exec, fetchImpl: () => assert.fail('reader must not authenticate to TestOps') }), 0); + const preparedPath = path.join(base, 'chat-completion-testops-prepare-case-upload', 'prepared.json'); + const prepared = JSON.parse(await readFile(preparedPath)), s = server(prepared, { uncertain, apply: !uncertain }); + const fetchImpl = async (url, options) => { + if (url.endsWith('/api/uaa/oauth/token')) return Response.json({ access_token: 'SYNTHETIC_ACCESS_TOKEN' }); + if (uncertain && s.posts && new URL(url).pathname === '/api/jobrun/77') return Response.json({ id: 78 }); + return Response.json(await s.api(new URL(url).pathname + new URL(url).search, options.method, options.body)); + }; + let captured = ''; const capture = t.mock.method(process.stderr, 'write', value => { captured += value; return true; }); + assert.equal(await run('upload-case', f.root, { ...env, ALLURE_TOKEN: 'SYNTHETIC_TESTOPS_SECRET', + CHAT_TESTOPS_PREPARED_FILE: preparedPath }, { exec, fetchImpl }), 1); + capture.mock.restore(); + const state = path.join(base, 'chat-completion-testops-upload-case'); + assert.equal(s.posts, 1); + const receipt = JSON.parse(await readFile(path.join(state, uncertain ? 'failure.json' : 'publication.json'))); + if (uncertain) { + assert.equal(receipt.code, 'chat_completion_testops_stream_upload_uncertain'); + assert.equal(receipt.uploadFailure.kind, 'transport_or_response_failure'); + await assert.rejects(readFile(path.join(state, 'publication.json')), { code: 'ENOENT' }); + } else { assert.equal(receipt.verification, 'VERIFIED'); assert.equal(receipt.exported.status, 'failed'); } + assert.deepEqual(JSON.parse(await readFile(path.join(state, 'once-upload_case.json'))), + { stage: 'upload_case', status: 'STARTED_NO_AUTOMATIC_RESEND' }); + assert.doesNotMatch(captured + JSON.stringify(receipt), /SYNTHETIC_|ACCESS_TOKEN/); + const yaml = await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8'); + for (const filename of ['publication.json', 'failure.json', 'once-upload_case.json']) { + assert.ok(yaml.includes(`/chat-completion-testops-upload-case/${filename}`)); + } + } +}); + +test('source host admission follows only the fixed same-commit callee with exact named caller secret references', async () => { + const { parse } = await import('yaml'); + const { paidWorker } = await import('./chat-completion-testops-host-control.mjs'); + const main = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); + const callee = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); + assert.equal(paidWorker(main, callee).environment, 'runner-e2e-private-paid'); + for (const change of [ + value => { value.jobs.chat.uses = 'foreign/repository/workflow.yml@master'; }, + value => { value.jobs.chat.secrets = 'inherit'; }, + value => { value.jobs.chat_external.secrets.ALLURE_TOKEN = ''; }, + value => { value.jobs.chat.with.selection_sha256 = 'arbitrary'; }, + ]) { const mutant = structuredClone(main); change(mutant); assert.throws(() => paidWorker(mutant, callee), /boundary/); } + const wrong = structuredClone(callee); wrong.jobs.chat.environment = 'runner-e2e-testops-reporting'; + assert.throws(() => paidWorker(main, wrong), /boundary/); +}); + + +test('the final CLI verifies all authenticated mixed-grade publications without re-export or upload and refuses a contradictory merged grade', async t => { + for (const contradiction of [false, true]) { + const base = await realpath(await mkdtemp(path.join(tmpdir(), 'chat-stream-cli-final-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const bundles = path.join(base, 'bundles'), publications = path.join(base, 'publications'), merged = path.join(base, 'merged'); + for (const root of [bundles, publications, merged]) await mkdir(root); + const actual = { ...context, operation: 'evaluate', actorId: 42, triggeringActorId: 42, workflowId: 99 }; + const originals = [], privateEvidence = [], exported = []; + for (const [index, row] of cases.entries()) { + const f = await fixture(t, index === 1, row, actual); + const bundle = path.join(bundles, `chat-catalog-case-bundle-123-1-${index}`); + const publication = path.join(publications, `chat-catalog-case-publication-123-1-${index}`); + await mkdir(bundle); await mkdir(publication); + await cp(f.root, path.join(bundle, 'projection'), { recursive: true }); + await cp(f.output, path.join(bundle, 'allure'), { recursive: true }); + await writeFile(path.join(bundle, 'prepared.json'), JSON.stringify(f.prepared)); + await writeFile(path.join(publication, 'publication.json'), JSON.stringify({ ...f.prepared, + schema: 'paperclip.chat-catalog-testops-case-publication/v1', exported: f.prepared.exported[0], verification: 'VERIFIED', + results: [{ executionId: row.executionId, caseId: row.caseId, attachmentsVerified: true }] })); + originals.push(JSON.parse(await readFile(path.join(f.root, 'normalized-results.json'))).results[0]); + privateEvidence.push(...JSON.parse(await readFile(path.join(f.root, 'binding.json'))).privateEvidence); + exported.push(f.prepared.exported[0]); + } + if (contradiction) Object.assign(originals[0], { status: 'failed', cleanup: 'failed', runIds: [], matcherResults: [] }); + const files = new Map([['normalized-results.json', Buffer.from(JSON.stringify({ schema: 'paperclip.runner-e2e.campaign/v2', + campaignId: 'gha-123-1', retries: 0, results: originals }))], ['junit.xml', Buffer.from('')]]); + for (const [name, bytes] of files) await writeFile(path.join(merged, name), bytes); + await writeFile(path.join(merged, 'binding.json'), JSON.stringify({ schema: 'paperclip.chat-completion-testops-export/v1', + ...context, campaignId: 'gha-123-1', executionIds: cases.map(row => row.executionId), privateEvidence, + files: [...files].map(([name, bytes]) => ({ path: name, bytes: bytes.length, sha256: hash(bytes) })) })); + const selectionRoot = path.join(base, 'chat-completion-testops-selection'); await mkdir(selectionRoot); + await writeFile(path.join(selectionRoot, 'testops-pilot-selection.json'), JSON.stringify(selection)); + const event = { repository: { full_name: 'paperclipai/paperclip', private: false, default_branch: 'master' }, + ref: 'refs/heads/master', sender: { id: 42 }, inputs: { operation: 'evaluate', ALLURE_JOB_RUN_ID: '77' } }; + const eventPath = path.join(base, 'event.json'); await writeFile(eventPath, JSON.stringify(event)); + const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: event.repository.full_name, + GITHUB_REF: event.ref, GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-catalog-testops.yml@refs/heads/master', + GITHUB_SHA: context.workflowSha, GITHUB_RUN_ATTEMPT: '1', GITHUB_RUN_ID: '123', GITHUB_ACTOR_ID: '42', + TESTOPS_ALLOWED_ACTOR_IDS: '[42]', GITHUB_RUN_NUMBER: '1', GITHUB_EVENT_PATH: eventPath, + GH_TOKEN: 'SYNTHETIC_GITHUB_SECRET', RUNNER_TEMP: base, GITHUB_WORKSPACE: base, ALLURE_JOB_RUN_ID: '77', + ALLURE_TOKEN: 'SYNTHETIC_REPORTING_SECRET', EXPECTED_SELECTION_SHA256: digest, + CHAT_TESTOPS_BUNDLES: bundles, CHAT_TESTOPS_PUBLICATIONS: publications, ALLURE_RESULTS: path.join(base, 'must-not-export') }; + const exec = async () => ({ stdout: JSON.stringify({ id: 123, run_attempt: 1, run_number: 1, workflow_id: 99, + head_sha: context.workflowSha, head_branch: 'master', event: 'workflow_dispatch', path: '.github/workflows/chat-catalog-testops.yml', + repository: event.repository, actor: { id: 42 }, triggering_actor: { id: 42 } }) }); + const details = []; + const fetchImpl = async (url, options) => { + const u = new URL(url), route = u.pathname + u.search; + if (u.pathname === '/api/uaa/oauth/token') return Response.json({ access_token: 'SYNTHETIC_ACCESS_TOKEN' }); + assert.equal(options.method, 'GET', 'final verification must not upload or change TestOps'); + if (route === '/api/jobrun/77') return Response.json({ id: 77, job: { id: 100, type: 'github' }, stage: 'finished', launchId: 12, url: runUrl }); + if (route === '/api/launch/12') return Response.json({ id: 12, projectId: 1 }); + if (u.pathname === '/api/testresult') return Response.json({ content: cases.map((row, i) => ({ id: 501 + i, testCaseId: row.caseId })), totalElements: 3, totalPages: 1, number: 0 }); + const match = u.pathname.match(/^\/api\/testresult\/(\d+)$/); + if (match) { + const i = Number(match[1]) - 501; details.push(cases[i].caseId); + return Response.json({ id: Number(match[1]), projectId: 1, launchId: 12, testCaseId: cases[i].caseId, fullName: cases[i].registeredFullName, + jobRun: { id: 77, launchId: 12, url: runUrl }, links: [], status: exported[i].status }); + } + assert.equal(u.pathname, '/api/testresult/attachment'); + const i = Number(u.searchParams.get('testResultId')) - 501; + return Response.json({ content: exported[i].attachments.map((row, index) => ({ ...row, id: 700 + index, missed: false, fromTestCase: false })), totalElements: 2, totalPages: 1 }); + }; + const before = await Promise.all((await readdir(bundles)).map(async name => [name, hash(await readFile(path.join(bundles, name, 'prepared.json')))])); + let stderr = ''; const capture = t.mock.method(process.stderr, 'write', value => { stderr += value; return true; }); + assert.equal(await run('verify-stream', merged, env, { exec, fetchImpl }), 1); + capture.mock.restore(); + const state = path.join(base, 'chat-completion-testops-verify-stream'); + if (contradiction) { + assert.equal(JSON.parse(await readFile(path.join(state, 'failure.json'))).code, 'chat_completion_testops_upload_receipt'); + assert.deepEqual(details, []); + } else { + const verified = JSON.parse(await readFile(path.join(state, 'verification.json'))); + assert.equal(verified.verification, 'VERIFIED'); assert.deepEqual(details.sort((a,b) => a-b), [26,100,388]); + assert.deepEqual(verified.results.map(row => row.status), ['passed','failed','passed']); + } + await assert.rejects(readFile(env.ALLURE_RESULTS), { code: 'ENOENT' }); + assert.deepEqual(await Promise.all((await readdir(bundles)).map(async name => [name, hash(await readFile(path.join(bundles, name, 'prepared.json')))])), before); + assert.doesNotMatch(stderr, /SYNTHETIC_/); + } +}); diff --git a/tests/runner-e2e/chat-completion-testops-api.mjs b/tests/runner-e2e/chat-completion-testops-api.mjs index 5b9f883c935..78236c02df8 100644 --- a/tests/runner-e2e/chat-completion-testops-api.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.mjs @@ -49,8 +49,24 @@ export function isWorkflowUrl(url, workflowId, context = {}) { const { workflow } = catalogDefinition(context); return url === `https://github.com/paperclipai/paperclip/actions/workflows/${workflow.split('/').at(-1)}` || positive(workflowId) && url === `https://github.com/paperclipai/paperclip/actions/workflows/${workflowId}` + || context.catalog === 'full' && url === `https://github.com/paperclipai/paperclip/blob/master/${workflow}` || url === (context.catalog === 'full' ? 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+catalog+TestOps%22' : 'https://github.com/paperclipai/paperclip/actions?query=workflow%3A%22Chat+completion+TestOps%22'); } +function validatedResultStatus(result, id, expected, selection, context, runUrl) { + const { targetSha } = catalogDefinition(context); + if (result?.id !== id || result.projectId !== 1 || result.launchId !== selection.launchId || result.testCaseId !== expected.caseId + || result.fullName !== (expected.registeredFullName ?? expected.fullName) || result.jobRun?.id !== Number(selection.allureJobRunId) + || result.jobRun?.launchId !== selection.launchId || result.jobRun?.url !== runUrl + || result.jobRun.job != null && (result.jobRun.job.id !== selection.jobId + || result.jobRun.job.type != null && result.jobRun.job.type !== 'github' + || result.jobRun.job.url != null && !isWorkflowUrl(result.jobRun.job.url, context.workflowId, context)) + || result.links != null && (!Array.isArray(result.links) || result.links.some(link => !link || typeof link.url === 'string' + && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl + || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== `https://github.com/paperclipai/paperclip/commit/${targetSha}`)))) reject('result_mismatch'); + if (!Object.hasOwn(result, 'status')) return undefined; + if (!['failed', 'broken', 'passed', 'skipped', 'unknown'].includes(result.status)) reject('result_mismatch'); + return result.status; +} function assertCase(actual, expected) { if (actual.id !== expected.caseId || actual.projectId !== 1 || actual.fullName !== (expected.registeredFullName ?? expected.fullName) || actual.automated !== true || actual.deleted === true) reject('case_mismatch'); @@ -135,32 +151,12 @@ export async function inspectPilotCatalog(api) { } export async function bootstrapPilotPlan(api, integrationId, markOnce, context = {}) { + // The pilot plan is retired. Reject before API reads or mutation markers. + if (context.catalog !== 'full') reject('plan_retired'); if (!positive(integrationId)) reject('integration'); const integration = await api(`/api/integration/${integrationId}/project/1`); if (integration.id !== integrationId || integration.projectId !== 1 || integration.info?.type !== 'github' || integration.disabled !== false) reject('integration'); - if (context.catalog === 'full') return { ...await reconcileFullCatalog(api, markOnce), integrationId }; - const inspected = await inspectPilotCatalog(api); - if (inspected.plans.length && inspected.cases.length !== cases.length) reject('plan_conflict'); - const selected = [...inspected.cases]; - for (const row of cases) { - if (selected.some(value => value.key === row.key)) continue; - await markOnce(`create_case_${row.key}`); - const created = await api('/api/testcase', 'POST', { projectId: 1, name: row.name, fullName: row.fullName, automated: true }); - if (!positive(created?.id)) reject('created_case'); - assertCase(created, { ...row, caseId: created.id }); - selected.push({ ...row, caseId: created.id }); - } - const ordered = cases.map(row => selected.find(value => value.key === row.key)); - let planId = inspected.plans[0]?.id; - if (!planId) { - await markOnce('create_plan'); - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: ordered.map(row => row.caseId) } }); - if (!positive(created?.id)) reject('created_plan'); planId = created.id; - } - const binding = { planId, integrationId, cases: ordered }; - assertPlan(await api(`/api/testplan/${planId}`), binding); - return binding; + return { ...await reconcileFullCatalog(api, markOnce), integrationId }; } export async function completePilotRegistration(api, prepared, official, context, workflowId, markOnce, observe = async () => {}) { @@ -235,9 +231,22 @@ export async function launchPilotPlan(api, env, context, workflowId, markOnce) { launchUrl: `${endpoint}/launch/${launched.id}`, status: 'REQUESTED', actualResult: null }; } -export async function verifyPilotUpload(api, selection, context, exported, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { +async function uploadBinding(get, selection, context) { + const run = await get(`/api/jobrun/${selection.allureJobRunId}`); + const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`; + if (run.id !== Number(selection.allureJobRunId) || run.job?.id !== selection.jobId || run.job?.type !== 'github' + || !['in_progress', 'finished'].includes(run.stage) + || run.launchId !== selection.launchId || run.url !== runUrl) reject('jobrun_mismatch'); + const launch = await get(`/api/launch/${selection.launchId}`); + if (launch.id !== selection.launchId || launch.projectId !== 1) reject('launch_mismatch'); + return { run, launch, runUrl }; +} + +async function verifyUploadScope(api, selection, context, exported, executionId, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { const { cases, targetSha, planName } = catalogDefinition(context); - const selected = validateSelection(selection, context, selection.allureJobRunId); + const authorized = validateSelection(selection, context, selection.allureJobRunId); + const selected = executionId === undefined ? authorized : authorized.filter(row => row.executionId === executionId); + if (executionId !== undefined && (context.catalog !== 'full' || selected.length !== 1)) reject('stream_scope'); if (!Array.isArray(exported) || exported.length !== selected.length || new Set(exported.map(row => row.caseId)).size !== selected.length) reject('export_mismatch'); for (const row of exported) { const found = selected.find(value => value.caseId === row.caseId); @@ -250,8 +259,8 @@ export async function verifyPilotUpload(api, selection, context, exported, { now || !Number.isSafeInteger(value.contentLength) || value.contentLength <= 0 || value.contentLength > 12 * 1024 * 1024 || value.contentType !== 'application/json')) reject('export_mismatch'); } - const metadataReadAllowance = context.catalog === 'full' && selected.length > 3 - ? (2 + Math.ceil(selected.length / 100) + 2 * Math.ceil(selected.length / 4)) * 2000 : 0; + const metadataReadAllowance = context.catalog === 'full' && authorized.length > 3 + ? (2 + Math.ceil(authorized.length / 100) + 2 * Math.ceil(authorized.length / 4)) * 2000 : 0; const deadline = now() + 120_000 + metadataReadAllowance; const get = route => { const remaining = deadline - now(); if (remaining < 1) reject('upload_unverified'); @@ -259,36 +268,26 @@ export async function verifyPilotUpload(api, selection, context, exported, { now }; const signature = rows => rows.map(row => JSON.stringify([row.name, row.contentType, row.contentLength])).sort(); for (let poll = 0; poll < 12 && now() < deadline; poll++) { - const run = await get(`/api/jobrun/${selection.allureJobRunId}`); - const runUrl = `https://github.com/paperclipai/paperclip/actions/runs/${context.runId}`; - if (run.id !== Number(selection.allureJobRunId) || run.job?.id !== selection.jobId || run.job?.type !== 'github' - || !['in_progress', 'finished'].includes(run.stage) - || run.launchId !== selection.launchId || run.url !== runUrl) reject('jobrun_mismatch'); - const launch = await get(`/api/launch/${selection.launchId}`); - if (launch.id !== selection.launchId || launch.projectId !== 1) reject('launch_mismatch'); - const listed = await resultRows(get, selection.launchId, selected.length, context.catalog === 'full'); + const { runUrl } = await uploadBinding(get, selection, context); + const listed = await resultRows(get, selection.launchId, authorized.length, context.catalog === 'full'); if (listed === null) { if (poll < 11 && now() + 5000 < deadline) await wait(5000); continue; } - if (listed.length > selected.length + if (listed.length > authorized.length || new Set(listed.map(row => row.id)).size !== listed.length || new Set(listed.map(row => row.testCaseId)).size !== listed.length) reject('result_page'); - let ready = listed.length === selected.length; const verified = []; + if (listed.some(row => !authorized.some(expected => expected.caseId === row.testCaseId))) reject('foreign_result'); + const matching = executionId === undefined ? listed : listed.filter(row => row.testCaseId === selected[0].caseId); + let ready = matching.length === selected.length; const verified = []; const verifyResult = async listedRow => { const expected = selected.find(value => value.caseId === listedRow.testCaseId); if (!expected || !positive(listedRow.id)) reject('foreign_result'); const wanted = exported.find(value => value.caseId === expected.caseId); const result = await get(`/api/testresult/${listedRow.id}`); - if (result.id !== listedRow.id || result.projectId !== 1 || result.launchId !== selection.launchId || result.testCaseId !== expected.caseId - || result.fullName !== (expected.registeredFullName ?? expected.fullName) || result.jobRun?.id !== Number(selection.allureJobRunId) - || result.jobRun?.launchId !== selection.launchId || result.jobRun?.url !== runUrl - || result.jobRun.job != null && (result.jobRun.job.id !== selection.jobId - || result.jobRun.job.type != null && result.jobRun.job.type !== 'github' - || result.jobRun.job.url != null && !isWorkflowUrl(result.jobRun.job.url, context.workflowId, context)) - || !Array.isArray(result.links) || result.links.some(link => typeof link.url === 'string' - && (link.url.startsWith('https://github.com/paperclipai/paperclip/actions/runs/') && link.url !== runUrl - || link.url.startsWith('https://github.com/paperclipai/paperclip/commit/') && link.url !== `https://github.com/paperclipai/paperclip/commit/${targetSha}`)) - || !['unknown', wanted.status].includes(result.status)) reject('result_mismatch'); + const status = validatedResultStatus(result, listedRow.id, expected, selection, context, runUrl); + if (context.catalog === 'full' && status === undefined) return { executionId: expected.executionId, caseId: expected.caseId, + resultId: result.id, cleanup: wanted.cleanup, attachmentsVerified: false }; + if (!Array.isArray(result.links) || !['unknown', wanted.status].includes(status)) reject('result_mismatch'); const attached = await get(`/api/testresult/attachment?${new URLSearchParams({ testResultId: String(result.id), page: '0', size: '100' })}`); const attachments = pageRows(attached); if (attached.totalElements !== attachments.length || attachments.length > 10 || new Set(attachments.map(row => row.id)).size !== attachments.length @@ -300,8 +299,8 @@ export async function verifyPilotUpload(api, selection, context, exported, { now return { executionId: expected.executionId, caseId: expected.caseId, resultId: result.id, status: result.status, cleanup: wanted.cleanup, attachmentsVerified: complete, resultUrl: `${endpoint}/testresult/${result.id}` }; }; - for (let offset = 0; offset < listed.length; offset += context.catalog === 'full' ? 4 : 1) { - const batch = await Promise.all(listed.slice(offset, offset + (context.catalog === 'full' ? 4 : 1)).map(verifyResult)); + for (let offset = 0; offset < matching.length; offset += context.catalog === 'full' ? 4 : 1) { + const batch = await Promise.all(matching.slice(offset, offset + (context.catalog === 'full' ? 4 : 1)).map(verifyResult)); ready &&= batch.every(row => row.attachmentsVerified); verified.push(...batch); } if (ready && now() < deadline) return { schema: 'paperclip.chat-completion-testops-verification/v1', ...context, @@ -311,3 +310,44 @@ export async function verifyPilotUpload(api, selection, context, exported, { now } reject('upload_unverified'); } + +export async function verifyPilotUpload(api, selection, context, exported, options) { + return verifyUploadScope(api, selection, context, exported, undefined, options); +} + +export async function verifyPilotCaseUpload(api, selection, context, executionId, exported, options) { + if (context.catalog !== 'full' || typeof executionId !== 'string') reject('stream_scope'); + return verifyUploadScope(api, selection, context, exported, executionId, options); +} + +// Inspect a stable current launch snapshot before the one allowed upload. Any +// existing row for this case requires readback, never another blind POST. +export async function inspectPilotCaseUpload(api, selection, context, executionId, { now = Date.now, wait = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { + if (context.catalog !== 'full') reject('stream_scope'); + const authorized = validateSelection(selection, context, selection.allureJobRunId); + const row = authorized.find(value => value.executionId === executionId); + if (!row) reject('stream_scope'); + const deadline = now() + 120_000 + (authorized.length > 3 + ? (2 + Math.ceil(authorized.length / 100) + 2 * Math.ceil(authorized.length / 4)) * 2000 : 0); + const get = route => { + const remaining = deadline - now(); if (remaining < 1) reject('upload_unverified'); + return api(route, 'GET', undefined, Math.min(30_000, remaining)); + }; + for (let poll = 0; poll < 12 && now() < deadline; poll++) { + const { run, launch, runUrl } = await uploadBinding(get, selection, context); + if (launch.closed === true) reject('launch_mismatch'); + const rows = await resultRows(get, selection.launchId, authorized.length, true); + if (rows !== null) { + if (new Set(rows.map(value => value.id)).size !== rows.length + || new Set(rows.map(value => value.testCaseId)).size !== rows.length) reject('result_page'); + if (rows.some(value => !positive(value.id) || !authorized.some(expected => expected.caseId === value.testCaseId))) reject('foreign_result'); + const existing = rows.find(value => value.testCaseId === row.caseId); + if (!existing) return { present: false, jobRunStage: run.stage }; + const result = await get(`/api/testresult/${existing.id}`); + return { present: validatedResultStatus(result, existing.id, row, selection, context, runUrl) !== undefined, + jobRunStage: run.stage }; + } + if (poll < 11 && now() + 5000 < deadline) await wait(5000); + } + reject('upload_unverified'); +} diff --git a/tests/runner-e2e/chat-completion-testops-api.test.mjs b/tests/runner-e2e/chat-completion-testops-api.test.mjs index 4ee58ebe003..aee4be2527c 100644 --- a/tests/runner-e2e/chat-completion-testops-api.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-api.test.mjs @@ -180,35 +180,9 @@ test('an uncertain launch response is never automatically resent', async () => { assert.equal(f.calls.filter(row => row.method === 'POST').length, 1); }); -test('registration creates only the three missing cases and the dedicated plan after once markers', async () => { - const calls = []; let nextCase = 100; - const get = async (route, method = 'GET', body) => { - calls.push({ route, method, body }); - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - if (route === '/api/testcase' && method === 'POST') return { ...body, id: nextCase++ }; - if (route === '/api/testplan' && method === 'POST') return { ...plan, name: planName }; - if (route === '/api/testplan/10') return plan; - assert.fail(route); - }; - const value = await bootstrapPilotPlan(get, 2, async stage => calls.push({ marker: stage })); - assert.equal(value.cases.length, 3); assert.equal(value.planId, 10); - assert.equal(calls.filter(row => row.method === 'POST').length, 4); - for (const [i, row] of calls.entries()) if (row.method === 'POST') assert.ok(calls[i - 1].marker); -}); - -test('uncertain case creation is never automatically resent', async () => { - let writes = 0, markers = 0; - const get = async (route, method = 'GET') => { - if (method === 'POST') { writes++; throw new Error('uncertain transport'); } - if (route === '/api/integration/2/project/1') return { id: 2, projectId: 1, info: { type: 'github' }, disabled: false }; - if (route.startsWith('/api/testcase?')) return { content: [], number: 0, totalPages: 0, totalElements: 0 }; - if (route.startsWith('/api/testplan?')) return { content: [], totalPages: 0 }; - assert.fail(route); - }; - await assert.rejects(bootstrapPilotPlan(get, 2, async () => { markers++; }), /uncertain transport/); - assert.equal(writes, 1); assert.equal(markers, 1); +test('retired pilot bootstrap cannot recreate a plan or call the integration API', async () => { + await assert.rejects(bootstrapPilotPlan(() => assert.fail('retired route must not call the API'), 2, + () => assert.fail('retired route must not mark a mutation')), /chat_completion_testops_plan_retired/); }); function api(replace = {}) { diff --git a/tests/runner-e2e/chat-completion-testops-cli.mjs b/tests/runner-e2e/chat-completion-testops-cli.mjs index 467700d7b62..3504ae87acc 100644 --- a/tests/runner-e2e/chat-completion-testops-cli.mjs +++ b/tests/runner-e2e/chat-completion-testops-cli.mjs @@ -7,6 +7,7 @@ import { admitDispatch, hash, reject, validateSelection, evidencePolicy, catalog import { testopsClient, bootstrapPilotPlan, completePilotRegistration, validatePilotSelection, launchPilotPlan, verifyPilotUpload } from './chat-completion-testops-api.mjs'; import { parseOfficialBootstrapContext } from './testops-chat-reporter.mjs'; import { exportPilotAllure, readPilotExport } from './chat-completion-testops-results.mjs'; +import { prepareCaseBundle, publishCaseBundle, publicationSet } from './chat-catalog-testops-streaming.mjs'; const execute = promisify(execFile); const fullCatalogFailureCodes = new Set([ @@ -31,12 +32,12 @@ export function executionPartition(selection, context, jobRunId) { const matrix = executionMatrix(selection, context, jobRunId); if (context.catalog !== 'full') reject('catalog'); const small = matrix.include.length <= 3; - const native = { include: matrix.include.filter(row => small || !row.external) }; - const external = { include: matrix.include.filter(row => !small && row.external) }; + const native = { include: matrix.include.filter(row => !row.external) }; + const external = { include: matrix.include.filter(row => row.external) }; const size = [matrix, native, external, selection.cases.map(row => row.executionId)] .reduce((sum, value) => sum + JSON.stringify(value).length * 2, 0); if (size > 1_000_000) reject('matrix_size'); - return { matrix, native, external, parallelism: small ? '1' : '8' }; + return { matrix, native, external, parallelism: small ? external.include.length ? '1' : '2' : '8' }; } async function readOwned(file, root, limit) { const absolute = path.resolve(file), base = await realpath(root); @@ -92,7 +93,8 @@ function mutationMarker(root) { export async function run(command, argument, env = process.env, { fetchImpl = fetch, exec = execute } = {}) { let state; try { - if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload'].includes(command)) reject('operation'); + if (!['guard', 'bootstrap', 'bootstrap-context', 'selection', 'launch-plan', 'verify-selection', 'upload', 'verify-upload', + 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command)) reject('operation'); const context = await actualContext(env, exec); if (command === 'guard') { await output(env, 'operation', context.operation); await output(env, 'target_sha', context.targetSha); @@ -104,8 +106,15 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe } if (['bootstrap', 'bootstrap-context'].includes(command) && context.operation !== 'bootstrap' || command === 'launch-plan' && context.operation !== 'launch-plan' - || ['selection', 'upload', 'verify-upload'].includes(command) && context.operation !== 'evaluate') reject('operation'); + || ['selection', 'upload', 'verify-upload', 'prepare-case-upload', 'upload-case', 'verify-stream'].includes(command) && context.operation !== 'evaluate') reject('operation'); state = await stateDirectory(env, command === 'bootstrap-context' ? 'bootstrap' : command, command === 'bootstrap-context'); + if (command === 'prepare-case-upload') { + if (Object.hasOwn(env, 'ALLURE_TOKEN')) reject('worker_credential'); + const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context); + const prepared = await prepareCaseBundle(path.resolve(argument), env.ALLURE_RESULTS, context, selected, + env.CHAT_TESTOPS_EXECUTION_ID, env.EXPECTED_SELECTION_SHA256); + await writeReceipt(state, 'prepared.json', prepared); return 0; + } const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); if (command === 'bootstrap') { const prepared = await bootstrapPilotPlan(api, Number(env.CHAT_TESTOPS_INTEGRATION_ID), mutationMarker(state), context); @@ -144,6 +153,24 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe await output(env, 'policy', JSON.stringify(context.catalog === 'full' ? null : evidencePolicy(selection, context, env.ALLURE_JOB_RUN_ID))); return 0; } const selected = await readSelection(path.join(env.GITHUB_WORKSPACE, 'chat-completion-testops-selection', 'testops-pilot-selection.json'), env, context); + if (command === 'upload-case') { + const prepared = JSON.parse(await readOwned(env.CHAT_TESTOPS_PREPARED_FILE, env.RUNNER_TEMP, 65_536)); + const publication = await publishCaseBundle(api, path.resolve(argument), env.ALLURE_RESULTS, prepared, context, selected, + env.CHAT_TESTOPS_EXECUTION_ID, env.EXPECTED_SELECTION_SHA256, () => mutationMarker(state)('upload_case')); + await writeReceipt(state, 'publication.json', publication); + return publication.exported.status === 'passed' ? 0 : 1; + } + if (command === 'verify-stream') { + const exported = await publicationSet(env.CHAT_TESTOPS_BUNDLES, env.CHAT_TESTOPS_PUBLICATIONS, context, selected, env.EXPECTED_SELECTION_SHA256); + const { campaign } = await readPilotExport(path.resolve(argument), context, selected); + if (exported.some(row => { + const original = campaign.results.find(value => value.executionId === row.executionId); + return !original || row.status !== (original.status === 'passed' && original.cleanup === 'passed' ? 'passed' : 'failed') || row.cleanup !== original.cleanup; + })) reject('upload_receipt'); + const verification = await verifyPilotUpload(api, selected, context, exported); + await writeReceipt(state, 'verification.json', verification); + return exported.some(row => row.status !== 'passed') ? 1 : 0; + } if (command === 'upload') { const exported = await exportPilotAllure(path.resolve(argument), env.ALLURE_RESULTS, context, selected); await writeReceipt(state, 'exported.json', exported); return exported.some(row => row.status !== 'passed') ? 1 : 0; @@ -161,6 +188,11 @@ export async function run(command, argument, env = process.env, { fetchImpl = fe const code = fullCatalogFailureCodes.has(error?.message) || /^(chat_completion_testops_[a-z_]{1,80}|testops_(?:api|auth)_http(?:_[1-5][0-9]{2})?)$/.test(error?.message ?? '') ? error.message : 'chat_completion_testops_unknown'; const receipt = { command, code, httpStatus: Number.isInteger(error?.status) ? error.status : null }; + if (code === 'chat_completion_testops_stream_upload_uncertain' + && ['testops_api_http', 'testops_response_invalid', 'testops_response_too_large', 'transport_or_response_failure'].includes(error.uploadFailure?.kind)) { + receipt.uploadFailure = { kind: error.uploadFailure.kind, + httpStatus: Number.isInteger(error.uploadFailure.httpStatus) ? error.uploadFailure.httpStatus : null }; + } if (state) await writeReceipt(state, 'failure.json', receipt).catch(() => {}); process.stderr.write(`Chat completion TestOps stopped: ${JSON.stringify(receipt)}\n`); return 1; } diff --git a/tests/runner-e2e/chat-completion-testops-contract.test.mjs b/tests/runner-e2e/chat-completion-testops-contract.test.mjs index c77d89031a8..01bd9517951 100644 --- a/tests/runner-e2e/chat-completion-testops-contract.test.mjs +++ b/tests/runner-e2e/chat-completion-testops-contract.test.mjs @@ -7,6 +7,26 @@ import path from 'node:path'; import { parse } from 'yaml'; import { admitDispatch, cases, targetSha, validateSelection, evidencePolicy } from './chat-completion-testops-contract.mjs'; import { actualContext } from './chat-completion-testops-cli.mjs'; +import { paidWorker } from './chat-completion-testops-host-control.mjs'; + +test('reusable workers bind named environment secrets without inherited or extra credentials', async () => { + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); + const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); + const named = { OPENROUTER_API_KEY: '${{ secrets.OPENROUTER_API_KEY }}', + ANTHROPIC_API_KEY: '${{ secrets.ANTHROPIC_API_KEY }}', ALLURE_TOKEN: '${{ secrets.ALLURE_TOKEN }}' }; + const supported = structuredClone(definition); + for (const id of ['chat', 'chat_external']) supported.jobs[id].secrets = { ...named }; + assert.equal(paidWorker(supported, called), called.jobs.chat); + assert.equal(paidWorker(definition, called), called.jobs.chat); + for (const id of ['chat', 'chat_external']) { + assert.deepEqual(definition.jobs[id].secrets, named); + for (const secrets of [{ ...named, OPENROUTER_API_KEY: '' }, { ...named, EXTRA: '${{ secrets.EXTRA }}' }, + { ...named, ALLURE_TOKEN: '${{ secrets.OTHER_TOKEN }}' }, 'inherit']) { + const mutant = structuredClone(supported); mutant.jobs[id].secrets = secrets; + assert.throws(() => paidWorker(mutant, called), /Workflow caller boundary mismatch/); + } + } +}); function fixture() { const env = { GITHUB_ACTIONS: 'true', GITHUB_EVENT_NAME: 'workflow_dispatch', GITHUB_REPOSITORY: 'paperclipai/paperclip', @@ -96,44 +116,17 @@ test('the executable guard binds the actual original GitHub run before service o } finally { await rm(root, { recursive: true, force: true }); } }); -test('the real workflow separates metadata authority, provider execution and result reporting', async () => { - const text = await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8'); - const w = parse(text); - assert.deepEqual(Object.keys(w.on.workflow_dispatch.inputs).sort(), ['ALLURE_JOB_RUN_ID', 'ALLURE_USERNAME', 'operation']); - assert.equal(w.concurrency['cancel-in-progress'], false); - assert.equal(w.jobs.chat.strategy['max-parallel'], 1); assert.equal(w.jobs.chat.strategy['fail-fast'], false); - assert.equal(w.jobs.select.environment, 'runner-e2e-testops-reporting'); - assert.equal(w.jobs.chat.environment, 'runner-e2e-private-paid'); - assert.equal(w.jobs.report.environment, 'runner-e2e-private-history'); - assert.equal(w.jobs.upload.environment, 'runner-e2e-testops-reporting'); - for (const [name, job] of Object.entries(w.jobs)) { - assert.ok(job.steps.slice(0, 3).some(step => step.run?.endsWith('chat-completion-testops-cli.mjs guard')), name); - if (name !== 'chat') assert.ok(!JSON.stringify(job).includes('secrets.OPENROUTER_API_KEY'), name); - if (!['select', 'upload'].includes(name)) assert.ok(!JSON.stringify(job).includes('secrets.ALLURE_TOKEN'), name); - } - assert.ok(w.jobs.upload.if.includes("needs.report.result == 'success'")); - assert.ok(w.jobs.report.needs.includes('build')); - assert.ok(w.jobs.report.if.includes("needs.build.result == 'success'")); - assert.ok(w.jobs.report.if.includes("needs.chat.result == 'failure'")); - assert.ok(!text.includes('<<:')); assert.ok(!text.includes('target_branch:')); - assert.equal(w.jobs.build.steps.find(step => step.name?.startsWith('Resolve the fixed')).run.includes('--ignore-scripts'), true); - const paid = w.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); - assert.equal(paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS, 'v1'); - assert.equal(paid.env.ALLURE_JOB_RUN_ID, '${{ inputs.ALLURE_JOB_RUN_ID }}'); - assert.ok(paid.run.includes('--max-automatic-retries 0')); - assert.ok(w.jobs.build.steps.some(step => step.run?.includes('chat-completion-testops-host-control.mjs'))); - assert.ok(w.jobs.chat.steps.indexOf(paid) > w.jobs.chat.steps.findIndex(step => step.name?.startsWith('Verify source-bound'))); -}); - test('target checkout cannot populate or overwrite the trusted helper path', async () => { - const definition = parse(await readFile(new URL('../../.github/workflows/chat-completion-testops.yml', import.meta.url), 'utf8')); + const definition = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops.yml', import.meta.url), 'utf8')); + const called = parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8')); + const jobs = [definition.jobs.build, called.jobs.chat]; const root = await realpath(await mkdtemp(path.join(os.tmpdir(), 'testops-checkout-boundary-'))); try { const git = (...args) => execFileSync('git', ['-c', 'core.hooksPath=/dev/null', '-c', 'commit.gpgsign=false', ...args], { cwd: root, stdio: 'pipe' }); git('init'); await writeFile(path.join(root, 'fixture.txt'), 'normal target'); git('add', '.'); git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'normal target'); - for (const job of [definition.jobs.build, definition.jobs.chat]) { - const target = job.steps.findIndex(step => step.with?.ref === '${{ needs.authorize.outputs.target_sha }}'); + for (const job of jobs) { + const target = job.steps.findIndex(step => ['${{ needs.authorize.outputs.target_sha }}', '${{ inputs.target_sha }}'].includes(step.with?.ref)); const boundary = job.steps.findIndex(step => step.id === 'target_boundary'); const trusted = job.steps.findIndex(step => step.with?.path === '.testops-trusted'); assert.ok(target >= 0 && boundary > target && trusted > boundary, 'Reject reserved target paths, then check out trusted main last'); @@ -143,7 +136,7 @@ test('target checkout cannot populate or overwrite the trusted helper path', asy await mkdir(path.join(root, '.testops-trusted')); await writeFile(path.join(root, '.testops-trusted', 'helper.js'), 'candidate-controlled helper'); git('add', '.testops-trusted'); git('-c', 'user.name=Fixture', '-c', 'user.email=fixture@example.invalid', 'commit', '-m', 'reserved path in target'); - for (const job of [definition.jobs.build, definition.jobs.chat]) { + for (const job of jobs) { const guard = job.steps.find(step => step.id === 'target_boundary'); assert.throws(() => execFileSync('bash', ['-euo', 'pipefail', '-c', guard.run], { cwd: root, stdio: 'pipe' })); } diff --git a/tests/runner-e2e/chat-completion-testops-host-control.mjs b/tests/runner-e2e/chat-completion-testops-host-control.mjs index c72f7183aa9..350cb69f54e 100644 --- a/tests/runner-e2e/chat-completion-testops-host-control.mjs +++ b/tests/runner-e2e/chat-completion-testops-host-control.mjs @@ -5,6 +5,26 @@ import { pathToFileURL } from 'node:url'; import { parse } from 'yaml'; import { model, catalogDefinition, catalogWorkflow } from './chat-completion-testops-contract.mjs'; +export function paidWorker(definition, called) { + const path = './.github/workflows/chat-catalog-testops-case.yml'; + for (const name of ['chat', 'chat_external']) { + const job = definition.jobs?.[name]; + if (job?.uses !== path || job.with?.protocol !== 'streaming-v1' + || job.with?.target_sha !== '${{ needs.authorize.outputs.target_sha }}' + || job.with?.selection_artifact_id !== '${{ needs.select.outputs.artifact_id }}' + || job.with?.selection_sha256 !== '${{ needs.select.outputs.selection_sha256 }}' + || JSON.stringify(job.secrets) !== JSON.stringify({ OPENROUTER_API_KEY: '${{ secrets.OPENROUTER_API_KEY }}', + ANTHROPIC_API_KEY: '${{ secrets.ANTHROPIC_API_KEY }}', ALLURE_TOKEN: '${{ secrets.ALLURE_TOKEN }}' })) { + throw new Error('Workflow caller boundary mismatch'); + } + } + if (!called?.on?.workflow_call || Object.keys(called.on).length !== 1 + || called.jobs?.chat?.environment !== 'runner-e2e-private-paid' + || called.jobs?.report?.environment !== 'runner-e2e-private-history' + || called.jobs?.upload?.environment !== 'runner-e2e-testops-reporting') throw new Error('Workflow environment boundary mismatch'); + return called.jobs.chat; +} + /** A credential-free caller/source control with synthetic IMDS responses. * It does not qualify an actual AWS host or make a provider call. */ @@ -15,7 +35,10 @@ export async function verifyHostContract(workspace, temporaryRoot, env = process const source = execFileSync('git', ['rev-parse', 'HEAD'], { cwd: workspace, encoding: 'utf8' }).trim(); if (source !== targetSha) throw new Error('Source control pin mismatch'); const definition = parse(await readFile(new URL(`../../${workflow}`, import.meta.url), 'utf8')); - const paid = definition.jobs.chat.steps.find(step => step.env?.OPENROUTER_API_KEY); + // Only the reviewed local callee at the same trusted checkout is read. A + // workflow input or another repository/ref cannot supply the worker body. + const worker = full ? paidWorker(definition, parse(await readFile(new URL('../../.github/workflows/chat-catalog-testops-case.yml', import.meta.url), 'utf8'))) : definition.jobs.chat; + const paid = worker.steps.find(step => step.env?.OPENROUTER_API_KEY); if (paid.env.PAPERCLIP_CHAT_COMPLETION_TESTOPS !== 'v1' || paid.env.ALLURE_JOB_RUN_ID !== '${{ inputs.ALLURE_JOB_RUN_ID }}' || !paid.run.includes('--max-automatic-retries 0') || definition.jobs.chat.strategy['max-parallel'] !== (full ? '${{ fromJSON(needs.select.outputs.parallelism) }}' : 1)) throw new Error('Workflow host contract mismatch'); const { runnerMatrix } = await import(pathToFileURL(path.join(workspace, 'tests/runner-e2e/catalog.ts')).href); diff --git a/tests/runner-e2e/chat-feature-testops-api.mjs b/tests/runner-e2e/chat-feature-testops-api.mjs index 2b4a4f1d049..813f6c868c2 100644 --- a/tests/runner-e2e/chat-feature-testops-api.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.mjs @@ -53,7 +53,7 @@ const diagnosticCodes = new Set(['untrusted_chat_testops_context', 'testops_toke 'testops_case_alias_conflict', 'testops_plan_name_conflict', 'testops_plan_identity_invalid', 'testops_integration_identity_invalid', 'testops_bootstrap_identity_invalid', 'testops_plan_without_matching_case', 'testops_created_case_identity_invalid', 'testops_created_plan_identity_invalid', 'testops_plan_scope_mismatch', - 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', + 'testops_plan_retired', 'testops_operation_invalid', 'testops_output_invalid', 'untrusted_public_testops_context', 'chat_testops_identity_missing', 'chat_testops_case_mismatch', 'chat_testops_plan_mismatch', 'chat_testops_plan_job_mismatch', 'chat_testops_job_mismatch', 'chat_testops_job_id_mismatch', 'chat_testops_job_type_mismatch', 'chat_testops_job_parameters_shape_mismatch', 'chat_testops_job_parameters_limit', 'chat_testops_job_url_mismatch', 'chat_testops_jobrun_mismatch', 'chat_testops_launch_mismatch', 'chat_testops_selection_mismatch', 'chat_testops_parameter_mismatch', 'chat_testops_nonce_invalid', 'chat_testops_nonce_conflict', 'chat_testops_catalog_incomplete', 'chat_testops_launch_admission', 'chat_testops_github_http', 'chat_testops_integration_mismatch', 'chat_testops_upload_mismatch', 'chat_testops_upload_unverified']); @@ -85,9 +85,21 @@ export async function testopsClient(token, fetchImpl = fetch) { return async (route, method = 'GET', body, timeoutMs = 30_000) => { if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 30_000) fail('chat_testops_upload_unverified'); if (!route.startsWith('/api/') || route.includes('://')) fail('testops_route_invalid'); + const multipart = body instanceof FormData; + if (multipart) { + const entries = [...body.entries()], files = entries.map(([, file]) => file); + if (method !== 'POST' || !/^\/api\/jobrun\/[1-9][0-9]{0,15}\/upload$/.test(route) + || !Number.isSafeInteger(Number(route.split('/')[3])) || entries.length !== 3 + || entries.some(([name]) => name !== 'file') + || files.some(file => !(file instanceof Blob) || file.type !== 'application/json' || file.size > 12 * 1024 * 1024 + || !/^[a-f0-9-]{36}-(?:attachment|result)\.json$/.test(file.name)) + || files.filter(file => file.name.endsWith('-result.json')).length !== 1 + || new Set(files.map(file => file.name)).size !== 3 + || files.reduce((bytes, file) => bytes + file.size, 0) > 32 * 1024 * 1024) fail('testops_route_invalid'); + } const response = await fetchImpl(`${endpoint}${route}`, { method, redirect: 'error', - headers: { Accept: 'application/json', Authorization: `Bearer ${value.access_token}`, ...(body ? { 'Content-Type': 'application/json' } : {}) }, - ...(body ? { body: JSON.stringify(body) } : {}), signal: AbortSignal.timeout(timeoutMs) }); + headers: { Accept: 'application/json', Authorization: `Bearer ${value.access_token}`, ...(body && !multipart ? { 'Content-Type': 'application/json' } : {}) }, + ...(body ? { body: multipart ? body : JSON.stringify(body) } : {}), signal: AbortSignal.timeout(timeoutMs) }); if (!response.ok) fail(`testops_api_http_${response.status}`); if (response.status === 204) return null; const text = await response.text(); @@ -139,32 +151,10 @@ export async function inspectChatMetadata(api) { return { projectId: 1, caseConflict: cases.size > 1, cases: [...cases.values()], plans: plans.map(value => ({ id: value.id, name: planName, count: value.testCasesCount })), integrations }; } -export async function ensureChatPlan(api, inspected, markOnce) { - if (inspected.projectId !== 1 || inspected.plans.length > 1) fail('testops_bootstrap_identity_invalid'); - if (inspected.caseConflict === true || inspected.cases.length > 1) fail('testops_case_alias_conflict'); - let selected = inspected.cases[0], plan = inspected.plans[0]; - if (plan && !selected) fail('testops_plan_without_matching_case'); - if (!selected || !plan) await markOnce(); - if (!selected) { - const created = await api('/api/testcase', 'POST', { projectId: 1, name: 'Create a task backlog through chat', fullName, - automated: true, description: 'Canonical scenario: tasks.create-backlog. Readiness and recorded outcomes remain separate.', - expectedResult: 'The real agent creates the requested backlog; public Product E2E assertions and cleanup must pass.' }); - if (!id(created?.id) || created.projectId !== 1 || created.fullName !== fullName) fail('testops_created_case_identity_invalid'); - selected = { id: created.id, fullName, automated: created.automated === true }; - } - if (!plan) { - const created = await api('/api/testplan', 'POST', { projectId: 1, name: planName, - treeSelection: { inverted: false, leafsInclude: [selected.id] } }); - if (!id(created?.id) || created.projectId !== 1 || created.name !== planName) fail('testops_created_plan_identity_invalid'); - plan = { id: created.id, name: planName }; - } - const actual = await api(`/api/testplan/${plan.id}`); - if (actual.projectId !== 1 || actual.name !== planName || actual.testCasesCount !== 1 - || actual.treeSelection?.inverted === true || !same(actual.treeSelection?.leafsInclude, [selected.id]) - || (actual.treeSelection?.groupsInclude?.length ?? 0) > 0 || (actual.treeSelection?.leafsExclude?.length ?? 0) > 0) fail('testops_plan_scope_mismatch'); - return { projectId: 1, case: selected, plan: { id: plan.id, name: planName, count: 1 }, - caseUrl: `${endpoint}/project/1/test-cases/${selected.id}`, planUrl: `${endpoint}/testplan/${plan.id}`, - actualResult: null, note: 'Plan preparation is not an executed measurement or a passing result.' }; +// Kept as an explicit retirement boundary for callers of the former smoke lane. +// Catalog registration owns the shared cases; deleting the old plan must persist. +export async function ensureChatPlan() { + fail('testops_plan_retired'); } diff --git a/tests/runner-e2e/chat-feature-testops-api.test.mjs b/tests/runner-e2e/chat-feature-testops-api.test.mjs index 318c616208c..6864aeb0ba4 100644 --- a/tests/runner-e2e/chat-feature-testops-api.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-api.test.mjs @@ -31,64 +31,14 @@ test('multiple alias identities, foreign project and incomplete metadata stop be await assert.rejects(inspectChatMetadata(async () => ({ content: [], totalPages: 11, totalElements: 1100, number: 0 })), /catalog_incomplete/); }); -test('existing exact plan reuses case and plan without a write', async () => { - const result = await ensureChatPlan(async route => { assert.equal(route, '/api/testplan/8'); return plan; }, - await inspectChatMetadata(inspectApi()), () => assert.fail('must not mark or write')); - assert.equal(result.case.id, 91); - assert.equal(result.plan.id, 8); - assert.equal(result.actualResult, null); -}); - -test('bootstrap creates only the missing exact plan after exclusive marker and reads back scope', async () => { - const calls = []; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - if (method === 'POST') { assert.equal(calls[0].route, 'MARK'); assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [91] } }); } - return plan; - }, { projectId: 1, cases: [caseRow], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(calls.filter(row => row.method === 'POST').length, 1); - assert.equal(result.plan.count, 1); -}); - -test('empty project bootstrap links the genuinely created case into only the new plan', async () => { - const calls = []; - const createdCase = { ...caseRow, id: 92 }; - const createdPlan = { ...plan, treeSelection: { inverted: false, leafsInclude: [92] } }; - const result = await ensureChatPlan(async (route, method, body) => { - calls.push({ route, method, body }); - assert.equal(calls[0].route, 'MARK'); - if (route === '/api/testcase') { - assert.equal(body.projectId, 1); assert.equal(body.fullName, fullName); assert.equal(body.automated, true); - return createdCase; - } - if (route === '/api/testplan') { - assert.deepEqual(body, { projectId: 1, name: planName, treeSelection: { inverted: false, leafsInclude: [92] } }); - return createdPlan; - } - assert.equal(route, '/api/testplan/8'); assert.equal(method, undefined); return createdPlan; - }, { projectId: 1, cases: [], plans: [] }, async () => calls.push({ route: 'MARK' })); - assert.equal(result.case.id, 92); assert.equal(result.plan.id, 8); - assert.deepEqual(calls.map(call => call.route), ['MARK', '/api/testcase', '/api/testplan', '/api/testplan/8']); -}); - -test('known conflicting plan stops before creating an otherwise missing case or marker', async () => { - await assert.rejects(ensureChatPlan(() => assert.fail('no API write or read needed'), - { projectId: 1, cases: [], plans: [plan] }, () => assert.fail('must not mark')), /plan_without_matching_case/); -}); - -test('new case is created only after alias reconciliation; an uncertain mutation is never resent', async () => { - let writes = 0, markers = 0; - await assert.rejects(ensureChatPlan(async (_route, method) => { - assert.equal(method, 'POST'); writes++; throw new Error('uncertain transport'); - }, { projectId: 1, cases: [], plans: [] }, async () => { markers++; }), /uncertain/); - assert.equal(writes, 1); assert.equal(markers, 1); -}); - -test('existing broad or conflicting plan never expands or changes its scope', async () => { - for (const actual of [{ ...plan, testCasesCount: 2 }, { ...plan, treeSelection: { leafsInclude: [92] } }, - { ...plan, treeSelection: { leafsInclude: [91], inverted: true } }, { ...plan, projectId: 2 }]) { - await assert.rejects(ensureChatPlan(async (_route, method) => { assert.equal(method, undefined); return actual; }, - { projectId: 1, cases: [caseRow], plans: [plan] }, () => assert.fail('no writes')), /scope_mismatch/); +test('retired singleton bootstrap cannot recreate an absent plan or mutate retained metadata', async () => { + for (const inspected of [ + { projectId: 1, cases: [], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [] }, + { projectId: 1, cases: [caseRow], plans: [plan] }, + ]) { + await assert.rejects(ensureChatPlan(() => assert.fail('retired route must not call the API'), inspected, + () => assert.fail('retired route must not mark a mutation')), /testops_plan_retired/); } }); @@ -844,3 +794,31 @@ test('deadline crossing after a complete attachment GET keeps the exact readines assert.equal(projected.verification.attachmentReadiness[0].withinDeadline, false); return true; }); }); + +test('existing-jobrun upload uses repeated literal file fields without JSON encoding or a lifecycle request', async () => { + const calls = []; + const api = await testopsClient('PRIVATE_KEY', async (url, options) => { + calls.push({ url, options }); + return calls.length === 1 ? { ok: true, json: async () => ({ access_token: 'PRIVATE_BEARER' }) } + : { ok: true, status: 200, text: async () => '{"filesCount":3}' }; + }); + const data = new FormData(); + for (const suffix of ['result', 'attachment', 'attachment']) data.append('file', new Blob(['{}'], { type: 'application/json' }), `${crypto.randomUUID()}-${suffix}.json`); + assert.deepEqual(await api('/api/jobrun/77/upload', 'POST', data), { filesCount: 3 }); + assert.equal(calls.length, 2); + assert.equal(calls[1].url, `${endpoint}/api/jobrun/77/upload`); + assert.equal(calls[1].options.body, data); + assert.equal(calls[1].options.headers['Content-Type'], undefined); + assert.deepEqual([...calls[1].options.body.keys()], ['file', 'file', 'file']); + assert.equal(calls[1].options.body.getAll('file[]').length, 0); + assert.equal(calls[1].options.redirect, 'error'); +}); + +test('multipart uploads cannot target another route or add undeclared parts', async () => { + let calls = 0; + const api = await testopsClient('PRIVATE_KEY', async () => { calls++; return { ok: true, json: async () => ({ access_token: 'PRIVATE_BEARER' }) }; }); + const form = new FormData(); form.append('file[]', new Blob(['{}']), 'unknown.json'); + await assert.rejects(api('/api/jobrun/77/upload', 'POST', form), /testops_route_invalid/); + await assert.rejects(api('/api/testcase', 'POST', form), /testops_route_invalid/); + assert.equal(calls, 1, 'only the existing authentication request'); +}); diff --git a/tests/runner-e2e/chat-feature-testops-cli.test.mjs b/tests/runner-e2e/chat-feature-testops-cli.test.mjs index 107f046b109..70238eb15f0 100644 --- a/tests/runner-e2e/chat-feature-testops-cli.test.mjs +++ b/tests/runner-e2e/chat-feature-testops-cli.test.mjs @@ -9,7 +9,7 @@ const context = { GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/ GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: 'a'.repeat(40), GITHUB_RUN_ID: '123', GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master' }; -test('actual metadata bootstrap caller creates no result or agent and reuses exact case/plan', async t => { +test('actual retired singleton caller refuses plan recreation and creates no result', async t => { const temporary = await realpath(await mkdtemp(path.join(tmpdir(), 'testops-bootstrap-'))); t.after(() => rm(temporary, { recursive: true })); const state = path.join(temporary, 'state'), requests = []; @@ -30,9 +30,10 @@ test('actual metadata bootstrap caller creates no result or agent and reuses exa return response({ id: 8, projectId: 1, name: planName, testCasesCount: 1, treeSelection: { leafsInclude: [91], inverted: false } }); }; assert.equal(await run({ ...context, RUNNER_TEMP: temporary, CHAT_TESTOPS_STATE_ROOT: state, - CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 0); - const plan = JSON.parse(await readFile(path.join(state, 'plan.json'))); - assert.equal(plan.actualResult, null); + CHAT_TESTOPS_OPERATION: 'bootstrap', GITHUB_EVENT_PATH: eventPath, ALLURE_JOB_RUN_ID: '9', ALLURE_TOKEN: 'SYNTHETIC_API_TOKEN' }, fetchImpl), 1); + const failure = JSON.parse(await readFile(path.join(state, 'failure.json'))); + assert.equal(failure.code, 'testops_plan_retired'); + await assert.rejects(readFile(path.join(state, 'plan.json')), { code: 'ENOENT' }); assert.equal(requests.filter(value => value.method === 'POST').length, 1, 'only auth token exchange; no case/plan write'); await assert.rejects(readFile(path.join(state, 'allure-results.json')), { code: 'ENOENT' }); }); diff --git a/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs b/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs index 9a6f803a7db..73dbb5f35f0 100644 --- a/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs +++ b/tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs @@ -2,7 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { fullCatalogCases, fullCatalogPlanName, catalogTargetSha } from './chat-full-catalog-testops-manifest.mjs'; import { admitDispatch, validateSelection, evidencePolicy, catalogWorkflow, catalogDefinition } from './chat-completion-testops-contract.mjs'; -import { bindingFromEnv, validatePilotSelection, verifyPilotUpload, completePilotRegistration } from './chat-completion-testops-api.mjs'; +import { bindingFromEnv, validatePilotSelection, verifyPilotUpload, verifyPilotCaseUpload, completePilotRegistration } from './chat-completion-testops-api.mjs'; import { executionMatrix, executionPartition, run } from './chat-completion-testops-cli.mjs'; import { mkdtemp, writeFile, readFile, readdir, rm, realpath } from 'node:fs/promises'; import os from 'node:os'; @@ -66,6 +66,22 @@ test('the complete 244 mapping uses compact actual IDs and preserves the legacy assert.equal(value.targetSha, catalogTargetSha); }); +test('selection accepts the exact master workflow URL returned by a build-server refresh', async () => { + const job = { id: 11, type: 'github', name: 'Chat catalog TestOps', + url: 'https://github.com/paperclipai/paperclip/blob/master/.github/workflows/chat-catalog-testops.yml', + parameters: [{ id: 34, name: 'Branch', defaultValue: 'master' }] }; + const refreshed = api({ '/api/job/11': job }); + const value = await validatePilotSelection(refreshed.get, env, context, 99); + assert.deepEqual(value.cases.map(row => row.caseId), selected.map(row => row.caseId)); + assert.equal(value.targetSha, catalogTargetSha); + for (const url of [job.url.replace('/master/', '/other-branch/'), job.url.replace('/paperclipai/', '/other-owner/'), + job.url.replace('chat-catalog-testops.yml', 'chat-catalog-testops-case.yml')]) { + const wrong = api({ '/api/job/11': { ...job, url } }); + await assert.rejects(validatePilotSelection(wrong.get, env, context, 99), /job_mismatch/); + assert.equal(wrong.calls.includes('/api/jobrun/77'), false); + } +}); + test('the actual full registration receipt fits one protected variable without a later source change', async () => { const prepared = { planId: 10, integrationId: 1, cases: selected.map(row => Object.fromEntries( ['key', 'scenario', 'executionId', 'fullName', 'registeredFullName', 'caseId'].map(key => [key, row[key]]))) }; @@ -103,14 +119,15 @@ test('full catalog source admission is independent from the old three-case workf assert.throws(() => validateSelection({ ...selection, targetSha: 'b'.repeat(40) }, context, '77'), /selection_binding/); }); -test('native and external partitions preserve exact selections, serial small runs and the GitHub output bound', () => { +test('native and external partitions preserve exact selections, bounded overlapping small runs and the GitHub output bound', () => { for (const rows of [selected, selected.slice(0, 3), selected.filter(row => row.externalInput), [selected[0], selected.find(row => row.externalInput)]]) { const input = { ...selection, cases: rows }, partition = executionPartition(input, context, '77'); const merged = [...partition.native.include, ...partition.external.include].map(row => row.executionId); assert.equal(new Set(merged).size, rows.length); assert.deepEqual(merged.sort(), rows.map(row => row.executionId).sort()); if (rows.length <= 3) { - assert.equal(partition.parallelism, '1'); assert.equal(partition.external.include.length, 0); + assert.equal(partition.parallelism, rows.some(row => row.externalInput) ? '1' : '2'); + assert.equal(partition.external.include.length, rows.filter(row => row.externalInput).length); } else { assert.equal(partition.parallelism, '8'); assert.ok(partition.external.include.every(row => row.external)); } @@ -224,3 +241,36 @@ test('the real bootstrap facade preserves fixed reconciliation failures and disc } finally { capture.mock.restore(); await rm(root, { recursive: true, force: true }); } } }); + +test('a finished case verifies while known sibling rows coexist in the same launch', async () => { + const f = api(); + const value = await verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]); + assert.equal(value.verification, 'VERIFIED'); + assert.deepEqual(value.results.map(row => row.caseId), [selected[0].caseId]); + assert.equal(f.calls.filter(route => /^\/api\/testresult\/\d+$/.test(route)).length, 1); +}); + +test('partial verification rejects foreign sibling identities before qualifying the current case', async () => { + const base = api(), first = await base.get('/api/testresult?launchId=12&page=0&size=100'); + first.content[1].testCaseId = 999999; + const f = api({ '/api/testresult?launchId=12&page=0&size=100': first }); + await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /foreign_result/); +}); + +test('partial verification retains the exact current-result jobrun binding', async () => { + const base = api(), wrong = await base.get('/api/testresult/2000'); wrong.jobRun.id = 78; + const f = api({ '/api/testresult/2000': wrong }); + await assert.rejects(verifyPilotCaseUpload(f.get, selection, context, selected[0].executionId, [exported[0]]), /result_mismatch/); +}); + +test('small external selections remain in the serial external lane while native overlap stays bounded', () => { + const external = selected.filter(row => row.externalInput).slice(0, 2); + assert.equal(external.length, 2); + for (const rows of [external, [selected.find(row => !row.externalInput), ...external]]) { + const partition = executionPartition({ ...selection, cases: rows }, context, '77'); + assert.deepEqual(partition.external.include.map(row => row.executionId), external.map(row => row.executionId)); + assert.ok(partition.native.include.every(row => !row.external)); + assert.equal(partition.parallelism, '1'); + assert.equal(partition.native.include.length + partition.external.include.length, rows.length); + } +}); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index c45a1d8a69d..b0c55f687c6 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -24,7 +24,7 @@ for (const [name, command] of [ "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", "tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs", - "tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-reconcile.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs"]], + "tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-reconcile.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs", "tests/runner-e2e/chat-catalog-testops-streaming.test.mjs"]], ]) { const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); if (result.error || result.signal) { diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index ec998eafdf0..e87e1a04585 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1905,7 +1905,8 @@ for (const execution of executions) { .filter({ hasText: `PAPERCLIP_E2E_WARM_T${completedTurn}_${nonce}` }).last(); await expect(turnReply).toBeVisible({ timeout: reviewUiTimeout() }); const pendingReview = waitingState.interactions.find(isPendingWarmConfirmation)!; - const reviewCard = page.locator(`[id=${JSON.stringify(`interaction-${pendingReview.id}`)}]`); + const reviewCard = page.getByTestId("task-chat-interaction") + .and(page.locator(`[id=${JSON.stringify(`interaction-${pendingReview.id}`)}]`)); await expect(reviewCard).toBeVisible({ timeout: reviewUiTimeout() }); await expect(reviewCard.getByRole("button", { name: "Continue work", exact: true })) .toBeEnabled({ timeout: reviewUiTimeout() }); diff --git a/tests/runner-e2e/testops-chat-reporter.test.mjs b/tests/runner-e2e/testops-chat-reporter.test.mjs index 917d6912f90..65c32e7a7e9 100644 --- a/tests/runner-e2e/testops-chat-reporter.test.mjs +++ b/tests/runner-e2e/testops-chat-reporter.test.mjs @@ -244,18 +244,6 @@ test('post-watch foreign backend binding returns safe nonzero and preserves orig assert.deepEqual(await readFile(path.join(f.state, 'job.json')), f.originalJob); }); -test('bootstrap workflow captures official context only after watch and retains exactly named closed receipts', async () => { - const workflow = await readFile(new URL('../../.github/workflows/chat-feature-testops.yml', import.meta.url), 'utf8'); - const start = workflow.indexOf(' - name: Register the public job without an evaluation or a synthetic result'); - const end = workflow.indexOf(' - name: Validate the one real TestOps selection', start); - const step = workflow.slice(start, end); - assert.ok(step.includes("if: needs.authorize.outputs.operation == 'bootstrap'")); - assert.match(step, /allurectl watch -- node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap\n\s+node tests\/runner-e2e\/testops-chat-reporter\.mjs bootstrap-context/); - assert.equal((step.match(/allurectl watch/g) ?? []).length, 1); assert.ok(!step.includes('continue-on-error')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration.json')); - assert.ok(workflow.includes('${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json')); -}); - test('post-watch refuses nonprivate state custody before any command auth or receipt write', async t => { const f = await bootstrapContextFixture(t); await chmod(f.state, 0o755); const result = await captureSafeErrors(() => reporter('bootstrap-context', undefined, f.env, () => assert.fail('no API'), () => assert.fail('no CLI'))); diff --git a/tests/runner-e2e/testops-chat-workflow.test.ts b/tests/runner-e2e/testops-chat-workflow.test.ts deleted file mode 100644 index 14484cbd30e..00000000000 --- a/tests/runner-e2e/testops-chat-workflow.test.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { readFileSync } from "node:fs"; -import { parse } from "yaml"; -import { expect, it } from "vitest"; -import { validateChatTestOpsDispatch } from "./testops-chat-dispatch.js"; - -const workflow = parse(readFileSync(new URL("../../.github/workflows/chat-feature-testops.yml", import.meta.url), "utf8")); -function context() { - return { env: { GITHUB_ACTIONS: "true", GITHUB_EVENT_NAME: "workflow_dispatch", GITHUB_REPOSITORY: "paperclipai/paperclip", GITHUB_REF: "refs/heads/master", - GITHUB_WORKFLOW_REF: "paperclipai/paperclip/.github/workflows/chat-feature-testops.yml@refs/heads/master", GITHUB_SHA: "a".repeat(40), GITHUB_RUN_ID: "12345", GITHUB_RUN_ATTEMPT: "1", GITHUB_ACTOR_ID: "123", TESTOPS_ALLOWED_ACTOR_IDS: "[123]" }, - event: { repository: { full_name: "paperclipai/paperclip", private: false, default_branch: "master" }, ref: "refs/heads/master", sender: { id: 123 }, inputs: { ALLURE_JOB_RUN_ID: "456" } as Record } }; -} -it("admits the actual TestOps service input and only the closed metadata bootstrap alternative", () => { - const f = context(); expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - f.event.inputs = { operation: "bootstrap" }; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("bootstrap"); - f.event.inputs.operation = "launch-plan"; expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("launch-plan"); - f.event.inputs.ALLURE_JOB_RUN_ID = "456"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); - delete f.event.inputs.ALLURE_JOB_RUN_ID; - f.event.inputs.operation = "other"; expect(() => validateChatTestOpsDispatch(f.env, f.event, 123)).toThrow(); -}); -it.each(["unlisted-trigger", "wrong-source", "replay", "extra-input", "missing-jobrun", "foreign-repo"])("rejects %s before any service credential operation", kind => { - const f = context(); let trigger = 123; - if (kind === "unlisted-trigger") trigger = 456; - if (kind === "wrong-source") f.env.GITHUB_REF = "refs/heads/unreviewed"; - if (kind === "replay") f.env.GITHUB_RUN_ATTEMPT = "2"; - if (kind === "extra-input") f.event.inputs.target_branch = "other"; - if (kind === "missing-jobrun") f.event.inputs = {}; - if (kind === "foreign-repo") f.event.repository.full_name = "foreign/repo"; - expect(() => validateChatTestOpsDispatch(f.env, f.event, trigger)).toThrow(); -}); -it("keeps provider and TestOps service credentials in separate jobs", () => { - const jobs = workflow.jobs; - for (const [id, job] of Object.entries(jobs) as Array<[string, any]>) { - const text = JSON.stringify(job); - if (text.includes("secrets.OPENROUTER_API_KEY")) expect(id).toBe("chat"); - if (text.includes("secrets.ALLURE_TOKEN")) expect(["select", "upload"]).toContain(id); - if (id !== "chat") expect(text).not.toContain("pnpm test:e2e:runner --"); - expect(job.steps.some((step: any) => step.run === "node tests/runner-e2e/testops-chat-dispatch.ts")).toBe(true); - } - expect(jobs.chat.environment).toBe("runner-e2e-private-paid"); - expect(jobs.select.environment).toBe("runner-e2e-testops-reporting"); - expect(jobs.upload.environment).toBe("runner-e2e-testops-reporting"); -}); -it("requires token-isolated selection before the fixed AWS run and keeps bootstrap outside all model jobs", () => { - expect(workflow.on).toHaveProperty("workflow_dispatch"); expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); - expect(Object.keys(workflow.on.workflow_dispatch.inputs)).toEqual(["operation", "ALLURE_JOB_RUN_ID", "ALLURE_USERNAME"]); - expect(workflow.on.workflow_dispatch.inputs.operation.default).toBe("evaluate"); - expect(workflow.jobs.chat.needs).toContain("select"); expect(workflow.jobs.build.needs).toContain("select"); - expect(workflow.jobs.chat.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.jobs.build.if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - const paid = workflow.jobs.chat.steps.find((s: any) => s.env?.OPENROUTER_API_KEY); - expect(paid.run).toContain("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0"); - expect(workflow.jobs.chat.steps.findIndex((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection")).toBeLessThan(workflow.jobs.chat.steps.indexOf(paid)); - expect(workflow.jobs.authorize.outputs.runner).toBe("runs-on/fleet=paperclip-public-pr-x64/env=public-ci"); - expect(workflow.jobs.chat["runs-on"]).toBe("${{ needs.authorize.outputs.runner }}"); -}); -it("uploads only the trusted safe projection after successful strict private report retention", () => { - const steps = workflow.jobs.report.steps; - expect(steps.find((s: any) => s.id === "export").if).toBe("always() && steps.private_report.outcome == 'success'"); - expect(steps.find((s: any) => s.with?.name?.startsWith("runner-e2e-testops-")).with.path).toBe("runner-e2e-testops/"); - expect(JSON.stringify(workflow)).not.toContain("MANUAL_PRIVATE_APPROVALS_JSON"); - expect(Object.values(workflow.jobs).flatMap((j: any) => j.steps).filter((s: any) => s.uses?.startsWith("actions/upload-artifact")).some((s: any) => s.with.path === "tests/runner-e2e/results/")).toBe(false); -}); - -it("initializes the actual TestOps association and never fabricates setup results", () => { - const steps = workflow.jobs.select.steps; - const installer = steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")); - expect(installer.uses).toBe("allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db"); - expect(installer.with["allurectl-version"]).toBe("2.23.1"); - const bootstrap = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs bootstrap")); - expect(bootstrap.run).toContain("mkdir -m 700"); - expect(bootstrap.run).toContain("allurectl watch -- node"); - expect(bootstrap.env.ALLURE_RESULTS).toBe("${{ runner.temp }}/chat-testops-bootstrap-results"); - const select = steps.find((s: any) => s.id === "selection"); - expect(select.run).toContain("allurectl watch -- node"); - for (const name of ["CHAT_TESTOPS_CASE_ID", "CHAT_TESTOPS_PLAN_ID", "CHAT_TESTOPS_JOB_ID"]) - expect(select.env[name]).toBe(`\${{ vars.${name} }}`); - const metadata = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-setup-")); - expect(metadata.if).toBe("always() && needs.authorize.outputs.operation == 'bootstrap'"); - expect(metadata.with.path).not.toContain("results"); -}); -it("authenticates the same-run selection again before converting and uploading", () => { - const steps = workflow.jobs.upload.steps; - const receipt = steps.find((s: any) => s.with?.path === "chat-testops-selection"); - expect(receipt.with["artifact-ids"]).toBe("${{ needs.select.outputs.artifact_id }}"); - const upload = steps.find((s: any) => s.run?.includes("testops-chat-reporter.mjs upload")); - expect(upload.run).toContain("allurectl watch -- node"); - expect(upload.env.EXPECTED_SELECTION_SHA256).toBe("${{ needs.select.outputs.sha256 }}"); - expect(steps.indexOf(receipt)).toBeLessThan(steps.indexOf(upload)); - expect(JSON.stringify(workflow.jobs.select)).not.toContain("OPENROUTER_API_KEY"); - expect(JSON.stringify(workflow.jobs.upload)).not.toContain("OPENROUTER_API_KEY"); -}); - -it("keeps the one plan launch outside Allure watch and every AWS/model job", () => { - const steps = workflow.jobs.select.steps; - const launch = steps.find((s: any) => s.run?.endsWith("testops-chat-reporter.mjs launch-plan")); - expect(launch.if).toBe("needs.authorize.outputs.operation == 'launch-plan'"); - expect(launch.run).not.toContain("watch"); - expect(launch.env.CHAT_TESTOPS_OPERATION_PARAMETER_ID).toBe("${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}"); - expect(launch.env.CHAT_TESTOPS_INTEGRATION_ID).toBe("${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_NONCE).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}"); - expect(launch.env.CHAT_TESTOPS_LAUNCH_RUN_NUMBER).toBe("${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}"); - expect(launch.env.TESTOPS_TRIGGERING_ACTOR_ID).toBe("${{ needs.authorize.outputs.triggering_actor_id }}"); - expect(launch.env.TESTOPS_ALLOWED_ACTOR_IDS).toBe("${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - expect(launch.env.GH_TOKEN).toBe("${{ github.token }}"); - expect(launch.run).not.toContain("sleep"); - expect(steps.find((s: any) => s.uses?.startsWith("allure-framework/setup-allurectl@")).if).toBe("needs.authorize.outputs.operation != 'launch-plan'"); - for (const name of ["build", "chat"]) expect(workflow.jobs[name].if).toBe("needs.authorize.outputs.operation == 'evaluate'"); - expect(workflow.concurrency["cancel-in-progress"]).toBe(false); -}); - -it("accepts the documented TestOps username metadata without treating it as an actor", () => { - const f = context(); f.event.inputs.ALLURE_USERNAME = "service-metadata"; - expect(validateChatTestOpsDispatch(f.env, f.event, 123).operation).toBe("evaluate"); - expect(() => validateChatTestOpsDispatch(f.env, f.event, 456)).toThrow(); - expect(JSON.stringify(workflow.jobs)).not.toContain("inputs.ALLURE_USERNAME"); -}); - -it("includes every native Node helper control in the normal unit owner", () => { - const launcher = readFileSync(new URL("./run-unit.mjs", import.meta.url), "utf8"); - const config = readFileSync(new URL("./vitest.config.ts", import.meta.url), "utf8"); - for (const file of ["chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs"]) { - expect(launcher).toContain(`"tests/runner-e2e/${file}"`); - expect(config).toContain(`"${file}"`); - } -}); - -it("verifies the actual server outcome after watch without masking an original failed upload", () => { - const steps = workflow.jobs.upload.steps; - const watch = steps.find((s: any) => s.id === "allure_upload"); - const verify = steps.find((s: any) => s.id === "verify_upload"); - expect(watch.run).toContain("allurectl watch -- node"); - expect(watch["continue-on-error"]).toBeUndefined(); - expect(verify.run).toBe("node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops"); - expect(verify.if).toBe("always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')"); - expect(verify.env).toEqual(watch.env); - expect(steps.indexOf(verify)).toBeGreaterThan(steps.indexOf(watch)); - const proof = steps.find((s: any) => s.with?.name?.startsWith("chat-testops-verification-")); - expect(proof.with.path.split("\n").filter(Boolean)).toHaveLength(2); - expect(proof.with.path).not.toContain("allure-results"); -}); - -it("carries only the existing source-reviewed lock through build and worker install", () => { - const build = workflow.jobs.build.steps; - const install = build.find((s: any) => s.run?.includes("pnpm install --frozen-lockfile")); - expect(install.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(install.run).not.toContain("--lockfile-only"); - const archive = build.find((s: any) => s.run?.includes("tar -czf chat-testops-build")); - expect(archive.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json"); - expect(archive.run).toContain("chat-testops-lock.json pnpm-lock.yaml"); - const worker = workflow.jobs.chat.steps.find((s: any) => s.name === "Verify immutable outputs and token-free TestOps selection"); - expect(worker.run).toContain("node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json"); - expect(worker.run).toContain("cmp chat-testops-lock.json chat-testops-worker-lock.json"); - expect(worker.run.indexOf("cmp chat-testops-lock.json")).toBeLessThan(worker.run.indexOf("pnpm install --frozen-lockfile")); -}); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 2cd918751d2..42707b008d3 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -7,7 +7,7 @@ export default defineConfig({ globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs"], + exclude: [...configDefaults.exclude, "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], testTimeout: 30_000, }, }); diff --git a/tests/runner-e2e/warm-managed-files.test.ts b/tests/runner-e2e/warm-managed-files.test.ts index 82bff9d4a76..a6de544736d 100644 --- a/tests/runner-e2e/warm-managed-files.test.ts +++ b/tests/runner-e2e/warm-managed-files.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { gradeWarmManagedFiles } from "./warm-managed-files.js"; +import type { RunnerApi } from "./api.js"; +import { gradeWarmManagedFiles, warmManagedFileEvidence } from "./warm-managed-files.js"; describe("managed warm file oracle", () => { const good = { turn: 2, nonce: "test", content: "T1-test\nT2-test\n", binary: Buffer.alloc(8388608, 93), deletedStatus: 404, native: true, save: { state: "saved", contract: "agent_files", checkpointStats: { copiedBytes: 16, copiedFiles: 1, hashedBytes: 16 } } }; @@ -10,3 +11,57 @@ describe("managed warm file oracle", () => { { save: { ...good.save, checkpointStats: { copiedBytes: 8388624, copiedFiles: 2, hashedBytes: 8388624 } } }, ])("rejects missing, stale, deleted, or fully recopied data", wrong => expect(gradeWarmManagedFiles({ ...good, ...wrong }).passed).toBe(false)); }); + +describe("managed warm file receipt evidence", () => { + const saved = { state: "saved", contract: "agent_files", errorCode: null, + checkpointStats: { copiedFiles: 3, copiedBytes: 8388625, hashedBytes: 8389685 } }; + type Event = { seq: number; runId: string; eventType: string; payload?: unknown }; + const receipt = (seq: number, payload: unknown = saved): Event => ({ seq, runId: "warm-run", eventType: "instruction_save", payload }); + function fixtureApi(events: Event[], summary: unknown = { instructionSave: saved }) { + const requestedPages: number[] = []; + const api = { + async get(path: string) { + if (path.includes("instructions-bundle")) return { content: "T1-test\n" }; + if (path.includes("/events?")) { + const after = Number(new URL(path, "http://fixture.invalid").searchParams.get("afterSeq")); + requestedPages.push(after); + return events.filter(event => event.seq > after).slice(0, 1000); + } + return { resultJson: summary }; + }, + request: { async get(path: string) { + return { ok: () => true, status: () => 200, + body: async () => path.includes("unchanged.bin") ? Buffer.alloc(8388608, 93) : Buffer.from("temporary") }; + } }, + } as unknown as RunnerApi; + return { api, requestedPages }; + } + const capture = (api: RunnerApi) => warmManagedFileEvidence(api, "agent", "warm-run", 1, "test", true); + + it("reads the durable save receipt when terminal result metadata omits it", async () => { + const { api } = fixtureApi([receipt(297)], {}); + expect((await capture(api)).passed).toBe(true); + }); + + it("reads all event pages before selecting the latest receipt", async () => { + const filler = Array.from({ length: 1000 }, (_, index) => ({ seq: index + 1, runId: "warm-run", eventType: "item.delta" })); + const { api, requestedPages } = fixtureApi([...filler, receipt(1001)], {}); + expect((await capture(api)).passed).toBe(true); + expect(requestedPages).toEqual([0, 1000]); + }); + + it.each([ + [], + [receipt(1), receipt(2, { ...saved, state: "pending_collection" })], + [receipt(1), receipt(2, { ...saved, errorCode: "AGENT_FILES_CHECKPOINT_UNSTABLE" })], + [receipt(1), receipt(2, { ...saved, checkpointStats: { copiedFiles: 1, copiedBytes: 8 } })], + ])("rejects missing or superseded evidence even when the run summary claims success", async (...events) => { + const { api } = fixtureApi(events); + expect((await capture(api)).passed).toBe(false); + }); + + it("rejects a receipt returned for a different run", async () => { + const { api } = fixtureApi([{ ...receipt(1), runId: "another-run" }]); + await expect(capture(api)).rejects.toThrow("different run"); + }); +}); diff --git a/tests/runner-e2e/warm-managed-files.ts b/tests/runner-e2e/warm-managed-files.ts index 2327e78d2c4..e4d1f7d8052 100644 --- a/tests/runner-e2e/warm-managed-files.ts +++ b/tests/runner-e2e/warm-managed-files.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import type { RunnerApi } from "./api.js"; +import { collectRunEvents } from "./run-observations.js"; const record = (v: unknown): Record => v && typeof v === "object" && !Array.isArray(v) ? v as Record : {}; export function gradeWarmManagedFiles(input: { @@ -26,7 +27,12 @@ export async function warmManagedFileEvidence(api: RunnerApi, agentId: string, r const binary = await api.request.get(`${base}notes%2Funchanged.bin&download=true`); if (!binary.ok()) throw new Error(`Managed binary download failed: ${binary.status()}`); const deleted = await api.request.get(`${base}notes%2Fdelete-me.txt&download=true`); - const run = await api.get<{ resultJson?: unknown }>(`/api/heartbeat-runs/${runId}`); - return gradeWarmManagedFiles({ turn, nonce, content: note.content, binary: await binary.body(), deletedStatus: deleted.status(), - save: record(run.resultJson).instructionSave, native }); + const events = await collectRunEvents<{ seq: number; runId: string; eventType: string; payload?: unknown }>((afterSeq, limit) => + api.get(`/api/heartbeat-runs/${encodeURIComponent(runId)}/events?afterSeq=${afterSeq}&limit=${limit}`)); + if (events.some(event => event.runId !== runId)) throw new Error("Managed file evidence returned events for a different run"); + // The durable run-log event is authoritative. Terminal result metadata may + // omit this receipt, and an older success must not hide a later failed save. + const receipt = events.filter(event => event.eventType === "instruction_save").at(-1); + return { ...gradeWarmManagedFiles({ turn, nonce, content: note.content, binary: await binary.body(), deletedStatus: deleted.status(), + save: receipt?.payload, native }), saveEventSeq: receipt?.seq ?? null }; } diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 91c0e3e9f29..ca84b9de5ab 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -606,8 +606,6 @@ describe("public repository paid workflow security", () => { "runner-full-stack-e2e.yml", "runner-live-evals.yml", "runner-protocol-live-evals.yml", - "chat-feature-testops.yml", - "chat-completion-testops.yml", ]); const names = (await readdir(workflowDirectory)).filter((name) => /\.ya?ml$/.test(name), @@ -893,24 +891,7 @@ describe("public repository paid workflow security", () => { for (const filename of await readdir(workflows)) { if (!/\.ya?ml$/u.test(filename) || ["runner-full-stack-e2e.yml", "runner-protocol-live-evals.yml"].includes(filename)) continue; const other = await readFile(path.join(workflows, filename), "utf8"); - if (["chat-feature-testops.yml", "chat-completion-testops.yml"].includes(filename)) { - const guard = filename === "chat-feature-testops.yml" ? "testops-chat-dispatch.ts" : "chat-completion-testops-cli.mjs guard"; - expect(other).toContain(`run: node tests/runner-e2e/${guard}`); - expect(other).toContain("ref: ${{ github.sha }}"); - expect(other).toContain("TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}"); - const chat = other.split("\n chat:")[1]!.split("\n report:")[0]!; - expect(chat).toContain("environment: runner-e2e-private-paid"); - expect(chat).toContain("needs: [authorize, select, build]"); - expect(chat).toContain(" - *guard"); - expect(chat).toContain("verify-selection"); - expect(chat).not.toContain("ALLURE_TOKEN"); - expect(chat.indexOf("verify-selection")).toBeLessThan(chat.indexOf("secrets.OPENROUTER_API_KEY")); - const report = other.split("\n report:")[1]!.split("\n upload:")[0]!; - expect(report).toContain("environment: runner-e2e-private-history"); - expect(report).toContain("manual-private-transfer.ts report-session"); - expect(report).not.toContain("ALLURE_TOKEN"); - continue; - } + for (const environment of environments) expect(other, filename).not.toContain(environment); } const protocol = await readFile(path.join(workflows, "runner-protocol-live-evals.yml"), "utf8"); diff --git a/ui/src/api/ai-connections.ts b/ui/src/api/ai-connections.ts index 419e60ed011..8ba35a7ba6f 100644 --- a/ui/src/api/ai-connections.ts +++ b/ui/src/api/ai-connections.ts @@ -1,6 +1,7 @@ -import type { AiConnectionList, AiConnectionUsage, CreateAiConnection, AiConnectionLoginIntent, LocalAiLoginAttempt, LocalAiLoginStatus } from "@paperclipai/shared"; -import { api } from "./client"; +import type { AiConnectionList, AiConnectionUsage, AiConnectionAccountIdentity, CreateAiConnection, AiConnectionLoginIntent, LocalAiLoginAttempt, LocalAiLoginStatus } from "@paperclipai/shared"; +import { api, type RequestOptions } from "./client"; export const aiConnectionsApi = { + accountIdentity: (companyId: string, connectionId: string, grantId: string, options?: RequestOptions) => api.get(`/companies/${companyId}/ai-connections/${connectionId}/identity?grantId=${encodeURIComponent(grantId)}`, { timeoutMs: 20_000, ...options }), loginSetup: (companyId: string, sessionId: string) => api.get<{ sessionId: string; intent: AiConnectionLoginIntent; environmentId: string; status: string; connectionId: string | null; grantId: string | null }>(`/companies/${companyId}/ai-connections/login/${encodeURIComponent(sessionId)}/setup`), probeUsage: (companyId: string, connectionId: string, grantId?: string) => api.get(`/companies/${companyId}/ai-connections/${connectionId}/usage${grantId ? `?grantId=${encodeURIComponent(grantId)}` : ""}`), startLocalLogin: (companyId: string, input: AiConnectionLoginIntent & { restart?: boolean }) => api.post(`/companies/${companyId}/ai-connections/local/attempts`, input), @@ -9,7 +10,7 @@ export const aiConnectionsApi = { cancelLocalLogin: (companyId: string, sessionId: string) => api.delete(`/companies/${companyId}/ai-connections/local/attempts/${sessionId}`), connectLocal: (companyId: string, input: AiConnectionLoginIntent & { localSessionId?: string }) => api.post<{ connectionId: string; grantId: string }>(`/companies/${companyId}/ai-connections/local`, input), activeRuns: (companyId: string, connectionId: string) => api.get>(`/companies/${companyId}/ai-connections/${connectionId}/active-runs`), - list: (companyId: string, agentId?: string) => api.get(`/companies/${companyId}/ai-connections${agentId ? `?agentId=${encodeURIComponent(agentId)}` : ""}`), + list: (companyId: string, agentId?: string, options?: RequestOptions) => api.get(`/companies/${companyId}/ai-connections${agentId ? `?agentId=${encodeURIComponent(agentId)}` : ""}`, { timeoutMs: 15_000, ...options }), create: (companyId: string, input: CreateAiConnection) => api.post<{ connectionId: string; grantId: string }>(`/companies/${companyId}/ai-connections`, input), setDefault: (companyId: string, grantId: string) => api.put(`/companies/${companyId}/ai-connections/default`, { grantId }), loginResult: (companyId: string, sessionId: string) => api.get<{ connectionId: string; grantId: string }>(`/companies/${companyId}/ai-connections/login/${encodeURIComponent(sessionId)}`), diff --git a/ui/src/api/client.test.ts b/ui/src/api/client.test.ts index 132fbc0b995..00156e922a8 100644 --- a/ui/src/api/client.test.ts +++ b/ui/src/api/client.test.ts @@ -41,6 +41,7 @@ beforeEach(() => { afterEach(() => { vi.restoreAllMocks(); vi.unstubAllGlobals(); + vi.useRealTimers(); }); describe("tenant-session recovery", () => { @@ -102,6 +103,55 @@ describe("tenant-session recovery", () => { }); describe("in-tab GET coalescing", () => { + it("times out a caller without cancelling another and starts a fresh fetch on retry", async () => { + vi.useFakeTimers(); + const first = deferred(); + fetchMock.mockReturnValueOnce(first.promise); + const survivorController = new AbortController(); + const survivor = api.get("/deadline-shared", { signal: survivorController.signal }); + const timed = api.get("/deadline-shared", { timeoutMs: 100 }).catch(error => error); + const sharedSignal = fetchMock.mock.calls[0][1].signal as AbortSignal; + await vi.advanceTimersByTimeAsync(100); + expect(await timed).toMatchObject({ status: 408, body: { error: "TIMEOUT" } }); + expect(sharedSignal.aborted).toBe(false); + fetchMock.mockResolvedValueOnce(jsonResponse({ fresh: true })); + await expect(api.get("/deadline-shared", { timeoutMs: 100 })).resolves.toEqual({ fresh: true }); + expect(fetchMock).toHaveBeenCalledTimes(2); + survivorController.abort(); + await expect(survivor).rejects.toMatchObject({ name: "AbortError" }); + expect(sharedSignal.aborted).toBe(true); + first.resolve(jsonResponse({ stale: true })); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("bounds response-body reads and aborts the last caller's fetch", async () => { + vi.useFakeTimers(); + const body = deferred(); + fetchMock.mockResolvedValue({ ok: true, status: 200, json: () => body.promise }); + const result = api.get("/deadline-body", { timeoutMs: 100 }).catch(error => error); + const signal = fetchMock.mock.calls[0][1].signal as AbortSignal; + await vi.advanceTimersByTimeAsync(100); + expect(await result).toMatchObject({ status: 408 }); + expect(signal.aborted).toBe(true); + body.resolve({ tooLate: true }); + await vi.advanceTimersByTimeAsync(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("clears a successful or cancelled caller's deadline", async () => { + vi.useFakeTimers(); + fetchMock.mockResolvedValueOnce(jsonResponse({ ok: true })); + await expect(api.get("/deadline-success", { timeoutMs: 100 })).resolves.toEqual({ ok: true }); + expect(vi.getTimerCount()).toBe(0); + const controller = new AbortController(); + fetchMock.mockReturnValueOnce(deferred().promise); + const result = api.get("/deadline-cancel", { signal: controller.signal, timeoutMs: 100 }); + controller.abort(); + await expect(result).rejects.toMatchObject({ name: "AbortError" }); + expect(vi.getTimerCount()).toBe(0); + }); + it("shares one underlying fetch for identical in-flight GETs", async () => { const d = deferred(); fetchMock.mockReturnValue(d.promise); diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 7b4b8f8c195..78ec560949b 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -1,6 +1,7 @@ import { getPageVisibility, getVisibilityHeaderValue } from "@/lib/page-visibility"; import { tenantSessionRecovery } from "@/lib/tenant-session-recovery"; import { ApiError, apiErrorFromResponse } from "./api-error"; +import { errorCopy } from "./errors"; export { ApiError, apiErrorFromResponse, type ApiErrorOptions } from "./api-error"; import { readApiJson } from "./response"; @@ -98,6 +99,8 @@ export async function apiFetch(input: string, init?: RequestInit): Promise(path: string, options?: RequestOptions): Promise { const releaseRef = () => { if (!activeEntry.refs.delete(ref)) return; // Last caller gone before the fetch settled → abort the shared request. - if (activeEntry.refs.size === 0 && inflightGets.get(path) === activeEntry) { - inflightGets.delete(path); + if (activeEntry.refs.size === 0) { + if (inflightGets.get(path) === activeEntry) inflightGets.delete(path); activeEntry.controller.abort(); } }; return new Promise((resolve, reject) => { - const onAbort = () => { + let timer: ReturnType | undefined; + const cleanup = () => { signal?.removeEventListener("abort", onAbort); + clearTimeout(timer); + }; + const onAbort = () => { + cleanup(); releaseRef(); reject(abortError()); }; if (signal) signal.addEventListener("abort", onAbort); + if (options?.timeoutMs !== undefined) { + timer = setTimeout(() => { + cleanup(); + // Other callers keep their request, but retries must not rejoin it. + if (inflightGets.get(path) === activeEntry) inflightGets.delete(path); + releaseRef(); + reject(new ApiError(errorCopy("TIMEOUT").body, 408, { error: "TIMEOUT" })); + }, options.timeoutMs); + } activeEntry.promise.then( (value) => { - signal?.removeEventListener("abort", onAbort); + cleanup(); activeEntry.refs.delete(ref); resolve(value as T); }, (err) => { - signal?.removeEventListener("abort", onAbort); + cleanup(); activeEntry.refs.delete(ref); reject(err); }, diff --git a/ui/src/api/issues.ts b/ui/src/api/issues.ts index abfc428b39a..739487a577c 100644 --- a/ui/src/api/issues.ts +++ b/ui/src/api/issues.ts @@ -194,6 +194,7 @@ export type IssueListFilters = { projection?: "index" | "rows"; attention?: "blocked"; status?: string; + displayStatus?: string; projectId?: string; parentId?: string; assigneeAgentId?: string; @@ -220,7 +221,7 @@ export type IssueListFilters = { q?: string; limit?: number; offset?: number; - sortField?: "updated" | "id"; + sortField?: "updated" | "created" | "id"; afterId?: string; sortDir?: "asc" | "desc"; }; @@ -229,6 +230,7 @@ function issueListSearchParams(filters?: IssueListFilters) { const params = new URLSearchParams(); if (filters?.attention) params.set("attention", filters.attention); if (filters?.status) params.set("status", filters.status); + if (filters?.displayStatus) params.set("displayStatus", filters.displayStatus); if (filters?.projectId) params.set("projectId", filters.projectId); if (filters?.parentId) params.set("parentId", filters.parentId); if (filters?.parentIds) params.set("parentIds", filters.parentIds.join(",")); diff --git a/ui/src/api/search.test.ts b/ui/src/api/search.test.ts new file mode 100644 index 00000000000..9f2d39324e5 --- /dev/null +++ b/ui/src/api/search.test.ts @@ -0,0 +1,14 @@ +import { expect, it, vi } from "vitest"; +import { parseSearchQuery } from "../lib/search-query-parser"; +import { searchApi } from "./search"; +import { api } from "./client"; + +vi.mock("./client", () => ({ api: { get: vi.fn() } })); + +it("keeps scheduled monitor waits in is:open search requests", async () => { + const parsed = parseSearchQuery("is:open deployment", {}); + await searchApi.search("company-1", { q: parsed.query, ...parsed.filters }); + const url = new URL(vi.mocked(api.get).mock.calls[0]![0], "http://localhost"); + expect(url.searchParams.getAll("displayStatus")).toEqual(["backlog", "todo", "in_progress", "monitoring", "in_review", "blocked"]); + expect(url.searchParams.get("q")).toBe("deployment"); +}); diff --git a/ui/src/api/search.ts b/ui/src/api/search.ts index 5715d8685b8..b9e97435685 100644 --- a/ui/src/api/search.ts +++ b/ui/src/api/search.ts @@ -1,4 +1,4 @@ -import type { CompanySearchResponse, CompanySearchScope, CompanySearchSort, IssuePriority, IssueStatus } from "@paperclipai/shared"; +import type { CompanySearchResponse, CompanySearchScope, CompanySearchSort, IssuePriority, IssueDisplayStatus } from "@paperclipai/shared"; import { api } from "./client"; export interface CompanySearchParams { @@ -6,7 +6,7 @@ export interface CompanySearchParams { scope?: CompanySearchScope; limit?: number; offset?: number; - status?: IssueStatus[]; + status?: IssueDisplayStatus[]; priority?: IssuePriority[]; assigneeAgentId?: string | null; assigneeUserId?: string; @@ -28,7 +28,7 @@ export const searchApi = { if (params.scope) search.set("scope", params.scope); if (params.limit !== undefined) search.set("limit", String(params.limit)); if (params.offset !== undefined) search.set("offset", String(params.offset)); - appendMulti(search, "status", params.status); + appendMulti(search, "displayStatus", params.status); appendMulti(search, "priority", params.priority); if (params.assigneeAgentId !== undefined) search.set("assigneeAgentId", params.assigneeAgentId ?? "null"); if (params.assigneeUserId !== undefined) search.set("assigneeUserId", params.assigneeUserId); diff --git a/ui/src/api/tools.ts b/ui/src/api/tools.ts index f8a726d4583..4bf3021dfc4 100644 --- a/ui/src/api/tools.ts +++ b/ui/src/api/tools.ts @@ -66,7 +66,7 @@ import type { ToolConnectionCreateCapabilities, ToolAppMetadataPreflightResult, } from "@paperclipai/shared"; -import { api, detachInflightGet } from "./client"; +import { api, detachInflightGet, type RequestOptions } from "./client"; /** * Tools & Access API client (Phase 6, PAP-10389). @@ -367,8 +367,8 @@ export const toolsApi = { // --- Identity grants (PAP-17835): who a connection acts as. The response // carries server-computed capabilities and the audience member directory, so // the UI never rebuilds the permission matrix from `membershipRole`. - listConnectionGrants: (connectionId: string) => - api.get(`/tool-connections/${connectionId}/grants`), + listConnectionGrants: (connectionId: string, options?: RequestOptions) => + api.get(`/tool-connections/${connectionId}/grants`, { timeoutMs: 15_000, ...options }), createConnectionGrantDelegation: (connectionId: string, grantId: string, agentId: string) => api.post( `/tool-connections/${connectionId}/grants/${grantId}/delegations`, diff --git a/ui/src/components/BlockedInboxView.tsx b/ui/src/components/BlockedInboxView.tsx index 0cb145a359f..fb854d377df 100644 --- a/ui/src/components/BlockedInboxView.tsx +++ b/ui/src/components/BlockedInboxView.tsx @@ -376,7 +376,7 @@ function BlockedInboxRow({ showDivider={presentation === "legacy"} statusSlot={presentation === "task" ? showStatusColumn - ? + ? :