Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-06 14:02:36 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge master with controller identity probe fix

Browse Source 
Co-Authored-By: Paperclip <noreply@paperclip.ing>

* commit '946735d03a55c1fb4cc5434d6bf1d87742daa9df':
  Persist task sorting in URLs and show newest routine tasks first
  perf(ui): reuse parent metadata when loading the inbox (#18645)
  fix(workspaces): reduce temporary disk use during remote capture (#18644)
  fix(server): keep controller identity independent of Git probes (#18649)
  fix(claude-local): skip the 2.1.291 copy hint in the setup-token prompt parser (#15550)
  fix(db): attribute bounded captures to execution owners (#18634)
  fix(ui): add space above failed attempt notices (#18636)
  fix(runner): handle blank assigned-tool schema references (#18637)
  fix(server): settle cleanup after reusable sandbox destruction (#18633)
  fix: retry saved chat input after a successful run (#18621)
  test(server): stop voice report waits from crossing refresh deadlines (#18628)
  feat(ui): show scheduled task waits as Monitoring (#18624)
  feat(observability): trace Daytona lease acquisition and resume (#18626)
  test(evals): retire duplicate chat TestOps plans (#18625)
  fix(runner): keep requests responsive during retired backup cleanup (#18619)
  fix(redaction): scan long letter/digit runs in linear time (#16999)
This commit is contained in:
DottaandPaperclip committed 2026-11-02 13:35:05 -06:00
2 parents 14ee3f02c4 946735d03a
commit 541dd57986
197 files changed
+4470 -2162
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/chat-completion-testops.yml
-504
View File
@@ -1,504 +0,0 @@
name: Chat completion TestOps
run-name: Chat completion TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Run the exact TestOps selection, register its plan, or launch it once
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps job-run identity; source, model and clocks are fixed
ALLURE_USERNAME:
type: string
required: false
description: Reserved service metadata; never grants authority
permissions:
contents: read
actions: read
concurrency:
group: chat-completion-testops
cancel-in-progress: false
env:
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
operation: ${{ steps.guard.outputs.operation }}
target_sha: ${{ steps.guard.outputs.target_sha }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify actual main-only dispatch and both actors
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
run: node tests/runner-e2e/chat-completion-testops-cli.mjs guard
select:
name: Register the dedicated plan or validate its real TestOps selection
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-testops-reporting
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
execution_ids: ${{ steps.selection.outputs.execution_ids }}
matrix: ${{ steps.selection.outputs.matrix }}
policy: ${{ steps.selection.outputs.policy }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register three exact cases with no synthetic test result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-bootstrap-results
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap
node tests/runner-e2e/chat-completion-testops-cli.mjs bootstrap-context
- name: Validate case, plan, job, launch and same-run source before provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-selection-results
CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }}
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs selection
- name: Request the one reviewed plan launch and exit
if: needs.authorize.outputs.operation == 'launch-plan'
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
CHAT_COMPLETION_TESTOPS_BINDING: ${{ vars.CHAT_COMPLETION_TESTOPS_BINDING }}
CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_RUN_NUMBER }}
CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_COMPLETION_TESTOPS_LAUNCH_NONCE }}
run: node tests/runner-e2e/chat-completion-testops-cli.mjs launch-plan
- name: Retain token-free registration and launch receipts
if: always() && needs.authorize.outputs.operation != 'evaluate'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-completion-testops-bootstrap/
${{ runner.temp }}/chat-completion-testops-launch-plan/
retention-days: 30
if-no-files-found: error
- name: Retain the exact authenticated selection
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-selection-${{ github.run_id }}-1
path: testops-pilot-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable catalog source without service or model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- name: Resolve the fixed candidate graph with lifecycle scripts disabled
run: |
pnpm install --lockfile-only --ignore-scripts
pnpm install --frozen-lockfile --ignore-scripts
- name: Build existing TypeScript and native Runner outputs
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
- name: Exercise the real workflow and pinned source host contract before model access
run: node --import ./cli/node_modules/tsx/dist/loader.mjs .testops-trusted/tests/runner-e2e/chat-completion-testops-host-control.mjs
- name: Build native Runner outputs
run: pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package the resolved lock and exact source outputs
env:
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$TARGET_SHA"
tar -czf chat-completion-build.tar.gz pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-completion-build.tar.gz > chat-completion-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-build-${{ github.run_id }}-1
path: |
chat-completion-build.tar.gz
chat-completion-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: ${{ matrix.executionId }}
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
strategy:
fail-fast: false
max-parallel: 1
matrix: ${{ fromJSON(needs.select.outputs.matrix) }}
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
actions: read
id-token: write
env: &retention
TARGET_SHA: ${{ needs.authorize.outputs.target_sha }}
MANUAL_PRIVATE_IDS: ${{ needs.select.outputs.execution_ids }}
MANUAL_PRIVATE_APPROVALS_JSON: ${{ needs.select.outputs.policy }}
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
steps:
- *checkout
- *node
- *guard
- *pnpm
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.target_sha }}
persist-credentials: false
- name: Reject target-owned trusted helper paths
id: target_boundary
run: test -z "$(git ls-tree --name-only HEAD -- .testops-trusted)"
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-completion-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- name: Verify source-bound build and authenticated selection before credentials
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
test "$(git -C .testops-trusted rev-parse HEAD)" = "$GITHUB_SHA"
node .testops-trusted/tests/runner-e2e/chat-completion-testops-cli.mjs verify-selection chat-completion-testops-selection/testops-pilot-selection.json
(cd chat-completion-build && sha256sum -c chat-completion-build.tar.gz.sha256)
tar -xzf chat-completion-build/chat-completion-build.tar.gz -C "$GITHUB_WORKSPACE"
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && node tests/runner-e2e/prepare-report-lock.mjs && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-retention-store")
- name: Qualify the pinned provider runtime and preinstalled browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Bind exact private evidence under the authenticated TestOps authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only this cell's exact-key storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify private custody before delivering the model key
env: &cell_session
SELECTED_EXECUTION_ID: ${{ matrix.executionId }}
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the selected original case with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_CHAT_COMPLETION_TESTOPS: v1
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-${{ matrix.executionId }}
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id "${{ matrix.executionId }}" --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain original evidence after success or failure
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and build the exact safe TestOps projection
needs: [authorize, select, build, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success' && needs.build.result == 'success' && (needs.chat.result == 'success' || needs.chat.result == 'failure')
runs-on: ubuntu-latest
timeout-minutes: 20
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies with lifecycle scripts disabled
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-completion-report-store"
- name: Prepare unchanged exact-key report and cell policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact first-cell reader
id: reader_0
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the first exact original
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.reader_0.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_0.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_0.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Obtain exact second-cell reader
id: reader_1
if: fromJSON(steps.session.outputs.cell_count) > 1
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_1 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the second exact original
if: steps.reader_1.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '1'
AWS_ACCESS_KEY_ID: ${{ steps.reader_1.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_1.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_1.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Obtain exact third-cell reader
id: reader_2
if: fromJSON(steps.session.outputs.cell_count) > 2
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_2 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the third exact original
if: steps.reader_2.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '2'
AWS_ACCESS_KEY_ID: ${{ steps.reader_2.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.reader_2.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.reader_2.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
- name: Verify every selected original and its completeness receipt
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing report while preserving failed source grades
id: normalize
continue-on-error: true
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: ${{ needs.select.outputs.execution_ids }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: refs/heads/codex/chat-catalog-master-integration
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only exact reviewed synthetic fixture results
id: export
if: always() && steps.private_report.outcome == 'success'
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/chat-completion-testops-export.ts
- name: Require a complete safe projection even after a failed normalization
if: always()
env:
EXPORT_OUTCOME: ${{ steps.export.outcome }}
run: test "$EXPORT_OUTCOME" = success
- name: Retain the metadata-only TestOps projection
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-projection-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Record and independently verify each TestOps result
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.report.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: chat-completion-testops-projection-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-completion-testops-selection
- *allure
- name: Upload original outcomes, checks and private-evidence receipts
id: allure_upload
env: &upload_env
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.selection_sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-completion-allure-results
run: allurectl watch -- node tests/runner-e2e/chat-completion-testops-cli.mjs upload runner-e2e-testops
- name: Verify actual selected results and complete result-owned attachments
id: verification
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *upload_env
run: node tests/runner-e2e/chat-completion-testops-cli.mjs verify-upload runner-e2e-testops
- name: Retain closed TestOps verification receipts
if: always() && !cancelled() && (steps.verification.outcome == 'success' || steps.verification.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-completion-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-completion-testops-verify-upload/
${{ runner.temp }}/chat-completion-testops-upload/exported.json
retention-days: 30
if-no-files-found: error
.github/workflows/chat-feature-testops.yml
-451
View File
@@ -1,451 +0,0 @@
name: Chat feature TestOps
run-name: Chat feature TestOps / ${{ inputs.operation || 'evaluate' }} / ${{ inputs.ALLURE_JOB_RUN_ID || github.run_id }}
on:
workflow_dispatch:
inputs:
operation:
type: choice
options: [evaluate, bootstrap, launch-plan]
default: evaluate
description: Fixed evaluation, zero-result registration, or one TestOps plan launch
ALLURE_JOB_RUN_ID:
type: string
required: false
description: TestOps supplies its job-run identity; no test/source selector is accepted
ALLURE_USERNAME:
type: string
required: false
description: Reserved TestOps service metadata; never used as authorization
permissions:
contents: read
concurrency:
group: chat-feature-testops
cancel-in-progress: false
jobs:
authorize:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.guard.outputs.sha }}
operation: ${{ steps.guard.outputs.operation }}
triggering_actor_id: ${{ steps.guard.outputs.triggering_actor_id }}
runner: runs-on/fleet=paperclip-public-pr-x64/env=public-ci
steps:
- &checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- &node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- &guard
name: Verify fixed default-source dispatch and both actor identities
id: guard
env:
GH_TOKEN: ${{ github.token }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
NODE_OPTIONS: ''
NODE_PATH: ''
run: node tests/runner-e2e/testops-chat-dispatch.ts
select:
name: Validate TestOps selection or register zero-result setup
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
outputs:
artifact_id: ${{ steps.selection_upload.outputs.artifact-id }}
sha256: ${{ steps.selection.outputs.sha256 }}
steps:
- *checkout
- *node
- *guard
- &allure
if: needs.authorize.outputs.operation != 'launch-plan'
uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: 2.23.1
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Register the public job without an evaluation or a synthetic result
if: needs.authorize.outputs.operation == 'bootstrap'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-bootstrap-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs bootstrap
node tests/runner-e2e/testops-chat-reporter.mjs bootstrap-context
- name: Validate the one real TestOps selection before any build or provider access
id: selection
if: needs.authorize.outputs.operation == 'evaluate'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-selection-results
run: |
mkdir -m 700 "$ALLURE_RESULTS"
allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs selection
- name: Start exactly the validated TestOps plan and exit without waiting for its child
if: needs.authorize.outputs.operation == 'launch-plan'
env:
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
CHAT_TESTOPS_CASE_ID: ${{ vars.CHAT_TESTOPS_CASE_ID }}
CHAT_TESTOPS_PLAN_ID: ${{ vars.CHAT_TESTOPS_PLAN_ID }}
CHAT_TESTOPS_JOB_ID: ${{ vars.CHAT_TESTOPS_JOB_ID }}
CHAT_TESTOPS_GITHUB_WORKFLOW_ID: ${{ vars.CHAT_TESTOPS_GITHUB_WORKFLOW_ID }}
CHAT_TESTOPS_OPERATION_PARAMETER_ID: ${{ vars.CHAT_TESTOPS_OPERATION_PARAMETER_ID }}
CHAT_TESTOPS_INTEGRATION_ID: ${{ vars.CHAT_TESTOPS_INTEGRATION_ID }}
CHAT_TESTOPS_LAUNCH_NONCE: ${{ vars.CHAT_TESTOPS_LAUNCH_NONCE }}
CHAT_TESTOPS_LAUNCH_RUN_NUMBER: ${{ vars.CHAT_TESTOPS_LAUNCH_RUN_NUMBER }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
GH_TOKEN: ${{ github.token }}
run: node tests/runner-e2e/testops-chat-reporter.mjs launch-plan
- name: Retain only closed metadata from zero-result setup
if: always() && needs.authorize.outputs.operation == 'bootstrap'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-setup-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-bootstrap-state/metadata.json
${{ runner.temp }}/chat-testops-bootstrap-state/once.json
${{ runner.temp }}/chat-testops-bootstrap-state/plan.json
${{ runner.temp }}/chat-testops-bootstrap-state/job.json
${{ runner.temp }}/chat-testops-bootstrap-state/failure.json
${{ runner.temp }}/chat-testops-bootstrap-state/registration.json
${{ runner.temp }}/chat-testops-bootstrap-state/registration-failure.json
retention-days: 30
if-no-files-found: error
- name: Retain the one plan-launch receipt and stop state
if: always() && needs.authorize.outputs.operation == 'launch-plan'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-plan-launch-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-launch-plan-state/once.json
${{ runner.temp }}/chat-testops-launch-plan-state/launch.json
${{ runner.temp }}/chat-testops-launch-plan-state/failure.json
retention-days: 30
if-no-files-found: error
- name: Retain only the token-free exact selection receipt
id: selection_upload
if: steps.selection.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-selection-${{ github.run_id }}-1
path: testops-selection.json
retention-days: 30
if-no-files-found: error
build:
name: Build the immutable shipped native candidate without model credentials
needs: [authorize, select]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 45
outputs:
artifact_id: ${{ steps.bundle.outputs.artifact-id }}
steps:
- *checkout
- *node
- *guard
- &pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: 'false'
NPM_CONFIG_FUND: 'false'
NPM_CONFIG_UPDATE_NOTIFIER: 'false'
with:
version: 9.15.4
- name: Install only the source-reviewed graph without lifecycle scripts
run: |
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
pnpm install --frozen-lockfile --ignore-scripts
- name: Build the existing TypeScript and native runner owners
run: |
pnpm --filter @paperclipai/paperclip-runner build:typescript
pnpm --filter @paperclipai/paperclip-runner build:runner-binaries
- name: Package source-bound outputs and resolved lock
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$GITHUB_SHA"
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-lock.json
tar -czf chat-testops-build.tar.gz chat-testops-lock.json pnpm-lock.yaml packages/paperclip-eval-kernel/dist packages/paperclip-runner/dist packages/paperclip-runner/runner/target/debug/paperclip-runnerd packages/paperclip-runner/runner/target/debug/conformance-tracer packages/paperclip-runner/runner/target/debug/fake-harness packages/paperclip-runner/runner/target/debug/fake-codex-app-server packages/paperclip-runner/runner/target/debug/fake-acpx-sidecar
sha256sum chat-testops-build.tar.gz > chat-testops-build.tar.gz.sha256
- name: Retain credential-free immutable outputs
id: bundle
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-build-${{ github.sha }}-${{ github.run_id }}-1
path: |
chat-testops-build.tar.gz
chat-testops-build.tar.gz.sha256
retention-days: 1
if-no-files-found: error
chat:
name: One real native OpenCode backlog request on AWS
needs: [authorize, select, build]
if: needs.authorize.outputs.operation == 'evaluate'
runs-on: ${{ needs.authorize.outputs.runner }}
timeout-minutes: 32
environment: runner-e2e-private-paid
permissions:
contents: read
id-token: write
env: &retention
PAPERCLIP_TESTOPS_CHAT_MAIN_ADMISSION: v1
TESTOPS_ADMITTED_SHA: ${{ needs.authorize.outputs.sha }}
TESTOPS_ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}
TESTOPS_TRIGGERING_ACTOR_ID: ${{ needs.authorize.outputs.triggering_actor_id }}
TARGET_SHA: ${{ needs.authorize.outputs.sha }}
MANUAL_PRIVATE_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
SELECTED_EXECUTION_ID: chat-features.runner-opencode-deepseek-1017.local.create-backlog
RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }}
NODE_OPTIONS: ''
NODE_PATH: ''
AWS_EC2_METADATA_DISABLED: 'true'
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Check out the same admitted source for trusted retention
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ needs.authorize.outputs.sha }}
path: .testops-trusted
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.build.outputs.artifact_id }}
path: chat-testops-build
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- name: Verify immutable outputs and token-free TestOps selection
env:
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
run: |
set -euo pipefail
(cd chat-testops-build && sha256sum -c chat-testops-build.tar.gz.sha256)
tar -xzf chat-testops-build/chat-testops-build.tar.gz -C "$GITHUB_WORKSPACE"
node tests/runner-e2e/testops-chat-reporter.mjs verify-selection chat-testops-selection/testops-selection.json
node tests/runner-e2e/testops-chat-lock.ts > chat-testops-worker-lock.json
cmp chat-testops-lock.json chat-testops-worker-lock.json
cp pnpm-lock.yaml .testops-trusted/pnpm-lock.yaml
pnpm install --frozen-lockfile --ignore-scripts
(cd .testops-trusted && pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-retention-store")
- name: Materialize and qualify pinned OpenCode runtime and browser
run: |
node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs
node -e 'const fs=require("node:fs");fs.chmodSync(process.execPath,fs.statSync(process.execPath).mode&~0o022);if(fs.statSync(process.execPath).mode&0o022)throw Error("Writable provider Node")'
google-chrome --version
pnpm exec playwright install ffmpeg
- name: Prepare private evidence with shipped-main authority
id: private
env:
TARGET_ROOT: ${{ github.workspace }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts prepare
- name: Obtain only the exact private cell storage session
id: cell_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.private.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Verify strict private custody before the model key
env: &cell_session
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
AWS_ACCESS_KEY_ID: ${{ steps.cell_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.cell_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.cell_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts verify
- name: Execute only the fixed original request with zero retries
env:
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT: ${{ steps.private.outputs.root }}/preflight.json
PRIVATE_EVIDENCE_ROOT: ${{ steps.private.outputs.root }}
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1-chat-features.runner-opencode-deepseek-1017.local.create-backlog
PAPERCLIP_E2E_ACCOUNTING_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
PAPERCLIP_PLAYWRIGHT_CHANNEL: chrome
run: pnpm test:e2e:runner -- --id chat-features.runner-opencode-deepseek-1017.local.create-backlog --max-parallel 1 --max-automatic-retries 0 > "$PRIVATE_EVIDENCE_ROOT/launcher-output.log" 2>&1
- name: Retain exact original grade and private evidence, including failures
if: always() && steps.cell_credentials.outcome == 'success'
env: *cell_session
run: node .testops-trusted/cli/node_modules/tsx/dist/cli.mjs .testops-trusted/tests/runner-e2e/manual-private-transfer.ts upload-cell
report:
name: Retain private originals and export only reviewed fixture results
needs: [authorize, select, chat]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 15
environment: runner-e2e-private-history
permissions:
contents: read
actions: read
id-token: write
env: *retention
steps:
- *checkout
- *node
- *guard
- *pnpm
- name: Install trusted reporting dependencies only
run: |
node tests/runner-e2e/prepare-report-lock.mjs
pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/chat-report-store"
- name: Prepare exact private report and cell-read policies
id: session
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts report-session
- name: Obtain private report writer
id: report_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.session_policy }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Obtain exact private cell reader
id: read_credentials
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
with:
role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }}
aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
inline-session-policy: ${{ steps.session.outputs.read_policy_0 }}
output-env-credentials: false
output-credentials: true
disable-retry: true
action-timeout-s: 60
- name: Download the exact private original and verify all inventories
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
SELECTED_CELL_INDEX: '0'
AWS_ACCESS_KEY_ID: ${{ steps.read_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.read_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.read_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: |
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts download-cell
node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts verify-downloads
- name: Build the existing normalized campaign without exposing original logs
env:
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: ${{ github.workspace }}/selected-runner-e2e
PAPERCLIP_RUNNER_E2E_REPORT_OUT: ${{ github.workspace }}/runner-e2e-merged-report/normalized
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: '["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]'
PAPERCLIP_E2E_CAMPAIGN_ID: gha-${{ github.run_id }}-1
PAPERCLIP_RUNNER_E2E_SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
PAPERCLIP_RUNNER_E2E_SOURCE_REF: ${{ github.ref }}
run: |
mkdir -p runner-e2e-merged-report
pnpm test:e2e:runner:report > runner-e2e-merged-report/report.log 2>&1
- name: Retain the complete merged report privately even when its grade failed
id: private_report
if: always() && steps.report_credentials.outcome == 'success'
env:
PRIVATE_REPORT_ROOT: ${{ github.workspace }}
PRIVATE_REPORT_SOURCE: ${{ github.workspace }}/runner-e2e-merged-report
AWS_ACCESS_KEY_ID: ${{ steps.report_credentials.outputs.aws-access-key-id }}
AWS_SECRET_ACCESS_KEY: ${{ steps.report_credentials.outputs.aws-secret-access-key }}
AWS_SESSION_TOKEN: ${{ steps.report_credentials.outputs.aws-session-token }}
AWS_DEFAULT_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/manual-private-transfer.ts upload-report
- name: Export only the bound normalized result, JUnit and marked PNGs
id: export
if: always() && steps.private_report.outcome == 'success'
env:
TESTOPS_CHAT_TARGET_SHA: ${{ needs.authorize.outputs.sha }}
run: node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/testops-chat-export.ts
- name: Upload only the sanitized TestOps projection
if: always() && steps.export.outcome == 'success'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops/
retention-days: 30
if-no-files-found: error
upload:
name: Report the original outcome to its exact TestOps case
needs: [authorize, select, chat, report]
if: always() && !cancelled() && needs.authorize.outputs.operation == 'evaluate' && needs.select.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
steps:
- *checkout
- *node
- *guard
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-testops-${{ github.run_id }}-1
path: runner-e2e-testops
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
artifact-ids: ${{ needs.select.outputs.artifact_id }}
path: chat-testops-selection
- *allure
- name: Verify and upload only the bound safe case outcome
id: allure_upload
env: &testops_reporter
ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }}
ALLURE_ENDPOINT: https://paperclip.testops.cloud
ALLURE_PROJECT_ID: '1'
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
EXPECTED_SELECTION_SHA256: ${{ needs.select.outputs.sha256 }}
ALLURE_RESULTS: ${{ runner.temp }}/chat-testops-allure-results
run: allurectl watch -- node tests/runner-e2e/testops-chat-reporter.mjs upload runner-e2e-testops
- name: Verify the actual TestOps result after upload processing
id: verify_upload
if: always() && !cancelled() && (steps.allure_upload.outcome == 'success' || steps.allure_upload.outcome == 'failure')
env: *testops_reporter
run: node tests/runner-e2e/testops-chat-reporter.mjs verify-upload runner-e2e-testops
- name: Retain only closed actual TestOps verification metadata
if: always() && !cancelled() && (steps.verify_upload.outcome == 'success' || steps.verify_upload.outcome == 'failure')
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: chat-testops-verification-${{ github.run_id }}-1
path: |
${{ runner.temp }}/chat-testops-verify-upload-state/verification.json
${{ runner.temp }}/chat-testops-verify-upload-state/failure.json
retention-days: 30
if-no-files-found: error
DESIGN.md
+2 -2
View File
@@ -117,8 +117,8 @@ completes without a provider reply. Preserve the reply through persistence.
Task chat shows execution errors and waits only while they remain relevant.
Exhausted attempts use a padded, rounded notice with a destructive-tinted background,
an error icon, and foreground text. The surrounding layout supplies separation from
the conversation. Automatic retry countdowns keep their quieter inline styling.
an error icon, and foreground text. A wrapper adds top padding above the tinted
surface. Automatic retry countdowns keep their quieter inline styling.
Completing or cancelling a task hides its old execution notices. A newer attempt by
the same agent or an explicit successor supersedes earlier run notices; an
unresolved execution hold remains visible. Historical turns keep their responses,
doc/DATABASE.md
+59 -19
View File
@@ -522,23 +522,59 @@ leave database construction unchanged. An expired window stays inactive after a
restart; remove the setting after the investigation.
The diagnostic measures awaited Postgres.js `unsafe` responses before Drizzle
column decoding. It retains the Drizzle builder caller for normal, prepared,
transaction, and dedicated-pool execution. It issues no queries.
It logs the public server caller locations, static schema table, operation,
query/row counts, and response value sizes. It emits at most 2,048 aggregate
groups and a summary. Measurement stops at the deadline. Environment-enabled
column decoding. It issues no queries. Configuration is read when each database
client is created; setting the environment on an already-running process does
not install the capture on existing clients. Prepare the deployment artifact
before choosing the UTC window, then start the window after rollout and warm-up.
Server entry points use `withDatabaseOperation(owner, action)` to name the work
that executes queries. Owners are fixed semantic labels, never request paths,
company or run IDs, user input, or SQL. The execution owner is captured for each
query and retained through completion, including transactions and dedicated
pools. A prepared query reused by different operations keeps the owner of each
execution. The builder caller is separate from that execution owner.
Owner context is active while a valid future window is armed, up to 30 minutes
before its start, so work that crosses the start boundary keeps its owner. Query
accounting starts only at the window start. With no armed window, or after
expiry, the owner helper calls the action directly without a context scope.
The capture aggregates owner, static schema table, operation, calls, returned
rows, query errors, and total/maximum completion wall time. `totalWallMs` and
`maxWallMs` include pool, database, network, and event-loop waits; they are not
CPU measurements, and concurrent query durations overlap. Timing excludes the
capture's response sizing. Rank calls and returned rows both per second and per
completed service operation; a busy owner is not necessarily wasteful.
Response sizing samples successful results 1, 17, 33, and so on per aggregate
group (`sampleEvery: 16`). It examines at most
256 cells, 65,536 string code units, and 65,536 value bytes per sampled result;
an individual string over 16,384 code units is not measured. Parsed objects and
other complex values are not serialized. Oversized, complex, or unexamined
values leave an incomplete sample. `sampledValueBytes` is a lower bound over
sampled primitive values, not a total response-size estimate or network bytes.
Do not extrapolate it as the total bytes returned by the database. Inspect
`sampledResults`, `sampledRows`, `measuredValues`, `unmeasuredValues`, and
`incompleteSamples` alongside it. `sizingWallMs` records the diagnostic's sizing
work separately. Use a separate capture-disabled CPU profile to measure normal
processing cost.
The diagnostic emits at most 2,048 aggregate groups and a summary. Measurement
stops at the deadline. Environment-enabled
captures emit at most 64 records per second and can take another 33 seconds to
finish writing the bounded output. Wait for the summary before collecting it.
It never logs SQL, parameters, result
values, connection strings, or query errors. In-flight queries continue normally.
For a bundled server, keep the linked source maps and enable Node's
`--enable-source-maps` option so the diagnostic can resolve public caller
locations. Restore the prior Node options after capture. Sizes still count when
caller locations are unavailable.
Semantic owners work in a bundled server without source maps. For queries
without an explicit owner, keep the linked source maps and enable Node's
`--enable-source-maps` option to resolve public caller locations. Restore prior
Node options after capture. Known semantic owners avoid repeated stack capture.
Queries without an available owner/caller still contribute to the totals.
Look for `paperclip.db_response_capture.entry.v1` and
`paperclip.db_response_capture.summary.v1` in the process logs. A log provider
Look for `paperclip.db_response_capture.entry.v2` and
`paperclip.db_response_capture.summary.v2` in the process logs. Version 2 uses
sampled, bounded primitive-value sizing; do not interpret it as the version 1
all-result byte estimate. A log provider
can store `event` as a structured field instead of message text. Filter by that
field or read the bounded window from the exact deployment. Check the summary
and group totals, including the unattributed overflow group.
@@ -552,22 +588,26 @@ Fetch a complete capture as follows:
from zero through `groupPages - 1`, filtering on the entry event, `captureId`,
and `groupPage`. Request at least `groupPageSize` matching records per page.
A structured query can use
`@event:paperclip.db_response_capture.entry.v1 AND @captureId:<id> AND @groupPage:0`.
`@event:paperclip.db_response_capture.entry.v2 AND @captureId:<id> AND @groupPage:0`.
3. If the provider's record limit is smaller than a page, fetch each
`groupIndex` separately with the same capture ID. Each index matches one
entry. Do not split by timestamp: entries can share a timestamp.
4. Deduplicate by `(captureId, groupIndex)`. Require every index from zero
through `groups - 1`, with no gaps. Entry count must equal `groups`, summed
`calls` must equal `completedQueries`, and summed `responseValueBytes` must
equal the summary total. A missing page is an incomplete measurement.
`calls` must equal `completedQueries`, and summed `rows`, `sampledResults`,
and `sampledValueBytes` must equal their summary totals. A missing page is an
incomplete measurement. Keep overflow and unattributed groups visible.
Tagged raw-driver calls, streams, unawaited raw-driver
dispatch, and protocol overhead are not measured. Relational queries use a driver-level caller fallback.
Queries still in flight at the deadline are reported separately. Sizes describe
driver values before column decoding, not billed network bytes. Complex parsed
values use JSON size estimates, counted in `estimatedValues`. Reconcile them with
provider metering and socket measurements. Serialization adds temporary CPU and memory work;
measure the final transfer rate after the window expires.
Queries still in flight at the deadline are reported separately. The census does
not include every PostgreSQL statement or internal transaction command, so its
calls are not interchangeable with database-wide commit counters. It does not
measure SQL compilation, driver parsing CPU, or Drizzle row-mapping CPU. Use the
ranked owners to choose a narrow follow-up profile or synchronous-stage
measurement; do not attribute CPU by subtracting process CPU usage across an
asynchronous query. Remove the capture setting after collecting its final
summary; the absolute deadline already prevents further measurement.
### Push the schema
doc/DEVELOPING.md
+10
View File
@@ -2621,6 +2621,16 @@ connections, including provider-native tools. Company permissions, approval
gates, and workspace isolation still apply. Explicit restrictive modes remain
restrictive; omitted settings use full auto.
Assigned app tools can use `paperclip_search_assigned_tools` and
`paperclip_call_assigned_tool` instead of direct tool definitions. Search with
`{"query":"github"}` and call an exact returned tool name. Omitted, empty or
whitespace-only `schemaTool` values perform normal discovery, including when
`schemaOffset` is zero. To read a large schema, set `schemaTool` to the exact
`inputSchemaRef` from discovery and follow `nextSchemaOffset`. A positive
`schemaOffset` requires a nonblank reference. Unknown or revoked references
return guidance to search again; discovery still checks current grants and work
modes against the session's pinned catalog.
Test questions, saved plan revisions, approval before task handoff, status
lookups, and `/new` preserving chat history. Hiring additionally requires the
operator-controlled [runner API tools](runner-api-tools.md) rollout; enabling
doc/PRODUCT.md
+13 -2
View File
@@ -97,6 +97,11 @@ When an agent creates a subtask or follow-up task, its source task's feed shows
a linked card at creation time with the current title, task number, and status.
The Tasks tab keeps the full relationship list.
Tasks waiting between automatic checks show **Monitoring** with a blue clock.
Task lists, board columns, filters, search, recent navigation and task properties
use the same read-only display state. The task retains its execution lifecycle
status, and the monitor banner shows when the next check is due.
Automatic monitor checks pause when a task is blocked. A person can use Check now
to request one check of a retained monitor through normal execution admission.
Dependencies, recovery, pause, budgets and ownership still apply. Provider-usage
@@ -141,9 +146,15 @@ choice stays selected while task data refreshes.
The task Filters menu also offers **Routine tasks**. It selects tasks with
`originKind=routine_execution`, including manually triggered routine runs, even
when other automatic tasks are hidden. The routines list's **View all tasks**
link opens `/issues?view=all&originKind=routine_execution` with that checkbox
link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox
selected. The URL retains the selection across reloads and task navigation.
Search and pagination apply the routine origin filter on the server.
Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides
saved collection sorting, and the routines link opens newest-created tasks first.
Shared sort links work across Coordinator, Streamlined UI, and Classic UI, including
sort fields without a visible column in the selected interface. Search relevance
and board-column membership keep their existing server order.
Search and pagination apply the routine origin filter on the server. Creation-time
sorting also runs before pagination, with task ID as a stable tie-breaker.
Tasks and project lists hide agent-created and system-created tasks by default.
The Filters menu's **Show agent-created tasks** checkbox includes routine runs,
doc/SPEC-implementation.md
+10 -2
View File
@@ -423,6 +423,7 @@ retain their titles and default to no generation request.
- `title` text not null
- `description` text null
- `status` enum: `backlog | todo | in_progress | in_review | done | blocked | cancelled`
- Task reads additionally expose read-only `displayStatus`: the lifecycle status, or `monitoring` for an agent-owned scheduled ordinary monitor wait with no active execution, locks, review participant or active recovery action. Board collection/search `displayStatus` filters and counts use the same projection before pagination. This field is not persisted and cannot authorize a status transition; existing `status` filters retain their lifecycle semantics.
- `priority` enum: `critical | high | medium | low`
- `review_policy` nullable enum: `anyone | not_creator | human_only`; null is equivalent to `anyone`
- `assignee_agent_id` uuid fk `agents.id` null
@@ -2712,9 +2713,16 @@ its upstream authors; operators can pause updates to review changes manually.
The task Filters menu also offers **Routine tasks**. It selects tasks with
`originKind=routine_execution`, including manually triggered routine runs, even
when other automatic tasks are hidden. The routines list's **View all tasks**
link opens `/issues?view=all&originKind=routine_execution` with that checkbox
link opens `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` with that checkbox
selected. The URL retains the selection across reloads and task navigation.
Search and pagination apply the routine origin filter on the server.
Sort controls save `sortField` and `sortDir` in the URL. URL sorting overrides
saved collection sorting, and the routines link opens newest-created tasks first.
Shared sort links retain their order across task interfaces. Workflow sorting
requests the existing authorized dependency-edge projection. Search and board
queries keep their existing server order. Blocked-attention lists reject
`sortField=created` with `422` because they use their own activity order.
Search and pagination apply the routine origin filter on the server. Creation-time
sorting also runs before pagination, with task ID as a stable tie-breaker.
Tasks and project collections default to hiding agent/system-created tasks and
provide a saved **Show agent-created tasks** filter. Classify creation using
doc/evals.md
+3 -2
View File
@@ -57,8 +57,9 @@ OpenCode Product E2E cells. They
verify saving an unstarted task and delegating a saved announcement back to chat,
using ordinary production instructions, real browser messages and independent
state/provenance checks. A separately selected native OpenCode/DeepSeek1017
backlog cell measures TestOps integration acceptance with the same oracle and
original bounds. Its source registration is not an AWS or TestOps pass. The
backlog cell supplied historical TestOps integration acceptance with the same
oracle and original bounds. Its former smoke plan and the three-case completion
pilot are retired; their cases and measurements remain in history. The
[full-catalog TestOps lane](../tests/runner-e2e/CHAT-FEATURES.md#full-catalog-in-testops)
maps all 244 YAML scenarios into a separate candidate-scoped plan. Its app and
fixtures use a preserved immutable catalog revision; trusted workflow helpers run
doc/execution-semantics.md
+34
View File
@@ -112,6 +112,22 @@ The issue is actively owned work.
For agent-owned issues, `in_progress` should not be allowed to become a silent dead state.
### Monitoring presentation
An agent-owned `in_progress` task displays **Monitoring** with a blue clock while
it has a persisted ordinary monitor deadline and no active run, execution or
checkout lock, review participant, or active recovery action. The persisted
status remains `in_progress`. Monitor admission, checkout, finalization, recovery,
and agent lifecycle filters keep their existing contracts. Review, blocked,
terminal, and provider-quota recovery states keep their own presentation.
Task reads expose optional, read-only `displayStatus`. Board task collections and
search can filter by `displayStatus`; filtering happens before pagination and
counts. Existing `status` filters retain their lifecycle meaning. Monitoring is
not a manually selectable task status or a board drag target. An overdue check
retains its visible deadline warning; the label does not attest that a provider
is running or that a check succeeded.
### `blocked`
The issue cannot proceed until something external changes.
@@ -764,6 +780,15 @@ conflict detection. Legacy adapter snapshots remain inline. Snapshot-only worker
do not load the adapter process-launch helpers. The directory merge loads those
helpers when it resolves its instance-scoped lock root.
Portable remote workspace capture stages only self-contained Git metadata. It
streams working files into the archive without a second working-tree copy and
does not repack an already dissociated mirror. Each streamed file must match its
recorded hash, size and mode and retain its filesystem identity while read.
Capture rechecks the complete source inventory and Git state before returning a
receipt. A changed source or cancelled capture removes its incomplete archive;
an existing archive remains untouched. Temporary space must still hold the
normalized Git metadata and the complete archive together.
### Remote Codex model compatibility
Fresh remote Codex Runner runs check the selected image CLI before saving the
@@ -862,6 +887,15 @@ The task thread exposes the guarded Retry action for failed or timed-out convers
A continuation cancelled by the queued-run gate while waiting for review did not start a provider. Its matching dispatch-gate receipt and unclaimed execution fields keep this deliberate wait out of unknown-action reconciliation. Pending review and decision gates still apply. If an older server already created a recovery hold for this exact state, Retry or a new user message can request one fresh turn after the server verifies there is no provider, coordinator, or unfinished cleanup. Saved user input is reconsidered through the same admission path; the cancelled turn is not replayed.
A successful connected run can still have newer admitted input waiting behind it.
Task Retry resumes that input's existing durable chat receipt through the inbound
owner, instead of replaying the successful run as a failed attempt. The original
actor, source, result, and publication history remain intact. Duplicate clicks
follow one canonical receipt. Current board permission, task ownership, source
authorization, and prior execution cleanup are rechecked before normal admission;
pause, budget, approval, and dependency gates remain effective. Completion or
newer execution supersedes an unadmitted retry.
A valid recovery action must name:
- the source issue and company
doc/idle-sleep-safety.md
+11
View File
@@ -133,6 +133,17 @@ fingerprint and RPC company scope for later read-only verification.
Retained allocations, unfinished runs/accounting/retries, export intents and
live setup credentials continue to block sleep.
For a reused remote allocation, a completed predecessor can resolve termination
through the final owner's destruction receipt only when the full host-recorded
lease handoff chain matches. Company, agent, task, workspace, environment, provider
and allocation identities must agree. A stopped successor, broken handoff or
unrelated owner grants no cleanup authority. This resolution does not rewrite
historical leases, execute remotely or claim that uncollected files were saved.
The normal agent-file recovery worker retains the no-save outcome and completes
cleanup after confirmed destruction. A completed cleanup receipt with a stale
retry deadline is reconciled once through the existing deadline worker. Clearing
that deadline compares the receipt so a concurrent cleanup obligation is kept.
Pending human-only connection setup can wait asleep when no active enabled
connection of the requested purpose exists. Available access reblocks sleep so
the existing reconciliation path can run; answering a setup card continues to
doc/observability.md
+44
View File
@@ -1900,6 +1900,49 @@ name to `sandbox.daytona.<name>`. The host maps any other value to
daytona provider emits these spans today, so the segment is the literal
`daytona`.
### Daytona lease acquisition and resume
The worker invocation carries nested, monotonic timing receipts for lease work.
The host also admits the following fixed `lease.*` names and prefixes them with
`sandbox.daytona.`. These use the existing opt-in endpoint and host-owned parent
context; they do not create an exporter or first-party Telemetry events.
```text
execution.plugin_dispatch
└── plugin.worker
├── sandbox.daytona.lease.wait_for_teardown (only when teardown blocks resume)
├── sandbox.daytona.lease.lookup
├── sandbox.daytona.lease.refresh (missing-container confirmation)
├── sandbox.daytona.lease.channel_cleanup (before restart)
├── sandbox.daytona.lease.start (or lease.recover)
└── sandbox.daytona.lease.verify
├── sandbox.daytona.lease.workspace
├── sandbox.daytona.lease.sentinel
└── sandbox.daytona.lease.shell
```
`lease.lookup` includes cached-handle lookup and any required provider refresh.
`lease.start` and `lease.recover` measure the actual SDK operation, including its
internal readiness wait. They also cover starts triggered by execution or sync.
They do not separate SDK-internal requests or provider-side queue time.
Their `paperclip.sandbox.startup.state` attribute records the state before the
call: `started`, `stopped`, `archived`, `error`, or `other`. Both the plugin and
host restrict this value; no provider error text or resource identity is sent.
Fresh acquisition emits `lease.create`, `lease.workspace`, `lease.shell`,
`lease.expiry`, and `lease.sentinel`. `lease.create` covers the existing creation
helper, including qualification-only snapshot resolution and creation-failure
cleanup. Setup-failure deletion is a separate `lease.cleanup` sibling, so it
does not inherit a failed workspace or verification phase. Existing deadlines,
ownership, cleanup and retry behavior are unchanged. Failed awaited steps close
with error status; the receipt contains no exception message.
Children nest beneath the current `plugin.worker` receipt and verification
children nest beneath `lease.verify`. Never add parent and child durations.
Receipt budgets and clock validation still apply. Older workers without
`tracer.withSpan` keep the original behavior without these detail spans. Tracing
failures preserve the operation's result or error and never repeat it.
The `sandbox.daytona.*` spans use this closed attribute allowlist. The host
drops every other key, so a command, an argument, a path, an id, a standard
output, or a standard error never rides a provider span. The host records only
@@ -1908,6 +1951,7 @@ the attributes that the producer sends for one span.
| Attribute | Type | Optional | Meaning |
| ------------------------------------------------ | ------ | -------- | --------------------------------------------------------------------------------------------------------- |
| `paperclip.sandbox.startup.provider` | string | no | The normalized provider family. |
| `paperclip.sandbox.startup.state` | string | yes | State before start/recovery: `started`, `stopped`, `archived`, `error`, or `other`. |
| `paperclip.sandbox.startup.outcome` | string | yes | The step outcome (`ok`, `skipped`, or `failed`). |
| `paperclip.sandbox.startup.pack.wall_ms` | number | yes | The host-local wall time of the pack step. It rides the `sandbox.daytona.pack` span. |
| `paperclip.sandbox.startup.transfer.wall_ms` | number | yes | The wall time of the transfer step. It rides the `sandbox.daytona.transfer` span. |
doc/performance/native-backup-publication.md
+53
View File
@@ -0,0 +1,53 @@
# Native backup publication benchmark
Retiring a native harness backup can involve many immutable directories and
small files. Cleanup repairs directory permissions before removing the retired
tree. Publication waits for cleanup, preserves the previous backup when lease
stamping fails, and emits its completion notification after cleanup succeeds.
Cleanup uses asynchronous filesystem operations so unrelated requests can
continue while it runs. Directory preparation is serial and does not follow
symlinks into live trees.
Run the focused correctness tests from an installed checkout:
```sh
pnpm --filter @paperclipai/server exec vitest run src/services/native-runtime/native-harness-backup-cleanup.test.ts src/services/native-runtime/native-harness-backup-publication.test.ts
```
Run the performance comparison from the repository root with the Node version
used by the target server. Use a quiet machine and an explicit baseline commit:
```sh
node server/scripts/benchmark-native-backup-publication.mjs \
--baseline-ref 774ddc72713d6aa7cb516691c71f1fc23bcfa5ff \
--output /tmp/native-backup-publication.json
```
The script compiles the real publication service and its dependencies for both
the baseline commit and the working tree. Each run uses a separate server
process. Fixture preparation occurs before measurement. The default fixture has
100 immutable directories with 100 small files in each of the `current` and
`previous` backups: 20,000 files to remove. Both backups also have a symlink to a
read-only live directory outside the retired trees. The new pending backup has
a distinct manifest. Every completed run checks the new manifest, complete
removal of retired and pending trees, and unchanged live contents and mode.
A separate process sends requests to a small loopback HTTP endpoint every five
milliseconds. Arrivals use a fixed clock and include overdue requests. The
report includes request completion and error counts, generator lateness, p95
and maximum foreground latency, requests served during publication, publication
wall time, process CPU time, and maximum event-loop delay. Three pairs alternate
baseline and treatment order. Change the fixture with `--pairs`, `--directories`,
`--files-per-directory`, or `--interval-ms`.
The result records Node/platform, baseline SHA, treatment HEAD and uncommitted
service diff. Compiled modules and source maps stay in the printed evidence
directory. Successful fixtures are removed; failed fixtures remain for diagnosis.
This is a service benchmark. The HTTP endpoint is a responsiveness probe, not
the assembled Paperclip API. The lease-stamp callback resolves without database
work. It does not prove a hosted latency or total-CPU improvement. Process CPU
during publication includes foreground requests that the synchronous baseline
cannot serve until cleanup completes. Async cleanup can consume more CPU while
keeping the event loop responsive. Qualify the affected run/checkpoint/recovery
journey and compare matched hosted windows before claiming deployed acceptance.
doc/plans/2026-10-31-zero-downtime-web-deploys.md
+197 -6
View File
@@ -1,10 +1,201 @@
# Zero-downtime web deployments
Date: 2026-10-31. Status (2026-11-01 15:39 UTC): existing-volume staging enrollment
and manual sleep/wake preserve saved files and read/save behavior. The full local
active-agent journey now passes on `cc23019691`; hosted browser continuity and
the ten-deploy staging campaign remain unqualified. Natural database autosuspension
passes after abandoned test-resource cleanup. No production canary yet.
Current state (2026-11-02 18:31 UTC): cleanup recovery PR
[#18633](https://github.com/paperclipai/paperclip/pull/18633) merged as
`60a99307b3bb1062078d35af6677e3f377d8bab8`. Its final head passed all PR CI,
full local typecheck and build, and seven focused cleanup regressions. Greptile
reviewed that head at 5/5; both review threads are resolved. The duplicate full
local test run was stopped after six chat shards passed, while general suites
were still running. Its incomplete local result is preserved separately from
successful full PR CI.
Canonical Linux release verification failed on the unchanged two-second busy-row
startup test on several consecutive revisions, including the cleanup merge.
The held-row diagnostic completed in 420 ms on macOS without a database lock
wait. Inspection found that `currentNativeControllerIdentity` always called the
server-info snapshot, which synchronously probes Git three times even when the
OS process birth time is available. Controlled one-second delays in real Git
commands reproduced the exact startup assertion failure with real PostgreSQL.
The follow-up separates the constant fallback process birth time from diagnostic
Git metadata. New regressions cover both available and unavailable OS birth time.
The two-second startup limit and release gates remain unchanged. Linux release
verification must pass before the new source can enter hosted qualification.
The follow-up is [#18649](https://github.com/paperclipai/paperclip/pull/18649).
Its first head received two 5/5 reviews and passed every CI test shard except
an existing voice-retention precision test. That test advances its fake clock
automatically while asserting behavior across a one-millisecond boundary. A
controlled 60 ms database-entry delay reproduced the exact failure. Holding the
clock fixed makes the same assertion pass with that delay; the final correction
only disables automatic clock advancement in that one precision test. The
original CI failure and both controlled results are retained. No production
voice code or assertion was weakened.
The earlier ten-roll result remains evidence for its recorded images. Neither
the cleanup fix nor the controller-identity follow-up has reached the staging
fixture. Final acceptance still requires a qualified compatible image pair,
fresh staging replacements, natural idle and browser sleep/wake, followed by the
isolated production canary. Cloud source is `b44304603d`; its private packages
require comparison against the new Core SDK and shared archive. Earlier candidate
Core `42c64d4ec4` differs in SDK; `b9e94b85a6` differs in shared types. Neither is
a valid rollback pair for the cleanup merge. No release gate was bypassed.
The production browser was signed in as Dotta B on the ordinary new-organization
form. No organization was submitted and no production config changed. The only
hosted target changed so far is the pinned staging qualification fixture.
Disposable local PostgreSQL and Moto S3 fixtures passed an independent object
read/write preflight. Docker subsequently stopped; local container qualification
requires restoring that dependency. Hosted acceptance uses real object storage.
Earlier retained evidence and failure history follows.
Date: 2026-10-31. Status (2026-11-02 17:23 UTC): Core #18554 is merged as
`1a6017d6460267ad89d7e1784e2b521c28d8e9ac`. The ten-roll and native-agent
proof used Cloud `461c4aea28`. Separately reviewed Cloud #1091/#1092 subsequently
reached staging as `89dd0cfec2`; their changes concern workspace cleanup and
Agent Computer resource fencing. Railway, fleet and deployment workflow paths
are unchanged. Production remains on its original controller revision; the
next manual controller deployment must use current reviewed master.
Both immutable Cloud distributions built from that fixed source passed packaged
Linux rehearsals through the Railway wrapper: 260 reads, 53 saves, zero failures,
legacy-volume migration, an unready replacement, and removal of both predecessor
homes. Each fixture cleaned its containers. These rehearsals do not exercise an
active agent or an actual browser. The two distributions have distinct image
digests but identical application source; their compatible rollback qualifies
artifact replacement, not compatibility between different source revisions.
The final Core PR has two 5/5 reviews and all required checks passed. One legacy
browser CI shard failed because its deferred-wake fixture observed no queued wake;
the unchanged case passed locally on isolated PostgreSQL and its single CI retry
passed. The original failure remains in the PR evidence. The full local live-agent
journey passed on `cc23019691` before the final non-suspended dispatch refinement;
that refinement passed its three focused Linux regressions, server typecheck,
build and final CI. Do not describe the earlier live run as a rerun of the merge.
The fixed canonical package baseline passed on the pinned migrated staging fixture
using the allowed one-time maintenance path. An actual Chromium tab running the
older UI reconnected, retained an unsaved title edit, and saved it through the new
server; an independent API read confirmed the title. That old tab opened the
Agents page, and all four previously captured asset hashes still match after the
old deployment was removed. Baseline maintenance interruptions are preserved and
are not counted as zero-downtime qualification.
The first ten-roll attempt was cancelled at step one, before Railway replacement.
The controller's pre-existing hot-upgrade allowlist named an older test stack,
so it drained rather than negotiated preservation of the new fixture's active run.
Continuous read/save and browser probes remained healthy, but this is NOT active
execution continuity evidence. The original command completed normally once; its
downloaded output bytes and native terminal were independently verified. The
owned fixture has been added to the existing allowlist without changing fleet
defaults. The normal staging controller workflow is applying that correction.
The rollout harness now checks both canary policy and positive tenant readiness
before submitting a campaign. The cancelled attempt remains failed evidence,
with no automatic campaign retry.
The configuration deployment then failed before Railway submission when Sentry
release registration failed or timed out. Its source checks passed; the original
logs are retained. One unchanged whole-workflow retry preserves attempt-bound
preflight requirements. That retry failed at the same pre-Railway Sentry request.
A read-only Sentry lookup confirms the release exists (HTTP 200). The canary-only
configuration was then applied by redeploying the existing successful controller
deployment: the source and replacement image digests are identical, and health
attests unchanged Cloud `461c4aea28`. Both failed workflow attempts are retained;
no Sentry gate was disabled and no new application release was built in this
configuration recovery. The new active-command attempt and gated sequence are
starting with the corrected controller. The first preflight stopped before task
admission on a missing optional plan's 404. `useIssuePlanDocument` explicitly
maps that response to null; an independent read confirmed no plan on the owned
completed task. The raw response remains recorded and the browser gate permits
only that verified path. The configuration redeploy's actual browser reconnect
was 1.22 seconds, with no probe read/save failure.
All ten Railway replacements are independently qualified. Final continuous proof
passed 12,521 reads and 1,566 saves, with every acknowledged save present exactly
once and all 34 file hashes matching. There were zero HTTP failures. Actual
browser reconnects across the ten deployments ranged from 1.16 to 1.29 seconds.
The old UI assets and original agent artifact remain readable after deployment
ten and provider cleanup. Both traffic observers have stopped cleanly. The first
replaced the webserver while the original eight-minute remote command was
running. Its one execution ID started before replacement and completed afterward
with exit zero. The second replaced the server after completion; a later follow-up
retained the same native/provider session and runner, with accepted native
terminals and finalized accounting for both runs. The 91-byte artifact was
downloaded and independently checked for exactly one start and completion.
Actual browser reconnects were 1.28, 1.16 and 1.25 seconds; continuous reads and
saves have no failures so far. Previously unopened Projects and Skills pages rendered in the
retained old tab. A title draft survived the third cutover and was saved through
the UI, then independently read back from the replacement.
The harness stopped after deployment three on another owned task's expected
missing optional-plan 404. The UI explicitly treats this as null. Both exact task
paths were independently checked, and the provider deployment, package eligibility,
serving identity, removal of the predecessor, browser timing and continuous probe
were revalidated. The failed original receipt remains unchanged alongside an
explicit classification review. The remaining seven replacements continue without
redeploying or retrying deployment three. An announcement also obscured one
sidebar click; its ordinary Dismiss control resolved that retained interaction
timeout. A separate large run-log diagnostic timed out; a narrower public-events
read succeeded and proves the original command's single execution identity.
A later bounded 64-KiB log read succeeded from shared storage after both the
originating webserver and remote sandbox were removed.
The native recovery event records SIGTERM without a retry run, then adoption of
the same runner at controller generation 2. Each of the two runs has exactly one
accepted result and terminal event. The independent accounting inspector has no
findings or pending work; the task totals 52.941 cents across two runs. Including
the earlier no-replacement attempt, staging has three estimated cost events totaling
113.84284 cents. The temporary agent is paused, its secret removed and the prior
company budget restored; all three provider resources are independently confirmed absent. The first delete
was refused while the sandbox stopped; that failure remains alongside the
successful cleanup after a fresh stopped-state observation. Both owned local
PostgreSQL/object-storage containers are removed; test evidence is retained.
Deployment nine also completed, but the local test reader observed the browser
snapshot while its writer had truncated the file. The campaign response was
already parsed and successful; the only non-atomic JSON read in that failing
checkpoint is the local browser report. A concurrent-write regression reproduced
the problem. The reader now retries only local JSON syntax errors for at most
100 ms; persistent corruption still fails, and HTTP/acceptance failures are never
retried. Deployment nine was independently revalidated without redeployment.
Both raw failed sequence receipts and their explicit classification reviews remain;
the ten provider deployments themselves have not been retried. The final staging
driver stopped at the harness failure and is being resumed from the retained
sequence. The normal UI's 256,000-byte run-log read also passes after replacement.
The first repair check stopped before mutation because an unrelated staging
campaign was active. That campaign has finished, and the pinned canary remains
active and unlocked on the qualified final image. Both one-target configuration repair campaigns passed: backup policy was
restored, the temporary marker removed, and durable files/shared assets remained
enabled on the same volume-free image. The current-image eight-minute natural-idle check failed with all browser pages and
traffic probes closed. Bounded SQL diagnosis found three completed native runs
still in agent-file collection recovery. Normal archive of both completed test
workspaces independently passed close-readiness and preserved saved workspace
files. It produced provider-confirmed destruction receipts for the latest leases.
Two copies then completed no-save cleanup, but retained stale retry deadlines;
the earlier run of a reused sandbox still lacks direct termination evidence even
though its validated successor allocation is destroyed. These are source recovery
gaps, not an idle-proof pass. No historical rows were manually rewritten. The
failed idle and cancelled browser lifecycle receipts remain unchanged. Fix
ownership belongs to remote termination proof resolution and agent-directory
cleanup scheduling; no polling cadence or sleep-safety check should be relaxed.
The isolated source branch is now based on current master `97c4dc969e`. The
recovery fix resolves the destroyed allocation through the existing validated
handoff-chain contract at read time; it does not stamp historical leases or infer
stop from task status. Agent-file cleanup also reconciles stale retry deadlines
with a receipt comparison. Both owning service suites passed 169 tests, and the
server typecheck passed. The first new deadline test caught a public guard that
skipped settlement; that failed attempt is retained and the corrected five-case
selection passed. The final seven-case selection passed, including a concurrent receipt change and
cross-company rejection. Repository-wide checks and PR review are pending.
This source has not been deployed and is not covered by the prior ten-roll proof.
Next: review and ship the cleanup fix, complete current-image idle/sleep/wake, then the
isolated production forward/rollback canary. No production deployment has occurred.
Dotta completed the authorized GitHub sign-in; the production browser is back on
the ordinary organization list. The production canary still waits for staging. Its new organization form is
prepared but unsubmitted. Production remains on its original controller revision,
with no campaigns, the single-volume default and continuous deployment disabled. Earlier
staging sleep/wake and natural database-idle proofs remain valid for their image.
Core #18543 merged as `7751fc3107` after 55 successful CI checks, Greptile 5/5
and no unresolved threads. It supplies runtime admission, immutable-cache
@@ -202,7 +393,7 @@ Measure one-time setup separately from repeated attempts. Keep failed evidence.
| Item | State | Next action |
| --- | --- | --- |
| Prior prerequisites | #18403 and all five storage workstreams merged | Retain the ephemeral filesystem contract |
| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 reviewed at 5/5 with CI pending; local packaged migration, browser overlap, cold restoration and durable command proofs recorded below | Publish and qualify the exact fixed release after CI/review merge |
| Core implementation | #18518, #18526, #18529, #18530, #18531, #18533, #18543 and #18552 merged; #18554 merged with all required checks and 5/5 reviews; both exact fixed images passed packaged migration and deletion; active hosted output verified | Finish hosted continuation and accounting reconciliation |
| Cloud implementation | #1096, #1098, #1099, #1100, #1101 and #1102 merged; #1102 serving on staging | Verify repeated Railway overlap behavior after Core recovery qualification |
| Staging | Original enrollment complete; migrated data and signing key preserved; volume detached and retained | Sleep/wake and natural database idle pass; verify repair and ten app-only replacements with browser continuity |
| Production | No canary mutation performed | Proceed only after the staging and active-agent acceptance gates pass |
maps/agent-access-map/chat-channels.md
+7
View File
@@ -1028,6 +1028,13 @@ receipt preserves the original skipped receipt and older failed native result.
Current access and normal native stop/cleanup guards still govern execution;
board wake metadata alone cannot manufacture chat authority or replay old work.
After a successful connected run, board Retry can resume a newer deferred inbound
receipt through the original chat owner. It rechecks the board's current task
permission and the saved source's current authority, and waits for prior cleanup.
It grants no permission to replay the successful result or manufacture a chat
request. Concurrent clicks share the canonical retry receipt; completed tasks
and superseding execution retire unadmitted retry intent.
Deferred chat Retry excludes independent board queues from its pinned batch.
An explicit board Retry can restore a historical mixed batch only with exact
never-started evidence, current source authority, unchanged board comments and
maps/agent-access-map/runs-adapters.md
+3 -1
View File
@@ -214,7 +214,9 @@ use the server-owned authenticated relay. The provider sees bounded
then uses pinned aliases. Search describes its 1–20 page limit and default of 5
in the tool and parameter descriptions. It intersects fresh grants and work modes; calls
retain the run's original gateway policy, validation, approvals, revocation and
audit path. Prompt arguments cannot select gateway authority. Assigned gateway
audit path. Blank schema references perform normal discovery; positive schema
offsets require an exact reference, and unavailable references explain how to
search again. Prompt arguments cannot select gateway authority. Assigned gateway
launch credentials stay on the controller. Former direct-catalog OpenCode
checkpoints refresh; other provider fingerprints stay unchanged. The
[projection controls](../../server/src/services/native-runtime/assigned-mcp-tools.test.ts)
maps/agent-access-map/tasks.md
+11 -2
View File
@@ -74,7 +74,7 @@ API: `present` — GET /api/companies/{companyId}/issues; optional view=compact.
Agent access: `partial` — REST/call_api full query; CLI issue list exposes status/agent/project and local --match on one fetched page; native search_tasks query up to 500 characters, statuses, limit 1..200 (default 50), and a nextCursor in ascending task-ID order. Responses expose effective limit, returnedCount and hasMore. Exhaust nextCursor with the same filters before treating an inventory as complete. Each page checks current access; concurrent task changes require a fresh scan. Project/label/workspace knobs remain REST-only.
Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|id, sortDir asc|desc, afterId. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results.
Policy: Company/read privacy filtering and inaccessible edge redaction; task-bridge company list denied. Default 500, maximum 1000, offset>=0, sortField updated|created|id, sortDir asc|desc, afterId. Blocked attention lists reject created and id ordering with 422. Supports status/assignee/participant/project/workspace/parent/descendant/label/origin/text/date and routine/plugin/blocker query groups. includeAgentCreatedTasks=false filters independent roots before search/pagination; omission retains agent/API behavior. includeParticipatedTasks and user me filters require board identity. Compact ETag is actor-private; 429 is not empty results.
Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [issues.ts](../../server/src/routes/issues.ts), [issues.ts](../../server/src/services/issues.ts), [paperclip-runner-tool-authority.ts](../../server/src/services/native-runtime/paperclip-runner-tool-authority.ts), [issue.ts](../../cli/src/commands/client/issue.ts), [issues-list-query-parsing.test.ts](../../server/src/__tests__/issues-list-query-parsing.test.ts), [issues-creation-visibility.test.ts](../../server/src/__tests__/issues-creation-visibility.test.ts), [issue-list-assignee-filter-routes.test.ts](../../server/src/__tests__/issue-list-assignee-filter-routes.test.ts), [private-json-etag.test.ts](../../server/src/__tests__/private-json-etag.test.ts). No live execution claimed.
@@ -120,7 +120,10 @@ Agent access: `not-applicable` — no agent layout API; permitted underlying tas
Policy: IssuesList loads/saves browser preferences by company/collection, with separate agent-created visibility. The Tasks page also owns a routine-only
checkbox and `originKind=routine_execution` URL state; it uses the existing
company-scoped task-list origin filter before pagination. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred.
company-scoped task-list origin filter before pagination. Sort controls write
`sortField` and `sortDir` to the URL; explicit URL sorting overrides browser
preferences. `created` sorting runs on the server before pagination, with ID as
a stable tie-breaker. The routines task link requests `created` descending. Contextual ancestors/descendants do not become unrelated independent results. No shared server view preference inferred.
Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7a`; [IssuesList.tsx](../../ui/src/components/IssuesList.tsx), [Issues.tsx](../../ui/src/pages/Issues.tsx), [Issues.test.tsx](../../ui/src/pages/Issues.test.tsx). No live execution claimed.
@@ -522,3 +525,9 @@ Evidence: `source-only` — reviewed at `3bbc2104b65caf9f7763938a01260a5673e12f7
Verification: `not-run` — Compare eligible owner/admin ordinary denied with explicit read/audit+notice; ordinary operator/agent/foreign denied, grants unchanged.
Gaps: Local implicit/instance admin bypass cannot establish ordinary two-user private isolation.
Task reads expose optional read-only `displayStatus`. `monitoring` describes a
scheduled ordinary monitor wait; it is not accepted as a writable task `status`.
Existing lifecycle `status` filters retain their meaning. Board collection and
search readers can use the separate `displayStatus` filter before pagination.
Agents keep using the existing monitor and semantic completion tools.
maps/agent-access-map/workspaces.md
+3
View File
@@ -74,6 +74,9 @@ without a controller checkout fallback. Current authorization and responsible-us
checks still apply;
read-only cross-project referenced folders and explicitly concurrent shared
writers are rejected. Managed mutable same-project repositories are supported.
Portable capture streams validated working files while staging self-contained
Git metadata. The smaller temporary-space requirement grants no new agent access
or execution authority. Changed sources still fail before a save receipt.
An inherited isolated workspace or operator branch can be reserved through the
task's current persisted binding. The original source task remains provenance.
Reservation checks task/run, binding, trust, and retained source visibility before
maps/feature-map/chat-channels.md
+8
View File
@@ -278,6 +278,14 @@ After successful admission, reload and verify that the prior skipped receipt doe
not strand a later real provider reply or tree resume. Provider fixtures support
this behavior; live fixed-source runtime and destination proof remain separate.
Also complete an older review successfully while a newer authorized PR event
remains deferred. Use Retry and verify that the existing saved-input receipt
starts exactly one new turn, repeated clicks share that turn, and the older
accepted result remains unchanged. Revoke the connection or change the source
binding before dispatch and verify no run starts. Keep prior cleanup pending,
then settle it and verify the same receipt resumes once. These database-backed
provider fixtures do not establish live GitHub publication at the fixed revision.
Also defer Retry behind cleanup, queue an independent board message, then
complete cleanup. Verify one run receives the exact GitHub batch and the board
message enters its own turn. For an older mixed batch rejected before provider
maps/feature-map/inbox-search.md
+5 -1
View File
@@ -91,7 +91,11 @@ Cold-open verification: load Mine with the task-index response delayed while
approvals/runs resolve. Only the stable list placeholder should appear until the
initial sources settle; a failed-run row must not jump when tasks arrive. Release
visible-row details separately and compare row/title bounds. Mine reads its own
100-row index, with parent/run context by ID; sidebar badges use the count endpoint.
100-row index. Missing parents use the same index projection for their captions;
parents already in the index are reused. Failed-run tasks still load rich row
context by ID, including when they are also parents or already indexed, so resolved
interruptions stay hidden. The initial reveal waits for both context sources.
Sidebar badges use the count endpoint.
Coordinator desktop and mobile navigation share `/attention/summary` for the
outstanding count. Needs You owns its complete feed; Tasks → Needs attention
owns a compact complete task-ID index. Navigation reuses either page response. Coordinator keeps the optimized Inbox even when
maps/feature-map/routines.md
+1 -1
View File
@@ -30,7 +30,7 @@ Open `/routines`, create one, and inspect `/routines/:routineId` and its edit se
Use the routine's trigger controls and Runs/Recent Runs/Audit links; lists also
offer Run now. **View all tasks** opens the Tasks list with **Routine tasks**
selected, using `/issues?view=all&originKind=routine_execution`. Uncheck the
selected, using `/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc`. Uncheck the
filter to return to other tasks. **History** records configuration versions, not executions.
## Driving it with Playwright and the Paperclip CLI
maps/feature-map/runs-adapters.md
+4
View File
@@ -273,6 +273,10 @@ For native OpenCode with assigned connections, inspect the provider tool surface
on both local and sandbox execution. It should contain bounded assigned-tool
search/call schemas. Confirm the search description states the 1–20 page limit
and default of 5. Follow `nextOffset` to inspect later pages.
Repeat discovery with empty and whitespace-only `schemaTool` values, including
`schemaOffset: 0`, and confirm it still returns searchable tools. An unknown
schema reference or a positive schema offset without a reference must explain
how to correct the call.
Discover an allowed tool, call its returned alias, revoke
its grant, and confirm discovery and execution enforce that revocation. Repeat
in planning/ask mode with a write tool and confirm denial. The controller must
maps/feature-map/tasks.md
+9 -1
View File
@@ -56,9 +56,17 @@ Use New task from the global launcher, task list, or project task view.
### `task-list`
Scheduled ordinary monitor waits display **Monitoring** with a blue clock in
current task indicators, groups, board columns, and filters. Compare Monitoring
with In Progress, then start the check and verify the status returns to its
execution presentation. Retained monitors must not override blocked/review or
terminal states. Inspect due/overdue copy separately; no runner is implied.
Open `/issues` or a project’s Issues view; legacy `/tasks` links redirect.
Filters → **Routine tasks** selects tasks created by routine runs. The
`/issues?view=all&originKind=routine_execution` link opens that selection.
`/issues?view=all&originKind=routine_execution&sortField=created&sortDir=desc` link opens that selection with newest-created tasks first. Sort controls update
`sortField` and `sortDir` in the URL. Reloads and task return links keep that order.
Shared sort links also retain their order when opened in another task interface.
### `task-detail`
maps/feature-map/workspaces.md
+8
View File
@@ -83,6 +83,14 @@ or original Git remote. Read-only cross-project referenced folders must fail
admission. Separate workspaces may run concurrently; shared concurrent writers
must be rejected.
Repeat remote capture with enough free space for the archive and normalized Git
metadata, but not another working-tree copy. Verify saved files and Git state
after restore. A concurrent file change must fail capture without publishing a
receipt or retaining a partial archive. The
[portable archive suite](../../server/src/services/remote-workspace-snapshot-archive.test.ts)
checks the space allowance and concurrent-change boundary; its filesystem quota
is controlled test input, not a live sandbox qualification.
Create a child that inherits a saved isolated workspace or operator branch.
Verify it continues the same files after the prior writer saves, including on a
new allocation. Two bound children must not acquire concurrent writer authority.
packages/adapter-utils/src/acpx-engine/startup-timing.ts
+2
View File
@@ -67,6 +67,8 @@ export const EXECUTION_PREPARATION_SPAN_ATTRS = {
* command, a path, an argument, or an environment value can never ride a span.
*/
export const SANDBOX_STARTUP_SPAN_ATTRS = {
/** Provider state before a lease start or recovery; the host clamps values. */
state: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}state`,
/** The low-cardinality provider family (through `normalizeProviderFamily`). */
provider: `${SANDBOX_STARTUP_SPAN_ATTR_PREFIX}provider`,
/** The step or execution outcome: `ok`, `skipped`, or `failed`. */
packages/adapter-utils/src/command-redaction.test.ts
+39
View File
@@ -145,3 +145,42 @@ second-line\" status=401`;
expect(output).toContain(REDACTED_COMMAND_TEXT_VALUE);
});
});
describe("redactCommandText on long runs", () => {
const jwt =
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c";
function redactWithinTimeBudget(input: string) {
const startedAt = performance.now();
const output = redactCommandText(input);
expect(performance.now() - startedAt).toBeLessThan(1_000);
return output;
}
it("keeps redacting options, assignments, and tokens inside dash-separated runs", () => {
const cases = [
["x--token flag-secret", `x--token ${REDACTED_COMMAND_TEXT_VALUE}`],
["--x--token flag-secret", `--x--token ${REDACTED_COMMAND_TEXT_VALUE}`],
[
"a-b--api-key=assigned-secret --password 'quoted-secret'",
`a-b--api-key=${REDACTED_COMMAND_TEXT_VALUE} --password '${REDACTED_COMMAND_TEXT_VALUE}'`,
],
["prefix-MY_API_KEY=env-secret", `prefix-MY_API_KEY=${REDACTED_COMMAND_TEXT_VALUE}`],
[`--${jwt}`, `--${REDACTED_COMMAND_TEXT_VALUE}`],
[`session=${jwt}`, `session=${REDACTED_COMMAND_TEXT_VALUE}`],
["my-token is here", "my-token is here"],
];
for (const [input, expected] of cases) {
expect(redactCommandText(input)).toBe(expected);
}
});
it("scans long letter/digit and dash-separated runs in linear time", () => {
const base64Url = Buffer.from(Array.from({ length: 90_000 }, (_, index) => (index * 7919) % 256)).toString("base64url");
for (const run of ["a".repeat(120_000), "a-".repeat(60_000), "a--".repeat(40_000), base64Url]) {
expect(redactWithinTimeBudget(`${run} --token run-secret`)).toBe(`${run} --token ${REDACTED_COMMAND_TEXT_VALUE}`);
expect(redactWithinTimeBudget(`${run}--token run-secret`)).toBe(`${run}--token ${REDACTED_COMMAND_TEXT_VALUE}`);
}
});
});
packages/adapter-utils/src/command-redaction.ts
+7 -4
View File
@@ -11,16 +11,19 @@ export function isPublicExecutorToolSelector(value: string): boolean {
return PUBLIC_EXECUTOR_TOOL_SELECTORS.has(value);
}
const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
const SECRET_NAME_WORDS = String.raw`(?:api[-_]?key|(?:access[-_]?|auth[-_]?)?token|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
const SECRET_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*${SECRET_NAME_WORDS}`;
const SECRET_NAME_START = String.raw`(?<![A-Za-z0-9_-])`;
const SECRET_OPTION_NAME_PATTERN = String.raw`(?:[A-Za-z0-9_]|-(?!-))*${SECRET_NAME_WORDS}`;
const COMMAND_CLI_SECRET_OPTION_RE = new RegExp(
String.raw`(\B-{1,2}${SECRET_NAME_PATTERN}(?:\s+|=)(["']?))[^\s"'` +
String.raw`(\B-{1,2}${SECRET_OPTION_NAME_PATTERN}(?:\s+|=)(["']?))[^\s"'` +
"`" +
String.raw`]+(\2)`,
"gi",
);
const COMMAND_ENV_SECRET_ASSIGNMENT_RE = new RegExp(
String.raw`(\b${SECRET_NAME_PATTERN}\s*=\s*)(?:(\\["'])([\s\S]*?)\2|(["'])([^"'` +
String.raw`(${SECRET_NAME_START}${SECRET_NAME_PATTERN}\s*=\s*)(?:(\\["'])([\s\S]*?)\2|(["'])([^"'` +
"`" +
String.raw`\r\n]*)\4|([^\s"'` +
"`" +
@@ -32,7 +35,7 @@ const COMMAND_AUTHORIZATION_BEARER_RE =
const COMMAND_OPENAI_KEY_RE = /\bsk-[A-Za-z0-9_-]{12,}\b/g;
const COMMAND_GITHUB_TOKEN_RE = /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g;
const COMMAND_JWT_RE =
/\b[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]{8,}){2}(?:\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})?\b/g;
/\b(?<![A-Za-z0-9_]-*)[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]{8,}){2}(?:\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})?\b/g;
/** Recognize encoded JSON headers, without treating dotted identifiers as tokens. */
export function looksLikeCredentialJwt(value: string): boolean {
const segments = value.split(".");
packages/adapters/claude-local/src/server/setup-token-parse.test.ts
+8
View File
@@ -337,6 +337,14 @@ describe("parseSetupTokenPrompt", () => {
expect(parseSetupTokenPrompt(completeOutput(withKey))).toBeNull();
});
it("skips the 2.1.291 copy hint between the URL and the prompt", () => {
const hint = chaSpaced("Hold Shift while selecting to use your terminal's native copy".split(" "));
const text = [PREAMBLE_LINE, hardWrap(CLAUDE_COM_URL, 130), "", hint, "", "", PROMPT_LINE].join("\n");
const result = parseSetupTokenPrompt(text);
expect(result?.url).toBe(CLAUDE_COM_URL);
expect(result?.prompt).toBe(SETUP_TOKEN_PROMPT);
});
it("returns null when a code-like value sits on the line after the URL", () => {
// The line right after the URL must hold the browser-code prompt. A code-like
// value on that line cannot bind, so the parser returns null.
packages/adapters/claude-local/src/server/setup-token-parse.ts
+10 -3
View File
@@ -181,6 +181,10 @@ const URL_PREAMBLE_RE = /the url below to sign in/i;
// the line.
const PROMPT_LINE_RE = /^Paste code here if prompted\b[\s>*]*$/;
// The copy hint that `claude` 2.1.291 prints between the URL block and the
// prompt. The parser skips this exact line and binds no value from it.
const COPY_HINT_LINE_RE = /^Hold\s+Shift\s+while\s+selecting\s+to\s+use\s+your\s+terminal['’]s\s+native\s+copy\.?$/i;
// The maximum number of characters between the end of the URL preamble line and
// the start of the URL. The Claude UI prints the URL right after the preamble.
// The parser accepts a URL start only inside this window, so a URL far from the
@@ -363,9 +367,11 @@ function reassembleUrl(lines: string[], startLine: number): SetupTokenUrlMatch |
* line after the URL block. The parser reads the non-blank lines after the URL
* end line. The Claude UI can emit one full URL line per wrapped display row, so
* it repeats the same full authorization URL on a few consecutive lines. The
* parser skips a line that repeats `url`, then binds the prompt on the first
* non-blank, non-repeat line. That line must match the exact prompt shape.
* Returns null when the first non-blank, non-repeat line is not the prompt.
* parser skips a line that repeats `url`. The `claude` 2.1.291 UI also prints a
* copy hint before the prompt, so the parser skips a line that matches {@link
* COPY_HINT_LINE_RE}. The parser then binds the prompt on the first non-blank
* line that is not a repeat and not the copy hint. That line must match the
* exact prompt shape. Returns null when that line is not the prompt.
*/
function findBrowserCodePrompt(lines: string[], endLine: number, url: string): string | null {
for (let i = endLine + 1; i < lines.length; i += 1) {
@@ -375,6 +381,7 @@ function findBrowserCodePrompt(lines: string[], endLine: number, url: string): s
// punctuation first, the same way the repeated-URL comparison needs, so a
// repeated URL with a trailing punctuation mark still matches.
if (trimmed.replace(TRAILING_PUNCTUATION_RE, "") === url) continue;
if (COPY_HINT_LINE_RE.test(trimmed)) continue;
return PROMPT_LINE_RE.test(trimmed) ? SETUP_TOKEN_PROMPT : null;
}
return null;
packages/db/src/index.ts
+1
View File
@@ -50,3 +50,4 @@ export { issueReferenceMentions } from "./schema/issue_reference_mentions.js";
export * from "./schema/index.js";
export { databaseWorkScopeKey, signalDatabaseWork, subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "./work-signals.js";
export { withDatabaseOperation } from "./response-capture.js";
packages/db/src/response-capture-postgres.test.ts
+76 -13
View File
@@ -4,9 +4,9 @@ import { eq, sql } from "drizzle-orm";
import { pgTable, text } from "drizzle-orm/pg-core";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { createDb, withDedicatedDbConnection } from "./client.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture } from "./response-capture.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, withDatabaseOperation } from "./response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
import { encodeRunEventPayload, runEventPayload } from "./run-event-payload.js";
import { runEventPayload } from "./run-event-payload.js";
import { databaseWorkPending, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js";
const table = pgTable("response_capture_fixture", { id: text("id").primaryKey(), payload: text("payload").notNull() });
@@ -31,17 +31,17 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
vi.restoreAllMocks(); vi.unstubAllEnvs();
});
afterAll(async () => { await temporary?.cleanup(); });
function fixture(prepare?: boolean) {
function fixture(prepare?: boolean, maxConnections?: number, clock?: () => number) {
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const db = createDb(temporary.connectionString, { prepare });
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), clock);
const db = createDb(temporary.connectionString, { prepare, maxConnections });
clients.push(db); captures.push(capture);
expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db);
expect(installDatabaseResponseCapture(db, { table, encodedTable }, capture)).toBe(db);
return { db, capture, events };
}
function summary(events: Array<Record<string, unknown>>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!; }
function summary(events: Array<Record<string, unknown>>) { return events.find(event => event.event === "paperclip.db_response_capture.summary.v2")!; }
it("retains capture counts when ORM preparation is enabled", async () => {
const f = fixture(true), id = randomUUID(), payload = "prepared-capture-value";
@@ -68,12 +68,12 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await f.db.delete(table).where(eq(table.id, id)).returning()).toEqual([{ id, payload: "changed" }]);
f.capture.flush();
expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4 });
expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1").map(event => event.operation).sort())
expect(f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2").map(event => event.operation).sort())
.toEqual(["delete", "insert", "select", "update"]);
expect(JSON.stringify(f.events)).not.toContain(payload);
expect(JSON.stringify(f.events)).not.toContain(id);
});
it("counts the stored encoding before a custom column decoder changes the value", async () => {
it("skips complex stored values before a custom column decoder changes them", async () => {
const setup = createDb(temporary.connectionString); clients.push(setup);
const id = randomUUID(), payload = { text: "codec-payload-sentinel\u0000value".repeat(512) };
await setup.insert(encodedTable).values({ id, payload });
@@ -81,8 +81,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await f.db.select().from(encodedTable).where(eq(encodedTable.id, id))).toEqual([{ id, payload }]);
f.capture.flush();
const entry = f.events.find(event => event.table === "response_capture_encoded_fixture")!;
expect(entry.responseValueBytes).toBe(Buffer.byteLength(id) + Buffer.byteLength(encodeRunEventPayload(payload)));
expect(entry.responseValueBytes).toBeGreaterThan(Buffer.byteLength(JSON.stringify({ id, payload })));
expect(entry).toMatchObject({ sampledValueBytes: Buffer.byteLength(id), measuredValues: 1, unmeasuredValues: 1, incompleteSamples: 1 });
expect(JSON.stringify(f.events)).not.toContain("codec-payload-sentinel");
});
it("keeps a raw driver query lazy and counts repeated awaits without replay", async () => {
@@ -109,6 +108,69 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(summary(f.events)).toMatchObject({ startedQueries: 3, completedQueries: 3 });
expect(JSON.stringify(f.events)).not.toContain("prepared-value");
});
it("attributes concurrent prepared reuse, pooled queries, and transaction errors to their execution owners", async () => {
const setup = createDb(temporary.connectionString); clients.push(setup);
const id = randomUUID();
await setup.insert(table).values({ id, payload: "owner-private-value" });
const f = fixture(true, 1);
const prepared = withDatabaseOperation("builder.only", () =>
f.db.select().from(table).where(eq(table.id, sql.placeholder("id"))).prepare("owner_reused"));
await Promise.all([
withDatabaseOperation("response.presentation", async () => {
expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]);
await f.db.transaction(async tx => {
expect(await tx.select().from(table).where(eq(table.id, id))).toEqual([{ id, payload: "owner-private-value" }]);
});
}),
withDatabaseOperation("budget.admission", async () => {
expect(await prepared.execute({ id })).toEqual([{ id, payload: "owner-private-value" }]);
await expect(f.db.execute(sql`select 1/0`)).rejects.toThrow();
}),
]);
f.capture.flush();
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
const counts = (owner: string) => entries.filter(event => event.owner === owner).reduce((n, event) => n + Number(event.calls), 0);
expect(counts("response.presentation")).toBe(2);
expect(counts("budget.admission")).toBe(2);
expect(counts("builder.only")).toBe(0);
expect(entries.every(event => (event.callers as string[]).length === 0)).toBe(true);
expect(entries.find(event => event.queryErrors === 1)).toMatchObject({ owner: "budget.admission", rows: 0, sampledResults: 0 });
expect(summary(f.events)).toMatchObject({ startedQueries: 4, completedQueries: 4, rows: 3 });
expect(JSON.stringify(f.events)).not.toContain("owner-private-value");
});
it("attributes a lazy raw query to its first consumer rather than its builder or repeated await", async () => {
const f = fixture();
const query = withDatabaseOperation("builder.only", () => f.db.$client.unsafe("select 42 as answer"));
const result = await withDatabaseOperation("consumer.first", async () => await query);
expect(result).toEqual([{ answer: 42 }]);
expect(await withDatabaseOperation("consumer.again", async () => await query)).toBe(result);
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "consumer.first", calls: 1, rows: 1 });
expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
});
it("samples bounded driver values while retaining every query and row", async () => {
const f = fixture();
await withDatabaseOperation("sampling.owner", async () => {
for (let index = 0; index < 33; index += 1) {
expect(await f.db.execute(sql`select 42 as answer`)).toEqual([{ answer: 42 }]);
}
});
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "sampling.owner", calls: 33, rows: 33, sampledResults: 3,
sampledRows: 3, sampledValueBytes: 6, measuredValues: 3, unmeasuredValues: 0 });
expect(summary(f.events)).toMatchObject({ completedQueries: 33, rows: 33, sampledResults: 3, sampledValueBytes: 6 });
});
it("leaves a real in-flight query running when the capture deadline passes", async () => {
let clock = Date.now();
const f = fixture(undefined, 1, () => clock);
const pending = withDatabaseOperation("ending.owner", async () => await f.db.execute(sql`select pg_sleep(0.05), 42 as answer`));
await new Promise<void>(resolve => setImmediate(resolve));
clock += 60_000;
f.capture.flush();
expect(summary(f.events)).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1 });
expect((await pending)[0].answer).toBe(42);
expect(f.events).toHaveLength(1);
});
it("retains distinct builder callers through normal, prepared, and transaction execution", async () => {
const f = fixture(), id = randomUUID();
const setup = createDb(temporary.connectionString); clients.push(setup);
@@ -128,7 +190,7 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
expect(await fromB(tx, id)).toEqual([{ id, payload: "caller-sentinel" }]);
});
f.capture.flush();
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const entries = f.events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
expect(entries).toHaveLength(2);
for (const name of ["a", "b"]) {
expect(entries.find(event => (event.callers as string[]).some(location => location.startsWith(`services/capture-caller-${name}.ts:`))))
@@ -191,13 +253,14 @@ describe.skipIf(!support.supported)("database response capture PostgreSQL behavi
vi.stubEnv("PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW", `${new Date(now - 100).toISOString()}/${new Date(now + 59_000).toISOString()}`);
const db = createDb(temporary.connectionString, { prepare }); clients.push(db);
const capture = databaseResponseCaptureFromEnv()!; captures.push(capture);
expect(await withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`)))
expect(await withDatabaseOperation("dedicated.owner", () => withDedicatedDbConnection(db, async dedicated => dedicated.execute(sql`select ${"dedicated-value"}::text as value`))))
.toEqual([{ value: "dedicated-value" }]);
capture.flush();
await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v1")).toBe(true),
await vi.waitFor(() => expect(logs.some(line => JSON.parse(line).event === "paperclip.db_response_capture.summary.v2")).toBe(true),
{ timeout: 1_000, interval: 5 });
const events = logs.map(line => JSON.parse(line) as Record<string, unknown>);
expect(summary(events)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
expect(events.find(event => event.event === "paperclip.db_response_capture.entry.v2")).toMatchObject({ owner: "dedicated.owner" });
expect(logs.join("\n")).not.toContain("dedicated-value");
expect(logs.join("\n")).not.toContain(temporary.connectionString);
});
packages/db/src/response-capture.test.ts
+107 -18
View File
@@ -1,6 +1,7 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { integer, pgTable } from "drizzle-orm/pg-core";
import { afterEach, describe, expect, it, vi } from "vitest";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow } from "./response-capture.js";
import { createDatabaseResponseCapture, databaseResponseCaptureFromEnv, installDatabaseResponseCapture, publicDatabaseCallers, responseCaptureWindow, withDatabaseOperation } from "./response-capture.js";
const epoch = Date.parse("2020-01-01T00:00:00Z");
const metadata = { operation: "select", table: "capture_fixture", callers: ["services/fixture.ts:4:2"] };
@@ -14,6 +15,9 @@ function fixture(emitOverride?: (event: Record<string, unknown>) => void) {
captures.push(capture);
return { capture, events, now: (value: number) => { clock = value; } };
}
function driverRows(rows: unknown[][]) {
return Object.assign(rows, { columns: Array.from({ length: rows[0]?.length ?? 0 }, (_, index) => ({ name: `column_${index}` })) });
}
describe("bounded database response capture", () => {
it("paces a complete environment capture without extending the measurement window or leaving timers", () => {
@@ -36,13 +40,13 @@ describe("bounded database response capture", () => {
capture.flush();
vi.advanceTimersByTime(32_000);
expect(emitted).toHaveLength(2049);
const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v1");
const groups = emitted.filter(item => item.event.event === "paperclip.db_response_capture.entry.v2");
const summary = emitted.at(-1)!.event;
expect(groups).toHaveLength(2048);
expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v1", completedQueries: 2200,
expect(summary).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2200,
groups: 2048, closedAt: "2020-01-01T00:01:00.000Z" });
expect(groups.reduce((calls, item) => calls + Number(item.event.calls), 0)).toBe(summary.completedQueries);
expect(groups.reduce((bytes, item) => bytes + Number(item.event.responseValueBytes), 0)).toBe(summary.responseValueBytes);
expect(groups.reduce((bytes, item) => bytes + Number(item.event.sampledValueBytes), 0)).toBe(summary.sampledValueBytes);
expect(groups.map(item => item.event.groupIndex)).toEqual(Array.from({ length: 2048 }, (_, index) => index));
for (const { at } of emitted) expect(emitted.filter(item => item.at >= at && item.at < at + 1000).length).toBeLessThanOrEqual(64);
expect(JSON.stringify(emitted)).not.toContain("capture-response-sentinel");
@@ -89,13 +93,13 @@ describe("bounded database response capture", () => {
expect(installDatabaseResponseCapture(db, { table })).toBe(db);
expect(db.select).toBe(select); expect(db.transaction).toBe(transaction);
});
it("preserves exact values and only emits counts, sizes, and caller metadata", async () => {
const f = fixture(), rows = [{ text: "sentinel-result-value🙂\n\"", n: 4n }];
it("preserves exact values and only emits counts, sampled sizes, and caller metadata", async () => {
const f = fixture(), text = "sentinel-result-value🙂\n\"", rows = driverRows([[text, 4n]]);
expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows);
f.capture.flush();
const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v1")!;
expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, serializationErrors: 0,
responseValueBytes: Buffer.byteLength(JSON.stringify(rows, (_, value) => typeof value === "bigint" ? value.toString() : value)) });
const entry = f.events.find(event => event.event === "paperclip.db_response_capture.entry.v2")!;
expect(entry).toMatchObject({ calls: 1, rows: 1, queryErrors: 0, owner: "unattributed", sampledResults: 1,
sampledValueBytes: Buffer.byteLength(text) + 1, measuredValues: 2, unmeasuredValues: 0, incompleteSamples: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-result-value");
expect(f.events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1, inFlightAtEnd: 0,
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false });
@@ -106,10 +110,91 @@ describe("bounded database response capture", () => {
{ name: "sentinel-column-name" }, { name: "b" }, { name: "n" }, { name: "empty" } ] });
expect(await f.capture.execute(() => Promise.resolve(rows), metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ responseValueBytes: Buffer.byteLength(text) + 1 + 2, estimatedValues: 0 });
expect(f.events[0]).toMatchObject({ sampledValueBytes: Buffer.byteLength(text) + 1 + 2, measuredValues: 4, unmeasuredValues: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-driver-text");
expect(JSON.stringify(f.events)).not.toContain("sentinel-column-name");
});
it("counts every result but sizes only the first and every sixteenth successful result per group", () => {
const f = fixture();
for (let index = 0; index < 33; index += 1) {
const rows = driverRows([["abc", 12]]);
if (index % 16 !== 0) Object.defineProperty(rows[0], 0, { get() { throw new Error("unsampled cell read"); } });
expect(withDatabaseOperation("fixture.owner", () => f.capture.execute(() => rows, metadata))).toBe(rows);
}
f.capture.flush();
expect(f.events[0]).toMatchObject({ owner: "fixture.owner", calls: 33, rows: 33, sampledResults: 3,
sampledRows: 3, sampledValueBytes: 15, measuredValues: 6, unmeasuredValues: 0, incompleteSamples: 0 });
expect(f.events[0].totalWallMs).toBeGreaterThanOrEqual(0);
expect(f.events[0].maxWallMs).toBeLessThanOrEqual(Number(f.events[0].totalWallMs));
});
it("bounds sampled cells, string scanning, and bytes without traversing large or complex values", () => {
const f = fixture();
const rows = driverRows([["x".repeat(1_000_000), { large: "private-json" }, ...Array.from({ length: 300 }, () => "y".repeat(16_384))]]);
Object.defineProperty(rows[0], 256, { get() { throw new Error("cell budget exceeded"); } });
expect(f.capture.execute(() => rows, metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ sampledResults: 1, sampledValueBytes: 65_536, measuredValues: 4,
unmeasuredValues: 298, incompleteSamples: 1 });
expect(JSON.stringify(f.events)).not.toContain("private-json");
const cellBound = fixture(), cells = driverRows([Array.from({ length: 257 }, () => null)]);
Object.defineProperty(cells[0], 256, { get() { throw new Error("cell budget exceeded"); } });
cellBound.capture.execute(() => cells, metadata);
cellBound.capture.flush();
expect(cellBound.events[0]).toMatchObject({ sampledValueBytes: 0, measuredValues: 256, unmeasuredValues: 1, incompleteSamples: 1 });
});
it("restores nested owners and uses the direct path without an active capture", () => {
const returned = Promise.resolve("unchanged");
expect(withDatabaseOperation("disabled", () => returned)).toBe(returned);
const f = fixture();
withDatabaseOperation("outer", () => {
withDatabaseOperation("inner", () => f.capture.execute(() => driverRows([[1]]), metadata));
f.capture.execute(() => driverRows([[2]]), metadata);
});
f.now(epoch + 60_001);
withDatabaseOperation("expired", () => {
const next = fixture();
next.capture.execute(() => driverRows([[3]]), metadata);
next.capture.flush();
expect(next.events[0]).toMatchObject({ owner: "unattributed" });
});
f.capture.flush();
expect(f.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["inner", "outer"]);
});
it("retains an armed operation across the capture start without counting earlier queries", async () => {
let clock = epoch;
const events: Array<Record<string, unknown>> = [];
const capture = createDatabaseResponseCapture({ start: epoch + 1_000, end: epoch + 60_000 }, event => events.push(event), () => clock);
captures.push(capture);
let release!: () => void;
const gate = new Promise<void>(resolve => { release = resolve; });
const pending = withDatabaseOperation("started.before.window", async () => {
capture.execute(() => driverRows([[1]]), metadata);
await gate;
return capture.execute(() => driverRows([[2]]), metadata);
});
clock = epoch + 2_000;
release();
await pending;
capture.flush();
expect(events[0]).toMatchObject({ owner: "started.before.window", calls: 1, rows: 1 });
expect(events.at(-1)).toMatchObject({ startedQueries: 1, completedQueries: 1 });
});
it("keeps overlapping capture scopes but clears retained owners when the last capture closes", () => {
const first = fixture(), second = fixture();
let destination = second.capture;
const retained = withDatabaseOperation("previous.capture", () => AsyncLocalStorage.bind(() =>
destination.execute(() => driverRows([[1]]), metadata)));
first.capture.flush();
retained();
second.capture.flush();
expect(second.events[0]).toMatchObject({ owner: "previous.capture", calls: 1 });
const later = fixture();
destination = later.capture;
withDatabaseOperation("new.capture", () => destination.execute(() => driverRows([[2]]), metadata));
retained();
later.capture.flush();
expect(later.events.slice(0, -1).map(event => event.owner).sort()).toEqual(["new.capture", "unattributed"]);
});
it("does not alter synchronous or asynchronous errors or retry them", async () => {
const f = fixture(), error = new Error("sentinel-private-error");
const sync = vi.fn(() => { throw error; }), asyncFailure = vi.fn(() => Promise.reject(error));
@@ -117,14 +202,17 @@ describe("bounded database response capture", () => {
await expect(f.capture.execute(asyncFailure, metadata)).rejects.toBe(error);
expect(sync).toHaveBeenCalledTimes(1); expect(asyncFailure).toHaveBeenCalledTimes(1);
f.capture.flush();
expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, responseValueBytes: 0 });
expect(f.events[0]).toMatchObject({ calls: 2, queryErrors: 2, sampledValueBytes: 0 });
expect(JSON.stringify(f.events)).not.toContain("sentinel-private-error");
});
it("keeps unserializable results and logging failures out of the query outcome", () => {
it("does not serialize complex values or let logging failures change query outcomes", () => {
const f = fixture(), circular: Record<string, unknown> = {}; circular.self = circular;
expect(f.capture.execute(() => circular, metadata)).toBe(circular);
const toJSON = vi.fn(() => { throw new Error("must not serialize"); });
const rows = driverRows([[circular, { toJSON }]]);
expect(f.capture.execute(() => rows, metadata)).toBe(rows);
f.capture.flush();
expect(f.events[0]).toMatchObject({ calls: 1, serializationErrors: 1 });
expect(f.events[0]).toMatchObject({ calls: 1, sampledValueBytes: 0, unmeasuredValues: 2, incompleteSamples: 1 });
expect(toJSON).not.toHaveBeenCalled();
const brokenLogger = fixture(() => { throw new Error("logger fixture"); });
expect(brokenLogger.capture.execute(() => circular, metadata)).toBe(circular);
expect(() => brokenLogger.capture.flush()).not.toThrow();
@@ -152,12 +240,12 @@ describe("bounded database response capture", () => {
expect(await result).toBe(rows);
f.capture.flush();
expect(f.events).toHaveLength(1);
expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0 });
expect(f.events[0]).toMatchObject({ startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, sampledValueBytes: 0 });
expect(JSON.stringify(f.events)).not.toContain("late-before-flush-sentinel");
});
it("bounds cardinality without dropping response-byte totals", () => {
it("bounds cardinality while distinguishing sampled bytes from complete call and row counts", () => {
const f = fixture();
for (let i = 0; i < 2200; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] });
for (let i = 0; i < 2200; i += 1) f.capture.execute(() => driverRows([[i]]), { ...metadata, callers: [`services/f${i}.ts:1:1`] });
f.capture.flush();
const summary = f.events.at(-1)!;
expect(summary).toMatchObject({ startedQueries: 2200, completedQueries: 2200, groups: 2048, overflowCalls: 153,
@@ -169,7 +257,8 @@ describe("bounded database response capture", () => {
expect(entries.filter(entry => entry.groupPage === page)).toHaveLength(128);
}
expect(entries.reduce((sum, entry) => sum + Number(entry.calls), 0)).toBe(2200);
expect(summary.responseValueBytes).toBe(Array.from({ length: 2200 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0));
expect(summary).toMatchObject({ rows: 2200, sampledResults: 2057, sampleEvery: 16 });
expect(summary.sampledValueBytes).toBe(entries.reduce((sum, entry) => sum + Number(entry.sampledValueBytes), 0));
});
it.each([0, 129])("numbers every page including an empty or partial final page (%s)", count => {
const f = fixture();
packages/db/src/response-capture.ts
+89 -36
View File
@@ -3,11 +3,27 @@ import { randomUUID } from "node:crypto";
import { getTableName, is, Table } from "drizzle-orm";
type Metadata = { operation: string; table: string; callers: string[] };
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number; queryErrors: number; serializationErrors: number; estimatedValues: number };
type Entry = Metadata & { owner: string; calls: number; rows: number; queryErrors: number; totalWallMs: number; maxWallMs: number;
sampledResults: number; sampledRows: number; sampledValueBytes: number; maxSampledValueBytes: number;
measuredValues: number; unmeasuredValues: number; incompleteSamples: number; sizingWallMs: number };
type Event = Record<string, unknown>;
type Method = (this: unknown, ...args: unknown[]) => unknown;
export type ResponseCaptureWindow = { start: number; end: number };
const GROUP_PAGE_SIZE = 128;
const SAMPLE_EVERY = 16;
const MAX_SAMPLE_CELLS = 256;
const MAX_SAMPLE_STRING_LENGTH = 16_384;
const MAX_SAMPLE_STRING_UNITS = 65_536;
const MAX_SAMPLE_BYTES = 65_536;
let operationScope = new AsyncLocalStorage<string>();
const captureWindows = new Set<() => boolean>();
export function withDatabaseOperation<T>(owner: string, action: () => T): T {
for (const armed of captureWindows) {
if (armed()) return operationScope.run(owner, action);
}
return action();
}
export function responseCaptureWindow(value: string | undefined, now = Date.now()): ResponseCaptureWindow | undefined {
if (!value) return;
@@ -32,28 +48,37 @@ export function publicDatabaseCallers(stack: string | undefined): string[] {
return callers;
}
function responseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
function responseValueSize(value: unknown): { bytes: number; measuredValues: number; unmeasuredValues: number } {
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
if (!Array.isArray(value) || !Array.isArray(columns)) {
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
return { bytes: 0, measuredValues: 0, unmeasuredValues: 1 };
}
let bytes = 0, estimatedValues = 0;
let bytes = 0, measuredValues = 0, visitedValues = 0, stringUnits = 0;
const totalValues = value.length * columns.length;
if (columns.length === 0) return { bytes: 0, measuredValues: 0, unmeasuredValues: 0 };
for (const row of value) {
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
for (const cell of cells) {
if (cell === null || cell === undefined) continue;
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
else if (typeof cell === "boolean") bytes += 1;
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
else {
estimatedValues += 1;
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
for (let index = 0; index < columns.length; index += 1) {
if (visitedValues >= MAX_SAMPLE_CELLS || bytes >= MAX_SAMPLE_BYTES) {
return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues };
}
visitedValues += 1;
const cell = Array.isArray(row) ? row[index] : row[columns[index].name];
let size: number;
if (cell === null || cell === undefined) size = 0;
else if (typeof cell === "string") {
if (cell.length > MAX_SAMPLE_STRING_LENGTH || stringUnits + cell.length > MAX_SAMPLE_STRING_UNITS) continue;
stringUnits += cell.length;
size = Buffer.byteLength(cell);
} else if (typeof cell === "boolean") size = 1;
else if (typeof cell === "number") size = Buffer.byteLength(String(cell));
else if (typeof cell === "bigint" && cell >= -9_223_372_036_854_775_808n && cell <= 9_223_372_036_854_775_807n) size = Buffer.byteLength(String(cell));
else continue;
if (bytes + size > MAX_SAMPLE_BYTES) continue;
measuredValues += 1;
bytes += size;
}
}
return { bytes, estimatedValues };
return { bytes, measuredValues, unmeasuredValues: totalValues - measuredValues };
}
export function createDatabaseResponseCapture(
@@ -71,32 +96,45 @@ export function createDatabaseResponseCapture(
const send = (event: Event) => { try { emit(event); } catch {} };
const expired = () => closed || now() >= window.end;
const active = () => !expired() && now() >= window.start;
const armed = () => !expired();
captureWindows.add(armed);
const flush = () => {
if (closed) return;
closed = true;
clearTimeout(timer);
captureWindows.delete(armed);
if (captureWindows.size === 0) {
operationScope.disable();
operationScope = new AsyncLocalStorage<string>();
}
scope.disable();
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v1", captureId: id,
const ordered = [...entries.values()].sort((a, b) => b.calls - a.calls);
for (const [groupIndex, entry] of ordered.entries()) send({ event: "paperclip.db_response_capture.entry.v2", captureId: id,
groupIndex, groupPage: Math.floor(groupIndex / GROUP_PAGE_SIZE),
operation: entry.operation, table: entry.table, callers: entry.callers, calls: entry.calls, rows: entry.rows,
responseValueBytes: entry.bytes, maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors });
send({ event: "paperclip.db_response_capture.summary.v1", captureId: id,
...entry });
send({ event: "paperclip.db_response_capture.summary.v2", captureId: id,
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
closedAt: new Date(now()).toISOString(),
startedQueries: started, completedQueries: completed, inFlightAtEnd: inFlight, groups: entries.size,
groupPageSize: GROUP_PAGE_SIZE, groupPages: Math.ceil(ordered.length / GROUP_PAGE_SIZE),
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
overflowCalls, rows: ordered.reduce((sum, entry) => sum + entry.rows, 0),
sampledResults: ordered.reduce((sum, entry) => sum + entry.sampledResults, 0),
sampledValueBytes: ordered.reduce((sum, entry) => sum + entry.sampledValueBytes, 0),
sizingWallMs: ordered.reduce((sum, entry) => sum + entry.sizingWallMs, 0),
sampleEvery: SAMPLE_EVERY, sampling: "first-and-every-16th-successful-result-per-group",
maxSampleCells: MAX_SAMPLE_CELLS, maxSampleStringLength: MAX_SAMPLE_STRING_LENGTH,
maxSampleStringUnits: MAX_SAMPLE_STRING_UNITS, maxSampleBytes: MAX_SAMPLE_BYTES,
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
measurementBoundary: "postgres-js results before Drizzle column decoding",
wallTimeBoundary: "first query consumption to driver result; includes queue, database, network and event-loop waits; not CPU",
byteMeasurement: "sampled primitive values only; complex/oversized/budget-exceeding values unmeasured; no extrapolation",
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false });
entries.clear();
};
const timer = setTimeout(flush, Math.max(0, window.end - now()));
timer.unref();
function finish(metadata: Metadata, value: unknown, failed: boolean) {
function finish(metadata: Metadata & { owner: string }, value: unknown, failed: boolean, startedAt: number) {
if (!active()) return;
inFlight -= 1;
completed += 1;
@@ -104,38 +142,52 @@ export function createDatabaseResponseCapture(
let key = JSON.stringify(metadata);
if (!entries.has(key) && entries.size >= 2047) {
overflowCalls += 1;
metadata = { operation: "other", table: "other", callers: [] };
metadata = { owner: "other", operation: "other", table: "other", callers: [] };
key = "overflow";
}
let entry = entries.get(key);
if (!entry) {
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0, queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
entry = { ...metadata, calls: 0, rows: 0, queryErrors: 0, totalWallMs: 0, maxWallMs: 0,
sampledResults: 0, sampledRows: 0, sampledValueBytes: 0, maxSampledValueBytes: 0,
measuredValues: 0, unmeasuredValues: 0, incompleteSamples: 0, sizingWallMs: 0 };
entries.set(key, entry);
}
entry.calls += 1;
const wallMs = performance.now() - startedAt;
entry.totalWallMs += wallMs;
entry.maxWallMs = Math.max(entry.maxWallMs, wallMs);
if (failed) { entry.queryErrors += 1; return; }
entry.rows += Array.isArray(value) ? value.length : 0;
if ((entry.calls - entry.queryErrors - 1) % SAMPLE_EVERY !== 0) return;
entry.sampledResults += 1;
entry.sampledRows += Array.isArray(value) ? value.length : 0;
const sizingStartedAt = performance.now();
try {
const size = responseValueSize(value);
entry.bytes += size.bytes;
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
entry.estimatedValues += size.estimatedValues;
} catch { entry.serializationErrors += 1; }
entry.sampledValueBytes += size.bytes;
entry.maxSampledValueBytes = Math.max(entry.maxSampledValueBytes, size.bytes);
entry.measuredValues += size.measuredValues;
entry.unmeasuredValues += size.unmeasuredValues;
if (size.unmeasuredValues) entry.incompleteSamples += 1;
} catch { entry.incompleteSamples += 1; }
finally { entry.sizingWallMs += performance.now() - sizingStartedAt; }
} catch {}
}
function execute(action: () => unknown, metadata: Metadata) {
if (!active()) return action();
inFlight += 1; started += 1;
const owned = { ...metadata, owner: operationScope.getStore() ?? "unattributed" };
const startedAt = performance.now();
let result: unknown;
try { result = action(); }
catch (error) { finish(metadata, undefined, true); throw error; }
catch (error) { finish(owned, undefined, true, startedAt); throw error; }
let asynchronous = false;
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
if (asynchronous) {
return Promise.resolve(result).then(value => { finish(metadata, value, false); return value; },
error => { finish(metadata, undefined, true); throw error; });
return Promise.resolve(result).then(value => { finish(owned, value, false, startedAt); return value; },
error => { finish(owned, undefined, true, startedAt); throw error; });
}
finish(metadata, result, false);
finish(owned, result, false, startedAt);
return result;
}
return { window, roots, queries, factories, clients, driverQueries, scope, active, expired, execute, flush };
@@ -240,7 +292,7 @@ export function installDatabaseResponseCapture<T extends object>(db: T, schema:
}
return scoped ? { ...scoped, table: scoped.table === "other" ? table : scoped.table }
: { operation: text.match(/^\s*(select|insert|update|delete)\b/i)?.[1]?.toLowerCase() ?? "execute",
table, callers: publicDatabaseCallers(new Error().stack) };
table, callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] };
}
function client(value: unknown) {
if ((!object(value) && typeof value !== "function") || capture!.clients.has(value)) return;
@@ -277,7 +329,8 @@ export function installDatabaseResponseCapture<T extends object>(db: T, schema:
try {
if (method === "with") { root(result); return result; }
const operation = method.startsWith("select") ? "select" : method;
const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]), callers: publicDatabaseCallers(new Error().stack) };
const metadata = { operation, table: operation === "select" || method === "execute" ? "other" : tableName(args[0]),
callers: capture!.active() && !operationScope.getStore() ? publicDatabaseCallers(new Error().stack) : [] };
if (operation === "select") factory(result, "from", metadata, true);
else if (method === "insert") factory(result, "values", metadata);
else if (method === "update") factory(result, "set", metadata);
packages/plugins/sandbox-providers/daytona/src/lease-tracing.test.ts
+273
View File
@@ -0,0 +1,273 @@
import { createInterface } from "node:readline";
import { PassThrough } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { NOOP_PLUGIN_TRACER, startWorkerRpcHost, type PluginInvocationContext, type PluginTracer } from "@paperclipai/plugin-sdk";
import plugin, { __resetDaytonaSandboxHandleCacheForTest, __setDaytonaPluginContextForTest } from "./plugin.js";
import manifest from "./manifest.js";
import { leaseStateAttributes, withLeaseSpan } from "./lease-tracing.js";
const { get, create } = vi.hoisted(() => ({ get: vi.fn(), create: vi.fn() }));
vi.mock("@daytonaio/sdk", () => ({
Daytona: class { get = get; create = create; },
DaytonaNotFoundError: class extends Error {},
DaytonaTimeoutError: class extends Error {},
}));
type Receipt = {
name: string;
startTimeMs: number;
endTimeMs: number;
attributes?: Record<string, unknown>;
status?: { code: number };
children: Receipt[];
};
function sandbox(id = "fixture-sandbox", state = "archived") {
return {
id, state, target: "us", recoverable: true, errorReason: null as string | null,
start: vi.fn().mockResolvedValue(undefined),
stop: vi.fn().mockResolvedValue(undefined),
refreshData: vi.fn().mockResolvedValue(undefined),
recover: vi.fn().mockResolvedValue(undefined),
delete: vi.fn().mockResolvedValue(undefined),
getWorkDir: vi.fn().mockResolvedValue("/home/daytona"),
fs: { createFolder: vi.fn().mockResolvedValue(undefined), uploadFile: vi.fn().mockResolvedValue(undefined) },
process: { executeCommand: vi.fn(async (command: string) => ({
exitCode: 0, result: command.startsWith("cat ") ? '{"token":"private-token"}' : "bash",
})) },
};
}
const params = {
driverKey: "daytona", companyId: "fixture-company", environmentId: "fixture-environment",
providerLeaseId: "fixture-sandbox", config: { apiKey: "private-api-key", timeoutMs: 300000, reuseLease: true },
leaseMetadata: { workspaceSentinel: { token: "private-token", result: "written" } },
};
function workerHarness() {
const stdin = new PassThrough(), stdout = new PassThrough();
const lines = createInterface({ input: stdout });
const pending = new Map<string, { resolve(value: unknown): void; reject(error: Error): void }>();
const receipts: Array<{ invocationId: string; span: Receipt }> = [];
let sequence = 0;
lines.on("line", line => {
const message = JSON.parse(line);
if (message.method) {
if (message.method === "span.record") receipts.push({ invocationId: message.paperclipInvocationId, span: message.params });
if (message.id !== undefined) stdin.write(JSON.stringify({ jsonrpc: "2.0", id: message.id, result: null }) + "\n");
} else {
const waiter = pending.get(message.id);
pending.delete(message.id);
if (message.error) waiter?.reject(new Error(message.error.message));
else waiter?.resolve(message.result);
}
});
const worker = startWorkerRpcHost({ plugin, stdin, stdout });
function call(method: string, input: unknown, invocation?: PluginInvocationContext): Promise<unknown> {
const id = String(++sequence);
return new Promise((resolve, reject) => {
pending.set(id, { resolve, reject });
stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params: input,
...(invocation ? { paperclipInvocation: invocation } : {}) }) + "\n");
});
}
return {
call, receipts,
initialize: () => call("initialize", { manifest, config: {}, apiVersion: 1,
instanceInfo: { instanceId: "fixture", hostVersion: "0.0.0" } }),
close() { worker.stop(); lines.close(); stdin.destroy(); stdout.destroy(); },
};
}
function invocation(id: string, enabled = true): PluginInvocationContext {
return { id, scope: { companyId: "fixture-company" },
...(enabled ? { traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01" } : {}) };
}
describe("Daytona lease worker tracing", () => {
beforeEach(() => { get.mockReset(); create.mockReset(); __resetDaytonaSandboxHandleCacheForTest(); });
afterEach(() => { __setDaytonaPluginContextForTest(null); __resetDaytonaSandboxHandleCacheForTest(); });
it.each([true, false])("records the archived start and nested verification only with trace context: %s", async enabled => {
const resource = sandbox(); get.mockResolvedValue(resource);
const worker = workerHarness();
try {
await worker.initialize();
expect(await worker.call("environmentResumeLease", params, invocation("resume", enabled)))
.toMatchObject({ providerLeaseId: resource.id, metadata: { resumedFromState: "archived" } });
expect(resource.start).toHaveBeenCalledExactlyOnceWith(300);
expect(get).toHaveBeenCalledExactlyOnceWith(resource.id);
if (!enabled) { expect(worker.receipts).toEqual([]); return; }
expect(worker.receipts).toHaveLength(1);
const root = worker.receipts[0].span;
expect(root.name).toBe("plugin.worker");
expect(root.children.map(span => span.name)).toEqual(["lease.lookup", "lease.channel_cleanup", "lease.start", "lease.verify"]);
expect(root.children[2].attributes).toMatchObject({ "paperclip.sandbox.startup.state": "archived" });
expect(root.children[3].children.map(span => span.name)).toEqual(["lease.workspace", "lease.sentinel", "lease.shell"]);
function checkBounds(parent: Receipt) {
for (const child of parent.children) {
expect(child.startTimeMs).toBeGreaterThanOrEqual(parent.startTimeMs);
expect(child.endTimeMs).toBeGreaterThanOrEqual(child.startTimeMs);
expect(child.endTimeMs).toBeLessThanOrEqual(parent.endTimeMs);
checkBounds(child);
}
}
checkBounds(root);
expect(JSON.stringify(root)).not.toMatch(/private-|fixture-sandbox|fixture-company|\/home\//);
} finally { worker.close(); }
});
it("keeps simultaneous resume receipts separate and closes failed starts", async () => {
const slow = sandbox(), failed = sandbox("failed-sandbox", "stopped");
let release!: () => void, started!: () => void;
const waiting = new Promise<void>(resolve => { release = resolve; });
const entered = new Promise<void>(resolve => { started = resolve; });
slow.start.mockImplementation(async () => { started(); await waiting; });
failed.start.mockRejectedValue(new Error("private-provider-error"));
get.mockImplementation(async id => id === slow.id ? slow : failed);
const worker = workerHarness();
try {
await worker.initialize();
const first = worker.call("environmentResumeLease", params, invocation("slow"));
await entered;
await expect(worker.call("environmentResumeLease", { ...params, providerLeaseId: failed.id }, invocation("failed")))
.rejects.toThrow("private-provider-error");
release(); await first;
expect(worker.receipts.map(record => record.invocationId)).toEqual(["failed", "slow"]);
const failure = worker.receipts[0].span;
expect(failure.status).toEqual({ code: 2 });
expect(failure.children.find(span => span.name === "lease.start")?.status).toEqual({ code: 2 });
expect(failure.children.some(span => span.name === "lease.verify")).toBe(false);
expect(JSON.stringify(worker.receipts)).not.toContain("private-provider-error");
expect(slow.start).toHaveBeenCalledTimes(1);
expect(failed.start).toHaveBeenCalledTimes(1);
} finally { release(); worker.close(); }
});
it("waits for a pending teardown and records the wait under the resume worker", async () => {
const resource = sandbox("fixture-sandbox", "started"); get.mockResolvedValue(resource);
let release!: () => void, stopping!: () => void;
const waiting = new Promise<void>(resolve => { release = resolve; });
const entered = new Promise<void>(resolve => { stopping = resolve; });
resource.stop.mockImplementation(async () => { stopping(); await waiting; resource.state = "stopped"; });
const worker = workerHarness();
try {
await worker.initialize();
const teardown = worker.call("environmentReleaseLease", params, invocation("release"));
await entered;
let settled = false;
const resume = worker.call("environmentResumeLease", params, invocation("resume"))
.then(result => { settled = true; return result; });
// Process the resume request while the actual provider stop is held open.
await new Promise<void>(resolve => setImmediate(resolve));
expect(settled).toBe(false);
expect(get).toHaveBeenCalledTimes(1);
expect(resource.start).not.toHaveBeenCalled();
release();
await teardown;
expect(await resume).toMatchObject({ providerLeaseId: resource.id });
const root = worker.receipts.find(receipt => receipt.invocationId === "resume")!.span;
expect(root.name).toBe("plugin.worker");
expect(root.children.filter(span => span.name === "lease.wait_for_teardown")).toHaveLength(1);
expect(root.children[0].name).toBe("lease.wait_for_teardown");
expect(root.children[0].endTimeMs).toBeLessThanOrEqual(root.children[1].startTimeMs);
expect(get).toHaveBeenCalledTimes(2);
expect(resource.start).toHaveBeenCalledTimes(1);
} finally { release(); worker.close(); }
});
it.each([false, true])("records missing-container confirmation and preserves refresh failures: %s", async fails => {
const resource = sandbox("fixture-sandbox", "error");
resource.recoverable = false;
resource.errorReason = `not found: failed to inspect sandbox container ${resource.id}: Error response from daemon: No such container: ${resource.id}`;
get.mockResolvedValue(resource);
resource.refreshData.mockImplementation(async () => {
if (fails) throw new Error("private-refresh-failure");
resource.state = "started";
});
const worker = workerHarness();
try {
await worker.initialize();
const result = worker.call("environmentResumeLease", params, invocation("resume"));
if (fails) await expect(result).rejects.toThrow("private-refresh-failure");
else expect(await result).toMatchObject({ providerLeaseId: resource.id });
expect(resource.refreshData).toHaveBeenCalledTimes(1);
const root = worker.receipts[0].span;
expect(root.children.map(span => span.name)).toEqual(fails
? ["lease.lookup", "lease.refresh"] : ["lease.lookup", "lease.refresh", "lease.verify"]);
expect(root.children[1].status).toEqual(fails ? { code: 2 } : undefined);
expect(resource.start).not.toHaveBeenCalled();
expect(resource.delete).not.toHaveBeenCalled();
expect(create).not.toHaveBeenCalled();
expect(JSON.stringify(worker.receipts)).not.toContain("private-refresh-failure");
} finally { worker.close(); }
});
it.each(["started", "error"])("traces a %s sandbox without adding a provider start", async state => {
const resource = sandbox("fixture-sandbox", state); get.mockResolvedValue(resource);
const worker = workerHarness();
try {
await worker.initialize();
await worker.call("environmentResumeLease", params, invocation("resume"));
expect(resource.start).not.toHaveBeenCalled();
expect(resource.recover).toHaveBeenCalledTimes(state === "error" ? 1 : 0);
const phases = worker.receipts[0].span.children;
expect(phases.some(span => span.name === "lease.start")).toBe(false);
expect(phases.some(span => span.name === "lease.recover")).toBe(state === "error");
} finally { worker.close(); }
});
it("closes the verification group and failing workspace child", async () => {
const resource = sandbox(); get.mockResolvedValue(resource);
resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error"));
const worker = workerHarness();
try {
await worker.initialize();
await expect(worker.call("environmentResumeLease", params, invocation("resume")))
.rejects.toThrow("private-workspace-error");
const verify = worker.receipts[0].span.children.find(span => span.name === "lease.verify")!;
expect(verify.status).toEqual({ code: 2 });
expect(verify.children.map(span => span.name)).toEqual(["lease.workspace"]);
expect(verify.children[0].status).toEqual({ code: 2 });
expect(resource.delete).not.toHaveBeenCalled();
} finally { worker.close(); }
});
it("records setup failure and compensating cleanup as siblings", async () => {
const resource = sandbox(); create.mockResolvedValue(resource);
resource.getWorkDir.mockRejectedValue(new Error("private-workspace-error"));
const worker = workerHarness();
try {
await worker.initialize();
await expect(worker.call("environmentAcquireLease", { ...params, runId: "fixture-run" }, invocation("acquire")))
.rejects.toThrow("private-workspace-error");
expect(resource.delete).toHaveBeenCalledTimes(1);
const root = worker.receipts[0].span;
expect(root.children.map(span => span.name)).toEqual(["lease.create", "lease.workspace", "lease.cleanup"]);
expect(root.children[1].status).toEqual({ code: 2 });
expect(root.children[2].status).toBeUndefined();
} finally { worker.close(); }
});
});
describe("lease tracing cannot change provider work", () => {
it.each(["missing", "before", "after", "during"])("preserves the result and error with a %s tracer failure", async mode => {
const tracer: PluginTracer = { ...NOOP_PLUGIN_TRACER,
...(mode === "missing" ? {} : { async withSpan<T>(_name: string, work: () => T | Promise<T>): Promise<T> {
if (mode === "before") throw new Error("trace failure");
const result = work();
if (mode === "during") throw new Error("trace failure");
await result;
throw new Error("trace failure");
} }),
};
const success = vi.fn(async () => 42), error = new Error("provider failure"), failure = vi.fn(async () => { throw error; });
expect(await withLeaseSpan(tracer, "lease.lookup", success)).toBe(42);
await expect(withLeaseSpan(tracer, "lease.lookup", failure)).rejects.toBe(error);
expect(success).toHaveBeenCalledTimes(1); expect(failure).toHaveBeenCalledTimes(1);
});
it("normalizes unknown provider states before recording", () => {
expect(leaseStateAttributes("provider-private-state")).toEqual({ "paperclip.sandbox.startup.state": "other" });
});
});
packages/plugins/sandbox-providers/daytona/src/lease-tracing.ts
+49
View File
@@ -0,0 +1,49 @@
import type { PluginTracer } from "@paperclipai/plugin-sdk";
export type LeaseSpanName =
| "lease.wait_for_teardown"
| "lease.lookup"
| "lease.refresh"
| "lease.channel_cleanup"
| "lease.start"
| "lease.recover"
| "lease.verify"
| "lease.workspace"
| "lease.sentinel"
| "lease.shell"
| "lease.create"
| "lease.expiry"
| "lease.cleanup";
/** Keep provider state useful without exporting a new SDK value as free text. */
export function leaseStateAttributes(state: unknown): Record<string, string> {
return {
"paperclip.sandbox.startup.state":
state === "started" || state === "stopped" || state === "archived" || state === "error"
? state : "other",
};
}
/** Use the invocation's async parent, while keeping tracer faults out of work. */
export async function withLeaseSpan<T>(
tracer: PluginTracer,
name: LeaseSpanName,
work: () => Promise<T>,
attributes?: Record<string, string | number | boolean>,
): Promise<T> {
let result: Promise<T> | undefined;
// A tracer can fail before or after invoking work. Preserve its exact result
// or error and never repeat provider operations in either case.
const runOnce = () => result ??= Promise.resolve().then(work);
try {
if (tracer.withSpan) {
await tracer.withSpan(name, runOnce, {
"paperclip.sandbox.startup.provider": "daytona",
...attributes,
});
}
} catch {
// The operation's error is still held by result and is rethrown below.
}
return runOnce();
}
packages/plugins/sandbox-providers/daytona/src/plugin.ts
+65 -52
View File
@@ -50,6 +50,7 @@ import type {
} from "@paperclipai/plugin-sdk";
import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js";
import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js";
import { withLeaseSpan, leaseStateAttributes, type LeaseSpanName } from "./lease-tracing.js";
import { observeSessionStreamExit } from "./session-stream-exit.js";
import { DaytonaIdleActivity, trackDaytonaHooks, type DaytonaIdleProof } from "./idle-activity.js";
@@ -126,6 +127,10 @@ export function getPluginTracer(): PluginTracer {
return pluginContext?.tracer ?? NOOP_PLUGIN_TRACER;
}
function traceLeaseStep<T>(name: LeaseSpanName, work: () => Promise<T>, attributes?: Record<string, string | number | boolean>): Promise<T> {
return withLeaseSpan(getPluginTracer(), name, work, attributes);
}
/**
* Test seam: set the module-level plugin context, and return a restore function.
* `plugin.test.ts` uses it to inject a recording tracer without running `setup`.
@@ -528,12 +533,16 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P
const startBoundMs = timeoutSeconds * 1_000 + LIVENESS_START_TIMEOUT_MARGIN_MS;
if (sandbox.state === "error") {
if (sandbox.recoverable) {
await withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds));
await traceLeaseStep("lease.recover", () =>
withLivenessTimeout("sandbox.recover", startBoundMs, () => sandbox.recover(timeoutSeconds)),
leaseStateAttributes(sandbox.state));
return;
}
throw new Error(`Daytona sandbox ${sandbox.id} is in an unrecoverable error state: ${sandbox.errorReason ?? "unknown error"}`);
}
await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds));
await traceLeaseStep("lease.start", () =>
withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)),
leaseStateAttributes(sandbox.state));
}
function hasMissingSandboxContainer(sandbox: Sandbox): boolean {
@@ -1362,6 +1371,7 @@ const sandboxHandleActivityGates = (() => {
type SandboxLeaseAdmissionOptions = {
allowClosed?: boolean;
traceLeaseWait?: boolean;
};
const sandboxHandleLeaseAdmissionStates = (() => {
@@ -1402,7 +1412,8 @@ async function withSandboxActivityGate<T>(
const teardownGate = sandboxHandleTeardownGates.current(scope);
if (teardownGate) {
await teardownGate.promise;
if (options.traceLeaseWait) await traceLeaseStep("lease.wait_for_teardown", () => teardownGate.promise);
else await teardownGate.promise;
if (!options.allowClosed && sandboxHandleLeaseAdmissionStates.isClosed(scope)) {
throw new Error(`Daytona sandbox lease ${scope.providerLeaseId} is no longer active.`);
}
@@ -2682,39 +2693,39 @@ const plugin = definePlugin(trackDaytonaHooks({
return timeoutFailure();
};
const acquire = async (): Promise<PluginEnvironmentLease> => {
const sandbox = await createSandbox(params, config, {
const sandbox = await traceLeaseStep("lease.create", () => createSandbox(params, config, {
onCreateAttempt: (attempt) => { assertActive(); cleanup = attempt; },
onCreateFailure: observeFailure,
onSnapshotIdentity: identity => { snapshotIdentity = identity; },
});
}));
try {
assertActive();
if (cleanup) cleanup.observedProviderLeaseId = sandbox.id;
phase = "workspace";
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox);
const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox));
assertActive();
phase = "shell";
const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now())));
const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now()))));
assertActive();
// Configure a provider-side destroy time at or before a caller deadline, so
// an abandoned sandbox self-destroys even if Paperclip is down. The lease
// carries the real provider expiry (or none) as evidence of the bound.
phase = "expiry";
const expiresAt = await configureSandboxExpiry({
const expiresAt = await traceLeaseStep("lease.expiry", () => configureSandboxExpiry({
sandbox,
requestedExpiresAt: params.requestedExpiresAt,
nowMs: Date.now(),
preserveCreationExpiry: config.ttlMinutes != null,
});
}));
assertActive();
phase = "sentinel";
const workspaceSentinel = await writeWorkspaceSentinel({
const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => writeWorkspaceSentinel({
sandbox,
remoteCwd,
params,
config,
timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())),
});
}));
assertActive();
sandboxHandleLeaseAdmissionStates.open({
driverKey: params.driverKey,
@@ -2757,7 +2768,7 @@ const plugin = definePlugin(trackDaytonaHooks({
cleanupInProgress = true;
try {
const proof = idleActivity.proof(providerActivityKey(params, config, sandbox.id));
await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true);
await traceLeaseStep("lease.cleanup", () => sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now())), true));
idleActivity.confirm(proof);
} catch (cleanupError) {
if (cleanup) {
@@ -2821,7 +2832,7 @@ const plugin = definePlugin(trackDaytonaHooks({
throw new Error("The stopped Daytona sandbox is still settling cancelled work. Retry shortly.");
}
return await withSandboxActivityGate(scope, async () => {
const sandbox = await getSandboxOrNull(scope, { bypassTeardownGate: true });
const sandbox = await traceLeaseStep("lease.lookup", () => getSandboxOrNull(scope, { bypassTeardownGate: true }));
if (!sandbox) {
return { providerLeaseId: null, metadata: { expired: true } };
}
@@ -2831,7 +2842,7 @@ const plugin = definePlugin(trackDaytonaHooks({
// backup-guarded replacement path. Unknown errors preserve the lease.
if (hasMissingSandboxContainer(sandbox)) {
try {
await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData());
await traceLeaseStep("lease.refresh", () => withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()));
assertHandleMatchesLease(sandbox, params.providerLeaseId);
} catch (error) {
evictSandboxHandle(scope);
@@ -2859,48 +2870,50 @@ const plugin = definePlugin(trackDaytonaHooks({
// A stopped sandbox loses its pseudo-terminals, so a stored duplex channel
// is dead after a real restart. Close and drop every channel on this lease
// before the restart, so no stale channel id survives the resume.
await closeDaytonaDuplexChannelsForLease(params.providerLeaseId);
await traceLeaseStep("lease.channel_cleanup", () => closeDaytonaDuplexChannelsForLease(params.providerLeaseId));
}
const resumedFromState = sandbox.state ?? null;
await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs));
try {
const remoteCwd = await resolveSandboxWorkingDirectory(sandbox, false);
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await verifyWorkspaceSentinel({
sandbox,
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
});
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return {
providerLeaseId: null,
metadata: { expired: true, workspaceSentinel },
};
}
const shellCommand = await detectSandboxShellCommand(
sandbox,
toTimeoutSeconds(config.timeoutMs),
);
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
return await traceLeaseStep("lease.verify", async () => {
const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox, false));
// C3: a resumed lease must clear the workspace sentinel before it is
// trusted, even when the handle came from the cache. On any non-match we
// evict the cached handle and expire the lease so a stale/foreign sandbox
// is never reused on the subsequent (sentinel-skipping) exec path.
const workspaceSentinel = await traceLeaseStep("lease.sentinel", () => verifyWorkspaceSentinel({
sandbox,
shellCommand,
remoteCwd: requestedCwd ?? remoteCwd,
resumedLease: true,
resumedFromState,
sandboxState: "started",
workspaceSentinel,
}),
};
remoteCwd,
leaseMetadata: params.leaseMetadata,
timeoutSeconds: toTimeoutSeconds(config.timeoutMs),
}));
if (workspaceSentinel.result !== "matched") {
evictSandboxHandle(scope);
return {
providerLeaseId: null,
metadata: { expired: true, workspaceSentinel },
};
}
const shellCommand = await traceLeaseStep("lease.shell", () => detectSandboxShellCommand(
sandbox,
toTimeoutSeconds(config.timeoutMs),
));
sandboxHandleCache.markFresh(scope);
sandboxHandleLeaseAdmissionStates.open(scope);
return {
providerLeaseId: sandbox.id,
metadata: leaseMetadata({
config,
sandbox,
shellCommand,
remoteCwd: requestedCwd ?? remoteCwd,
resumedLease: true,
resumedFromState,
sandboxState: "started",
workspaceSentinel,
}),
};
});
} catch (error) {
evictSandboxHandle(scope);
// A timeout, rate limit, or provider 5xx does not prove this sandbox is
@@ -2910,7 +2923,7 @@ const plugin = definePlugin(trackDaytonaHooks({
throw error;
}
},
{ allowClosed: true },
{ allowClosed: true, traceLeaseWait: true },
);
},
packages/shared/src/constants.ts
+6
View File
@@ -198,6 +198,12 @@ export const ISSUE_STATUSES = [
] as const;
export type IssueStatus = (typeof ISSUE_STATUSES)[number];
/** Read-only task presentation. Monitoring never grants execution authority. */
export const ISSUE_DISPLAY_STATUSES = [
"backlog", "todo", "in_progress", "monitoring", "in_review", "done", "blocked", "cancelled",
] as const;
export type IssueDisplayStatus = (typeof ISSUE_DISPLAY_STATUSES)[number];
export const INBOX_MINE_ISSUE_STATUSES = [
"backlog",
"todo",
packages/shared/src/index.ts
+2
View File
@@ -364,6 +364,7 @@ export {
AGENT_ICON_NAMES,
PROJECT_ICON_NAMES,
ISSUE_STATUSES,
ISSUE_DISPLAY_STATUSES,
ISSUE_SUMMARIES_MAX_IDS,
INBOX_MINE_ISSUE_STATUSES,
INBOX_MINE_ISSUE_STATUS_FILTER,
@@ -562,6 +563,7 @@ export {
type AgentIconName,
type ProjectIconName,
type IssueStatus,
type IssueDisplayStatus,
type IssuePriority,
type IssueReviewPolicy,
type IssueWorkMode,
packages/shared/src/types/issue.ts
+6 -1
View File
@@ -20,6 +20,7 @@ import type {
IssueHarnessKind,
IssueOriginKind,
IssuePriority,
IssueDisplayStatus,
IssueReviewPolicy,
IssueRecoveryActionKind,
IssueRecoveryActionOutcome,
@@ -270,6 +271,7 @@ export interface AcceptedPlanDecompositionSummary extends AcceptedPlanDecomposit
}
export interface IssueRelationIssueSummary {
displayStatus?: IssueDisplayStatus;
id: string;
identifier: string | null;
title: string;
@@ -850,12 +852,14 @@ export type IssueChanges = Record<string, IssueChangeReceiptEntry>;
export type IssueSummary = Pick<
Issue,
"id" | "companyId" | "title" | "identifier" | "status" | "updatedAt"
"id" | "companyId" | "title" | "identifier" | "status" | "displayStatus" | "updatedAt"
> & {
externalConversationState: "active" | "waiting" | null;
};
export interface Issue {
/** Read-only status for task presentation; never use it for execution admission. */
displayStatus?: IssueDisplayStatus;
/** Collection projections omit task bodies and cannot seed the detail cache. */
listProjection?: "index" | "rows";
statusVersion?: number;
@@ -985,6 +989,7 @@ export type CompactIssue = Pick<
| "privacyRootIssueId"
| "title"
| "description"
| "displayStatus"
| "status"
| "workMode"
| "priority"
packages/shared/src/types/search.ts
+3 -1
View File
@@ -1,4 +1,4 @@
import type { IssuePriority, IssueStatus } from "../constants.js";
import type { IssueDisplayStatus, IssuePriority, IssueStatus } from "../constants.js";
export const COMPANY_SEARCH_SCOPES = ["all", "issues", "comments", "documents", "artifacts", "agents", "projects"] as const;
export type CompanySearchScope = (typeof COMPANY_SEARCH_SCOPES)[number];
@@ -38,6 +38,7 @@ export interface CompanySearchIssueSummary {
identifier: string | null;
title: string;
status: IssueStatus;
displayStatus?: IssueDisplayStatus;
externalConversationState?: "active" | "waiting" | null;
priority: IssuePriority;
assigneeAgentId: string | null;
@@ -76,6 +77,7 @@ export interface CompanySearchResult {
export interface CompanySearchFilterOptionCounts {
status: Partial<Record<IssueStatus, number>>;
displayStatus?: Partial<Record<IssueDisplayStatus, number>>;
priority: Partial<Record<IssuePriority, number>>;
assigneeAgentId: Record<string, number>;
assigneeUserId: Record<string, number>;
packages/shared/src/types/user-profile.ts
+2 -1
View File
@@ -1,4 +1,4 @@
import type { IssuePriority, IssueStatus } from "../constants.js";
import type { IssuePriority, IssueStatus, IssueDisplayStatus } from "../constants.js";
export interface UserProfileIdentity {
id: string;
@@ -42,6 +42,7 @@ export interface UserProfileIssueSummary {
identifier: string | null;
title: string;
status: IssueStatus;
displayStatus?: IssueDisplayStatus;
priority: IssuePriority;
assigneeAgentId: string | null;
assigneeUserId: string | null;
packages/shared/src/validators/search.ts
+4 -1
View File
@@ -1,5 +1,5 @@
import { z } from "zod";
import { ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
import { ISSUE_DISPLAY_STATUSES, ISSUE_PRIORITIES, ISSUE_STATUSES } from "../constants.js";
import { isUuidLike } from "../agent-url-key.js";
import {
COMPANY_SEARCH_EXTRACT_KINDS,
@@ -156,6 +156,9 @@ export const companySearchQuerySchema = z.object({
status: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "status", ISSUE_STATUSES)),
displayStatus: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "displayStatus", ISSUE_DISPLAY_STATUSES)),
priority: z.unknown()
.optional()
.transform((value, ctx) => parseEnumList(value, ctx, "priority", ISSUE_PRIORITIES)),
scripts/__tests__/release-verify-workflow.test.mjs
+1 -36
View File
@@ -579,8 +579,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
readWorkflow("runner-full-stack-e2e.yml"),
readWorkflow("e2e.yml"),
readWorkflow("runner-protocol-live-evals.yml"),
readWorkflow("chat-feature-testops.yml"),
readWorkflow("chat-completion-testops.yml"),
readWorkflow("chat-catalog-testops.yml"),
];
@@ -621,8 +619,6 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
"runner-full-stack-e2e.yml",
"runner-live-evals.yml",
"runner-protocol-live-evals.yml",
"chat-feature-testops.yml",
"chat-completion-testops.yml",
"chat-catalog-testops.yml",
];
const paidWorkflowNameSet = new Set(paidWorkflowNames);
@@ -705,38 +701,7 @@ test("Runner eval workflows pin actions and gate paid live execution", () => {
}
continue;
}
if (name === "chat-feature-testops.yml") {
assert.equal(providerJobs.length, 1);
const assertBoundary = (block) => {
assert.match(block, /^ chat:\n/);
assert.match(block, /\n needs: \[authorize, select, build\]\n/);
assert.match(block, /\n if: needs\.authorize\.outputs\.operation == 'evaluate'\n/);
assert.match(block, /\n environment: runner-e2e-private-paid\n/);
assert.match(block, /\n - \*checkout\n - \*node\n - \*guard\n/);
assert.match(block, /testops-chat-reporter\.mjs verify-selection chat-testops-selection\/testops-selection\.json/);
assert.match(block, /OPENROUTER_API_KEY: \$\{\{ secrets\.OPENROUTER_API_KEY \}\}/);
assert.match(block, /--id chat-features\.runner-opencode-deepseek-1017\.local\.create-backlog --max-parallel 1 --max-automatic-retries 0/);
assert.doesNotMatch(block, /ALLURE_TOKEN/);
assert.equal([...block.matchAll(providerSecretReference)].length, 1);
assert.ok(block.indexOf(" - *guard") < block.search(providerSecretReference));
assert.ok(block.indexOf("testops-chat-reporter.mjs verify-selection") < block.search(providerSecretReference));
};
const block = providerJobs[0];
assertBoundary(block);
assert.equal([...workflow.matchAll(providerSecretReference)].length, 1);
assert.match(workflow, /ref: \$\{\{ github\.sha \}\}/);
assert.match(workflow, /run: node tests\/runner-e2e\/testops-chat-dispatch\.ts/);
assert.match(workflow, /TESTOPS_ALLOWED_ACTOR_IDS: \$\{\{ vars\.RUNNER_E2E_ALLOWED_ACTOR_IDS \}\}/);
for (const mutant of [
block.replace("needs: [authorize, select, build]", "needs: [authorize, build]"),
block.replace("environment: runner-e2e-private-paid", "environment: unprotected"),
block.replace(" - *guard\n", ""),
block.replace("testops-chat-reporter.mjs verify-selection", "testops-chat-reporter.mjs unverified-selection"),
block.replace("OPENROUTER_API_KEY:", "ALLURE_TOKEN:"),
block.replace("--id chat-features.runner-opencode-deepseek-1017.local.create-backlog", "--id chat-features.runner-opencode.local.create-backlog"),
]) assert.throws(() => assertBoundary(mutant));
continue;
}
for (const block of providerJobs) {
if (name === "runner-protocol-live-evals.yml") {
assert.ok(block.includes("name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }}"));
scripts/benchmark-github-webhook-ingress.ts
+3 -3
View File
@@ -5,7 +5,7 @@ import { promises as fs, readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { applyPendingMigrations, chatActions, createDb, startEmbeddedPostgresTestDatabase, type Db } from "../packages/db/src/index.js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../packages/db/src/response-capture.js";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../server/src/__tests__/helpers/full-database-response-capture.js";
import { createGitHubWebhookIngress } from "../server/src/services/chat-channels/github/webhook-ingress.js";
import { captureIngressQueries, cleanupIngressFixture, githubIngressFixture, type CapturedQuery } from "../server/src/__tests__/helpers/github-webhook-ingress.js";
import { resolveExternalTestDatabaseUrl } from "../server/src/__tests__/helpers/external-test-database.js";
@@ -60,7 +60,7 @@ try {
queries.length = 0;
f.calls.length = 0;
const events: Array<Record<string, unknown>> = [], timestamp = Date.now();
const capture = iteration === 11 ? createDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined;
const capture = iteration === 11 ? createFullDatabaseResponseCapture({ start: timestamp - 1, end: timestamp + 59_000 }, event => events.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, { chatActions }, capture);
const start = performance.now();
const response = await service.processGitHubWebhookIngress(action.id);
@@ -80,7 +80,7 @@ try {
const [plan] = await db.$client.unsafe(`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ${candidate.sql}`, candidate.params as never[]);
const positionalRows = await db.$client.unsafe(candidate.sql, candidate.params as never[]).values();
measurements[name] = { queries: executed, initialQueryPlan: plan["QUERY PLAN"], initialReadJsonBytes: Buffer.byteLength(JSON.stringify(positionalRows)),
responseValueBytes: events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes,
responseValueBytes: events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes,
captureEvents: events };
}
}
server/scripts/benchmark-native-backup-publication.mjs
+196
View File
@@ -0,0 +1,196 @@
import assert from "node:assert/strict";
import { fork, execFileSync } from "node:child_process";
import { createServer, get, Agent } from "node:http";
import { promises as fs } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
import { setTimeout as delay } from "node:timers/promises";
const filename = fileURLToPath(import.meta.url);
const checkout = path.resolve(path.dirname(filename), "../..");
const arg = (name, fallback) => {
const position = process.argv.indexOf(`--${name}`);
return position < 0 ? fallback : process.argv[position + 1];
};
const percentile = (values, fraction) => {
const sorted = [...values].sort((a, b) => a - b);
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)] ?? null;
};
if (process.argv.includes("--worker")) {
const { publishNativeHarnessBackup } = await import(pathToFileURL(arg("module")));
let publishing = false;
const server = createServer((_request, response) => {
response.setHeader("x-publication-active", publishing ? "1" : "0");
response.end("ok");
});
await new Promise(resolve => server.listen(0, "127.0.0.1", resolve));
process.send({ kind: "ready", port: server.address().port });
process.on("message", async message => {
if (message.kind === "close") {
server.closeAllConnections();
server.close(() => process.disconnect());
return;
}
if (message.kind !== "publish") return;
const histogram = monitorEventLoopDelay({ resolution: 10 });
histogram.enable();
await delay(20);
const cpu = process.cpuUsage();
const started = performance.now();
publishing = true;
try {
await publishNativeHarnessBackup({
db: {}, companyId: "benchmark", scopeDigest: "fixture",
backupRoot: message.backupRoot, pendingRoot: message.pendingRoot,
stamp: async () => {},
});
const durationMs = performance.now() - started;
const cpuUsed = process.cpuUsage(cpu);
publishing = false;
await delay(20);
histogram.disable();
assert.equal(await fs.readFile(path.join(message.backupRoot, "current/manifest.json"), "utf8"), "replacement");
assert.deepEqual((await fs.readdir(message.backupRoot)).sort(), ["current"]);
assert.equal(await fs.readFile(path.join(message.liveRoot, "untouched.txt"), "utf8"), "live");
assert.equal((await fs.stat(message.liveRoot)).mode & 0o777, 0o555);
process.send({ kind: "result", durationMs, cpuMs: (cpuUsed.user + cpuUsed.system) / 1000,
eventLoopMaxMs: histogram.max / 1e6, workCompleted: true });
} catch (error) {
publishing = false;
process.send({ kind: "error", message: error.stack });
}
});
} else {
const baselineRef = arg("baseline-ref");
assert(baselineRef, "--baseline-ref is required");
const pairs = Number(arg("pairs", "3"));
const directories = Number(arg("directories", "100"));
const filesPerDirectory = Number(arg("files-per-directory", "100"));
const intervalMs = Number(arg("interval-ms", "5"));
for (const value of [pairs, directories, filesPerDirectory, intervalMs]) assert(Number.isSafeInteger(value) && value > 0);
const evidence = await fs.mkdtemp(path.join(tmpdir(), "native-backup-publication-proof-"));
const { build } = await import("esbuild");
const buildVariant = async variant => {
const outfile = path.join(evidence, `${variant}.mjs`);
await build({ absWorkingDir: checkout, entryPoints: ["server/src/services/native-runtime/native-harness-backup-publication.ts"],
outfile, bundle: true, platform: "node", format: "esm", target: "node24", sourcemap: true,
alias: { "@paperclipai/db/work-signals": path.join(checkout, "packages/db/src/work-signals.ts") },
plugins: variant === "baseline" ? [{ name: "pinned-source", setup(builder) {
builder.onLoad({ filter: /\.ts$/ }, args => {
const relative = path.relative(checkout, args.path);
if (relative.startsWith("..") || relative.includes("node_modules")) return;
return { contents: execFileSync("git", ["show", `${baselineRef}:${relative}`], { cwd: checkout, encoding: "utf8" }),
loader: "ts", resolveDir: path.dirname(args.path) };
});
} }] : [],
});
return outfile;
};
const modules = { baseline: await buildVariant("baseline"), treatment: await buildVariant("treatment") };
const makeFixture = async label => {
const root = await fs.mkdtemp(path.join(tmpdir(), `backup-publication-${label}-`));
const backupRoot = path.join(root, "backups");
const liveRoot = path.join(root, "live");
const pendingRoot = path.join(backupRoot, "pending");
await fs.mkdir(liveRoot);
await fs.writeFile(path.join(liveRoot, "untouched.txt"), "live");
await fs.chmod(liveRoot, 0o555);
for (const version of ["current", "previous"]) {
const versionRoot = path.join(backupRoot, version);
for (let directory = 0; directory < directories; directory++) {
const target = path.join(versionRoot, `skill-${directory}`);
await fs.mkdir(target, { recursive: true });
for (let file = 0; file < filesPerDirectory; file++) {
await fs.writeFile(path.join(target, `file-${file}.txt`), "retired instructions\n".repeat(4), { mode: 0o444 });
}
await fs.chmod(target, 0o555);
}
await fs.symlink(liveRoot, path.join(versionRoot, "outside"));
await fs.chmod(versionRoot, 0o555);
}
await fs.mkdir(pendingRoot);
await fs.writeFile(path.join(pendingRoot, "manifest.json"), "replacement");
return { root, backupRoot, liveRoot, pendingRoot };
};
const run = async variant => {
const fixture = await makeFixture(variant);
const worker = fork(filename, ["--worker", "--module", modules[variant]], { stdio: ["ignore", "inherit", "inherit", "ipc"] });
const exited = new Promise(resolve => worker.once("exit", resolve));
const nextMessage = kind => new Promise((resolve, reject) => {
const onMessage = message => {
if (message.kind !== kind && message.kind !== "error") return;
worker.off("message", onMessage); worker.off("exit", onExit);
if (message.kind === "error") reject(new Error(message.message)); else resolve(message);
};
const onExit = code => reject(new Error(`worker exited before ${kind}: ${code}`));
worker.on("message", onMessage); worker.once("exit", onExit);
});
const { port } = await nextMessage("ready");
const agent = new Agent({ keepAlive: true, maxSockets: 512 });
const requests = [];
let sent = 0;
const epoch = performance.now();
const dispatch = intended => {
const dispatched = performance.now();
requests.push(new Promise(resolve => {
const request = get({ hostname: "127.0.0.1", port, agent }, response => {
response.resume();
response.on("end", () => resolve({ intended, dispatchLateMs: dispatched - intended,
latencyMs: performance.now() - intended, active: response.headers["x-publication-active"] === "1", error: false }));
});
request.setTimeout(10_000, () => request.destroy(new Error("request timeout")));
request.on("error", () => resolve({ intended, dispatchLateMs: dispatched - intended,
latencyMs: performance.now() - intended, active: false, error: true }));
}));
};
let ticker;
const pump = () => {
const now = performance.now();
while (epoch + (sent + 1) * intervalMs <= now) dispatch(epoch + (++sent * intervalMs));
ticker = setTimeout(pump, Math.max(0, epoch + (sent + 1) * intervalMs - performance.now()));
};
ticker = setTimeout(pump, intervalMs);
try {
await delay(250);
const pending = nextMessage("result");
const start = performance.now();
worker.send({ kind: "publish", ...fixture });
const publication = await pending;
const end = performance.now();
await delay(250);
clearTimeout(ticker);
const completed = await Promise.all(requests);
const during = completed.filter(request => request.intended >= start && request.intended <= end);
assert(during.length > 0);
const result = { variant, ...publication, sent, completed: completed.length,
errors: completed.filter(request => request.error).length, duringRequests: during.length,
requestsServedDuringPublication: completed.filter(request => request.active).length,
foregroundP95Ms: percentile(during.map(request => request.latencyMs), 0.95),
foregroundMaxMs: Math.max(...during.map(request => request.latencyMs)),
generatorMaxLateMs: Math.max(...completed.map(request => request.dispatchLateMs)) };
console.log(JSON.stringify(result));
return result;
} finally {
clearTimeout(ticker); agent.destroy();
if (worker.connected) worker.send({ kind: "close" });
await exited;
await fs.chmod(fixture.liveRoot, 0o700);
if ((await fs.readdir(fixture.backupRoot)).join() === "current") await fs.rm(fixture.root, { recursive: true });
}
};
const measurements = [];
for (let pair = 0; pair < pairs; pair++) {
for (const variant of pair % 2 ? ["treatment", "baseline"] : ["baseline", "treatment"]) measurements.push(await run(variant));
}
const report = { baselineRef: execFileSync("git", ["rev-parse", baselineRef], { cwd: checkout, encoding: "utf8" }).trim(),
treatmentHead: execFileSync("git", ["rev-parse", "HEAD"], { cwd: checkout, encoding: "utf8" }).trim(),
treatmentDiff: execFileSync("git", ["diff", "HEAD", "--", "server/src/services/native-runtime/native-harness-backup-cleanup.ts", "server/src/services/native-runtime/native-harness-backup-publication.ts"], { cwd: checkout, encoding: "utf8" }),
node: process.version, platform: `${process.platform}/${process.arch}`, pairs, directories, filesPerDirectory,
filesRemovedPerPublication: 2 * directories * filesPerDirectory, intervalMs, evidence, measurements };
const output = arg("output", path.join(evidence, "result.json"));
await fs.writeFile(output, JSON.stringify(report, null, 2));
console.log(`Evidence: ${output}`);
}
server/scripts/benchmark-skill-list.ts
+3 -3
View File
@@ -16,7 +16,7 @@ import { authUsers, companies, companyMemberships, companySkills, companySkillSo
projectSkillSources, projects, applyPendingMigrations, closeRegisteredClients, createDb,
startEmbeddedPostgresTestDatabase } from "../../packages/db/src/index.js";
import * as schema from "../../packages/db/src/schema/index.js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "../../packages/db/src/response-capture.js";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../src/__tests__/helpers/full-database-response-capture.js";
import { withRunExecutionTiming } from "../src/services/execution-preparation-timing.js";
const argument = (name: string) => process.argv.find(value => value.startsWith(`${name}=`))?.slice(name.length + 1);
@@ -195,7 +195,7 @@ try {
};
const first = await request(); assert.ok(first.etag);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = captureValues ? createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined;
const capture = captureValues ? createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, schema, capture);
const samples = [];
try {
@@ -213,7 +213,7 @@ try {
} finally { capture?.flush(); }
assert.equal(await sourceHash(), measuredSourceHash, "Source changed during measurement");
const timings = samples.map(sample => sample.elapsedMs).sort((a, b) => a - b);
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1");
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1");
if (capture) assert.ok(summary && summary.inFlightAtEnd === 0 && summary.overflowCalls === 0);
const rowCount = (await db.select().from(companySkills).where(eq(companySkills.companyId, companyId))).length;
console.log(JSON.stringify({ schema: "paperclip.skill-list-benchmark.v1", sourceRevision, measuredSourceHash,
server/scripts/bundle-runtime.test.mjs
+36
View File
@@ -130,6 +130,42 @@ describe("bundleRuntime", () => {
}
});
it("attributes asynchronous database operations in a plain runtime bundle", async () => {
const captureModule = fileURLToPath(new URL("../../packages/db/src/response-capture.ts", import.meta.url));
write(path.join(root, "server", "dist", "services", "database-owner.js"), [
`import { withDatabaseOperation } from ${JSON.stringify(captureModule)};`,
'export const run = (owner, db) => withDatabaseOperation(owner, async () => {',
' await Promise.resolve();',
' return await db.$client.unsafe("select $1", ["private-parameter"]);',
'});',
].join("\n"));
write(path.join(root, "server", "dist", "index.js"), [
`import { createDatabaseResponseCapture, installDatabaseResponseCapture } from ${JSON.stringify(captureModule)};`,
'import { run } from "./services/database-owner.js";',
'const events = [];',
'const now = Date.now();',
'const capture = createDatabaseResponseCapture({ start: now, end: now + 60000 }, event => events.push(event));',
'const db = { $client: { unsafe: () => ({ then(resolve, reject) { return Promise.resolve(Object.assign([["private-result"]], { columns: [{ name: "value" }] })).then(resolve, reject); } }) } };',
'installDatabaseResponseCapture(db, {}, capture);',
'await Promise.all([run("fixture.first", db), run("fixture.second", db)]);',
'capture.flush();',
'console.log(JSON.stringify(events));',
].join("\n"));
const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir: path.join(root, "server", "dist-runtime") });
const env = { ...process.env };
delete env.NODE_OPTIONS;
delete env.PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW;
const { stdout } = await execFileAsync(process.execPath, [summary.outfile], { cwd: path.join(root, "server"), env, timeout: 30_000 });
const events = JSON.parse(stdout);
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v2");
expect(entries.map(entry => entry.owner).sort()).toEqual(["fixture.first", "fixture.second"]);
for (const entry of entries) expect(entry).toMatchObject({ calls: 1, rows: 1, callers: [] });
expect(events.at(-1)).toMatchObject({ event: "paperclip.db_response_capture.summary.v2", completedQueries: 2 });
expect(stdout).not.toContain("private-parameter");
expect(stdout).not.toContain("private-result");
expect(stdout).not.toContain(root);
});
it("emits a server module that loads without starting the server", async () => {
const outdir = path.join(root, "server", "dist-runtime");
const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir });
server/src/__tests__/agent-auth-identity-transfer.test.ts
+3 -3
View File
@@ -4,7 +4,7 @@ import request from "supertest";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, heartbeatRunEvents, heartbeatRuns, issueComments, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createLocalAgentJwt } from "../agent-auth-jwt.js";
import { actorMiddleware } from "../middleware/auth.js";
@@ -53,7 +53,7 @@ const support = await getEmbeddedPostgresTestSupport();
it("bounds actual middleware responses and preserves the complete stored run", async () => {
const fixture = await seed();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const response = await request(fixture.app).get("/actor").set("Authorization", `Bearer ${fixture.token}`);
@@ -61,7 +61,7 @@ const support = await getEmbeddedPostgresTestSupport();
expect(response.body).toMatchObject({ type: "agent", runId: fixture.runId,
identityContextId: fixture.identity.id, onBehalfOfUserId: "owner" });
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(3);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(16_384);
server/src/__tests__/agent-directory-working-copies.test.ts
+87 -1
View File
@@ -16,7 +16,7 @@ import { createHash, randomUUID } from "node:crypto";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { agentFileStore, fileHash, inspectAgentFile, inspectAgentDirectory, snapshotAgentFiles, MAX_AGENT_FILE_BYTES, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENTRIES } from "../services/agent-file-store.js";
import { agents, companies, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { agents, companies, issues, authUsers, companyMemberships, principalPermissionGrants, heartbeatRuns, environmentLeases, environments, agentInstructionWorkingCopies, agentInstructionRevisions, agentInstructionHeads, createDb, fileStorageDb } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js";
import { agentInstructionWorkingCopyService, instructionWorkingCopyGuidance } from "../services/agent-instruction-working-copies.js";
@@ -24,6 +24,7 @@ import { resolveManagedInstructionsRoot } from "../services/agent-instructions.j
import { buildNativeRuntimeContext } from "../services/native-runtime/runtime-context.js";
import type { EnvironmentRuntimeService } from "../services/environment-runtime.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { createReusableLeaseHandoff } from "../services/environment-lease-handoff.js";
import { AgentDirectoryReuseInvalidatedError, agentDirectoryWorkingCopyService } from "../services/agent-directory-working-copies.js";
import { withDirectoryMergeLock } from "@paperclipai/adapter-utils/workspace-restore-merge";
import { heartbeatRunEvents } from "@paperclipai/db";
@@ -123,6 +124,44 @@ describe("persistent agent directories", () => {
for (const copy of [active, completed, transferred]) expect((await fs.stat(copy.localRoot)).isDirectory()).toBe(true);
} finally { await worker.stop(); }
});
it("retires a stale retry on an already cleaned receipt after restart without filesystem work", async () => {
const copy = await run();
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: new Date(), receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
clock(); const worker = coordinator(), removes = vi.spyOn(fs, "rm");
try {
await agentInstructionWorkingCopyService(db).registerDirectoryCleanupWork(worker, () => true, async () => true).ready;
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(100);
expect((await copies.get(companyId, copy.runId))?.nextAttemptAt).toBeNull();
});
await expect.poll(() => idleWorkSnapshot().active).toBe(0);
const reads = vi.spyOn(db, "select");
await vi.advanceTimersByTimeAsync(120_000);
expect(reads).not.toHaveBeenCalled(); expect(removes).not.toHaveBeenCalled();
expect(worker.nextWakeAt()).toBeNull();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { await worker.stop(); }
});
it("honors a cleanup obligation published before stale-deadline settlement", async () => {
const copy = await run(), dueAt = new Date(Date.now() + 30_000);
await db.update(agentInstructionWorkingCopies).set({ state: "unavailable", processStoppedAt: new Date(),
nextAttemptAt: dueAt, receipt: { schema: "paperclip.agent-files.v1", cleanupPending: false } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const transact = db.transaction.bind(db);
const race = vi.spyOn(db, "transaction").mockImplementationOnce(async (callback, config) => {
await db.update(agentInstructionWorkingCopies).set({ receipt: { ...copy.receipt, cleanupPending: true } })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
return transact(callback, config);
});
try {
await copies.release(companyId, copy.runId);
expect(await copies.get(companyId, copy.runId)).toMatchObject({ receipt: { cleanupPending: false }, nextAttemptAt: null });
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
} finally { race.mockRestore(); }
});
it("reactivates from real collection, preserves saved files, retries failed cleanup, and becomes quiet", async () => {
const copy = await run(); await fs.writeFile(path.join(copy.localRoot, "memory.txt"), "saved memory");
clock(); const worker = coordinator();
@@ -1461,6 +1500,53 @@ describe("persistent agent directories", () => {
return { copy: (await copies.get(companyId, copy.runId))!, lease, environmentId };
}
it.each(["destroyed", "stopped", "broken_handoff", "extra_owner", "foreign_company"] as const)(
"recovers an earlier remote copy only from the exact destroyed successor chain (%s)", async scenario => {
const { copy, lease } = await unavailableRemote();
const issueId = randomUUID(), successorRunId = randomUUID(), successorId = randomUUID();
const acquiredAt = new Date(Date.now() - 2000), handoffAt = new Date(Date.now() - 1000);
await db.insert(issues).values({ id: issueId, companyId, title: "Completed retained workspace", status: "done" });
await db.insert(heartbeatRuns).values({ id: successorRunId, companyId, agentId, invocationSource: "on_demand", status: "succeeded" });
const [previous] = await db.update(environmentLeases).set({ status: "expired", leasePolicy: "reuse_by_environment",
issueId, acquiredAt, releasedAt: handoffAt, cleanupStatus: "success", metadata: { agentId } })
.where(eq(environmentLeases.id, lease.id)).returning();
const successor = { ...previous!, id: successorId, heartbeatRunId: successorRunId, acquiredAt: handoffAt,
releasedAt: new Date(), metadata: { agentId } };
if (scenario === "foreign_company") {
const foreignCompanyId = randomUUID();
await db.insert(companies).values({ id: foreignCompanyId, name: "Unrelated owner", issuePrefix: randomUUID().slice(0, 8) });
successor.companyId = foreignCompanyId;
}
const handoff = createReusableLeaseHandoff(previous!, successor);
const termination = remoteTerminationReceipt(successor, { providerLeaseId: lease.providerLeaseId,
state: scenario === "stopped" ? "stopped" : "destroyed" });
await db.update(environmentLeases).set({ metadata: { agentId, reusableLeaseSuccessor: handoff } })
.where(eq(environmentLeases.id, lease.id));
await db.insert(environmentLeases).values({ ...successor, metadata: { agentId, remoteExecutionTermination: termination,
reusableLeasePredecessor: scenario === "broken_handoff" ? { ...handoff, handoffId: randomUUID() } : handoff } });
if (scenario === "extra_owner") await db.insert(environmentLeases).values({ ...lease, id: randomUUID(), status: "active" });
await db.update(agentInstructionWorkingCopies).set({ state: "pending_collection", attempts: 0, candidateHash: null, candidateBase64: null })
.where(eq(agentInstructionWorkingCopies.runId, copy.runId));
const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand").mockRejectedValue(new Error("must not execute"));
try {
await copies.recoverStopped();
const recovered = (await copies.get(companyId, copy.runId))!;
if (scenario === "destroyed") {
expect(recovered).toMatchObject({ state: "unavailable", errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE",
receipt: { cleanupPending: false }, nextAttemptAt: null });
expect(recovered.processStoppedAt).toBeInstanceOf(Date);
await expect(fs.stat(copy.localRoot)).rejects.toMatchObject({ code: "ENOENT" });
} else {
expect(recovered).toMatchObject({ state: "pending_collection", processStoppedAt: null });
expect(await fs.readFile(path.join(copy.localRoot, entryFile), "utf8")).toBe(initial);
}
expect(shell).not.toHaveBeenCalled();
expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe(initial);
const [historical] = await db.select().from(environmentLeases).where(eq(environmentLeases.id, lease.id));
expect(historical!.metadata).not.toHaveProperty("remoteExecutionTermination");
} finally { shell.mockRestore(); }
});
it.each([undefined, "stopped", "companyId", "runId", "leaseId", "provider", "providerLeaseId"])(
"defers unavailable cleanup without exact destruction proof (%s)", async proof => {
const { copy } = await unavailableRemote({ state: proof === undefined ? undefined : proof === "stopped" ? "stopped" : "destroyed",
server/src/__tests__/agent-record-read-projections.test.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { agents, companies, costEvents, createDb } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -61,7 +61,7 @@ describe.skipIf(!support.supported)("agent roster read projections", () => {
async function observe<T>(run: (observed: ReturnType<typeof createDb>) => Promise<T>) {
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agents, costEvents }, capture);
try {
const result = await run(observed);
server/src/__tests__/attention-service.test.ts
+7 -7
View File
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import {
agents,
@@ -1164,13 +1164,13 @@ describeEmbeddedPostgres("attention service", () => {
})));
}
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, { userId: "board-user" });
expect(feed.items.filter(item => item.sourceKind === "failed_run").map(item => item.subject.id)).toEqual([failedId]);
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(64);
@@ -1209,7 +1209,7 @@ describeEmbeddedPostgres("attention service", () => {
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const feed = await attentionService(db).list(companyId, {
@@ -1218,7 +1218,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(feed.items.filter((item) => item.sourceKind === "failed_run").map((item) => item.subject.id).sort())
.toEqual([...keptIds].sort());
} finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeLessThan(4);
@@ -1407,7 +1407,7 @@ describeEmbeddedPostgres("attention service", () => {
const interactionItems = feed.items.filter((item) => item.sourceKind === "issue_thread_interaction");
const detailsByKind = new Map(interactionItems.map((item) => [item.detail?.kind, item]));
const summaryEvents: Array<Record<string, unknown>> = [], captureNow = Date.now();
const summaryCapture = createDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
const summaryCapture = createFullDatabaseResponseCapture({ start: captureNow - 100, end: captureNow + 59_000 }, event => summaryEvents.push(event));
installDatabaseResponseCapture(db, { documents, assets, issueAttachments }, summaryCapture);
try {
const summary = await attentionService(db).summary(companyId, { userId: "board-user" });
@@ -1418,7 +1418,7 @@ describeEmbeddedPostgres("attention service", () => {
expect(index.totalCount).toBe(feed.totalCount);
} finally { summaryCapture.flush(); }
const cardReads = summaryEvents.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const cardReads = summaryEvents.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["documents", "assets", "issue_attachments"].includes(String(event.table)));
expect(cardReads).toEqual([]);
server/src/__tests__/chat-channels.integration.test.ts
+235 -11
View File
@@ -33,11 +33,13 @@ import { budgetService } from "../services/budgets.js";
import { idleWorkSnapshot } from "../services/task-admission.js";
import { readFreshWorkspaceReadiness } from "../services/fresh-workspace-continuation.js";
import { requestTaskRetry, reconcileTaskChatRetryReceipts } from "../services/task-retry.js";
import { taskInterruptionPresentation } from "../services/task-interruption-presentation.js";
import { withQueuedCommentIdsInWakePayload } from "../services/issue-queued-comment-queue.js";
import { createSystemRecovery } from "../services/system-recovery-state.js";
import { remoteTerminationReceipt } from "../services/remote-execution-termination.js";
import { stageReceiptReactionRemovals } from "../services/chat-receipt-reactions.js";
import type { Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { githubResponseAuthorityFields } from "../services/chat-channels/github/response-authority-read.js";
import * as activityLogModule from "../services/activity-log.js";
import * as connectionCredentialBindings from "../services/connection-credential-bindings.js";
@@ -112,6 +114,7 @@ import {
isNotNull,
isNull,
like,
ne,
notInArray,
or,
sql,
@@ -2887,7 +2890,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const normalizedEvent = { ...delivery.normalizedEvent, unusedContent: bulk };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatMessageLinks, chatDeliveries, chatEndpoints, chatConversations,
chatEndpointResources, chatGitHubConfigurations, chatExternalPrincipals, chatIdentityLinks,
companyMemberships, chatActions, heartbeatRuns }, capture);
@@ -2916,7 +2919,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
} };
await db.update(chatDeliveries).set({ normalizedEvent }).where(eq(chatDeliveries.id, delivery.id));
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatDeliveries }, capture);
let projected: { event: Record<string, unknown> };
try {
@@ -7152,6 +7155,68 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
expect(await readChatConversationsByEndpointId(f.endpoint.id)).toHaveLength(0);
expect(await readChatGitHubReviewsByEndpointId(f.endpoint.id)).toHaveLength(0);
});
it("offers Retry for a saved automatic PR event after a successful older review", async () => {
const f = await reviewBotFixture();
const payload = {
action: "opened", installation: { id: 2468 },
repository: { id: 97531, full_name: "paperclipai/paperclip", name: "paperclip", owner: { id: 1357, login: "paperclipai" } },
sender: { id: 77, login: "maintainer" },
pull_request: { number: 81, title: "Review successive heads", body: "", draft: false,
base: { sha: "a".repeat(40), ref: "master" }, head: { sha: "b".repeat(40) },
user: { id: 42, login: "octocat", type: "User" }, labels: [] },
};
const send = async (event: typeof payload) => {
await f.service.handleWebhook(f.endpoint.publicId, "github", signedGitHubWebhookRequest({
event: "pull_request", delivery: randomUUID(), payload: event, webhookSecret: f.webhookSecret,
}));
await f.service.processPendingDeliveries();
};
await send(payload);
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await readChatConversationsByEndpointId(f.endpoint.id)).length;
}, { timeout: 10_000 }).toBe(1);
const [conversation] = await readChatConversationsByEndpointId(f.endpoint.id);
const [task] = await readIssuesById(conversation.issueId!);
const [original] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")));
await expect.poll(async () => (await readAgentWakeupRequestsById(original.id)).length).toBe(1);
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId,
issueId: task.id, status: "succeeded", finishedAt: new Date(), wakeupRequestId: original.id,
contextSnapshot: { issueId: task.id, taskKey: task.identifier, source: "chat:github", wakeCommentIds: [original.payload.commentId] },
}).returning();
await db.update(agentWakeupRequests).set({ status: "completed", runId: run.id }).where(eq(agentWakeupRequests.id, original.id));
await send({ ...payload, action: "synchronize", pull_request: { ...payload.pull_request, head: { sha: "c".repeat(40) } } });
await expect.poll(async () => {
await f.service.processPendingDeliveries();
return (await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup")))).length;
}).toBe(2);
const [pending] = await db.select().from(chatActions).where(and(
eq(chatActions.conversationId, conversation.id), eq(chatActions.kind, "inbound_wakeup"), ne(chatActions.id, original.id)));
await expect.poll(async () => (await readAgentWakeupRequestsById(pending.id)).length).toBe(1);
const [receipt] = await readAgentWakeupRequestsById(pending.id);
await db.update(agentWakeupRequests).set({ status: "deferred_issue_execution", runId: null,
payload: withQueuedCommentIdsInWakePayload(receipt.payload, [String(pending.payload.commentId)]),
requestedAt: new Date(Date.now() - 60_000) }).where(eq(agentWakeupRequests.id, pending.id));
expect(receipt.requestedByActorType).toBe("user");
await db.update(issues).set({ status: "blocked", executionRunId: null }).where(eq(issues.id, task.id));
expect(await taskInterruptionPresentation(db, { ...task, status: "blocked" }))
.toMatchObject({ state: "failed", canRetry: true, intentId: pending.id, observedRunId: run.id });
await db.update(agents).set({ status: "running", runtimeConfig: { heartbeat: { wakeOnDemand: true, maxConcurrentRuns: 1 } } })
.where(eq(agents.id, f.assignedAgentId));
await db.insert(heartbeatRuns).values({ companyId: f.companyId, agentId: f.assignedAgentId, status: "running", startedAt: new Date() });
f.wakeup.mockImplementation(heartbeatService(db).wakeup);
const retry = await requestTaskRetry({ db, companyId: f.companyId, issueId: task.id, actorId: "owner-user",
actorSource: "cloud_tenant", observedRunId: run.id, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry });
await reconcileTaskChatRetryReceipts(db, { companyId: f.companyId });
const [resumed] = await readAgentWakeupRequestsById(pending.id);
expect(resumed.runId).toBeTruthy();
expect((await readHeartbeatRunsById(resumed.runId!))[0]).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [pending.payload.commentId] } });
expect((await readAgentWakeupRequestsById(retry.requestId!))[0]).toMatchObject({ status: "coalesced", runId: resumed.runId });
});
it("admits signed PR events into an ordinary assigned task and deduplicates delivery", async () => {
const f = await reviewBotFixture();
const { buildSkillMentionHref } = await import("@paperclipai/shared");
@@ -62866,7 +62931,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
const commentsBeforeDisabledLifecycle = await db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId));
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id);
const priorLifecycleIds = new Set(
(
await db
@@ -62946,7 +63012,8 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
db
.select({ id: issueComments.id })
.from(issueComments)
.where(eq(issueComments.issueId, conversation.issueId)),
.where(eq(issueComments.issueId, conversation.issueId))
.orderBy(issueComments.id),
).resolves.toEqual(commentsBeforeDisabledLifecycle);
await service.shutdown();
});
@@ -64140,6 +64207,163 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
return { ...f, pending, originalDelivery };
}
async function successfulGitHubQueuedInputFixture() {
const f = await currentGitHubTaskRetryFixture(true);
const companyId = f.fixture.companyId, issueId = f.issue.id;
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
await db.update(agentWakeupRequests).set({ payload: withQueuedCommentIdsInWakePayload(saved.payload,
[String(f.pending!.payload.commentId)]) }).where(eq(agentWakeupRequests.id, saved.id));
const contractId = randomUUID(), resultId = randomUUID();
await db.insert(completionContracts).values({ id: contractId, companyId, issueId, revision: 1,
schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test", risk: "standard",
completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal", contractJson: {},
canonicalSha256: contractId, createdByActorType: "system", createdByActorId: "test" });
await db.update(heartbeatRuns).set({ status: "succeeded", errorCode: null, nativePhase: "committed",
completionContractId: contractId }).where(eq(heartbeatRuns.id, f.runId));
const [result] = await db.insert(nativeRunResults).values({ id: resultId, companyId, issueId, runId: f.runId,
completionContractId: contractId, serverFingerprint: resultId, schemaStatus: "accepted", canonicalSha256: resultId,
resultJson: { terminal: { runTerminalState: "succeeded" }, summary: "Earlier review remains saved" } }).returning();
await db.update(nativeRunFinalizations).set({ phase: "committed", resultId }).where(eq(nativeRunFinalizations.runId, f.runId));
await db.update(agentWakeupRequests).set({ status: "completed" }).where(eq(agentWakeupRequests.id, f.receipt.id));
const input = { db, companyId, issueId, actorId: "owner-user", actorSource: "cloud_tenant",
observedRunId: f.runId, idempotencyKey: randomUUID(), prepareChatRetry: f.service.prepareFailedChatRunRetry };
return { ...f, input, result };
}
it("retries saved GitHub input after a successful run without replaying that run", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const { input } = f;
const companyId = f.fixture.companyId;
const before = (await readHeartbeatRunsById(f.runId))[0];
const [request, duplicate] = await Promise.all([requestTaskRetry(input),
requestTaskRetry({ ...input, idempotencyKey: randomUUID() })]);
expect(request.accepted).toBe(true);
expect(duplicate.requestId).toBe(request.requestId);
await reconcileTaskChatRetryReceipts(db, { companyId });
const [source] = await readAgentWakeupRequestsById(f.pending!.id);
expect(source.runId).toBeTruthy();
const [successor] = await readHeartbeatRunsById(source.runId!);
expect(successor).toMatchObject({ status: "queued", retryOfRunId: null,
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(request.requestId!))[0])
.toMatchObject({ status: "coalesced", runId: successor.id });
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["connection_revoked", "source_rebound", "task_completed", "agent_paused", "cleanup_pending"] as const)(
"keeps saved-input Retry behind current admission gates: %s", async gate => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const request = await requestTaskRetry(f.input);
const before = (await readHeartbeatRunsById(f.runId))[0];
if (gate === "connection_revoked") await db.update(chatEndpoints).set({ status: "paused" }).where(eq(chatEndpoints.id, f.endpoint.id));
if (gate === "source_rebound") await db.update(chatActions).set({ payload: { ...f.pending!.payload, commentId: randomUUID() } })
.where(eq(chatActions.id, f.pending!.id));
if (gate === "task_completed") await db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issue.id));
if (gate === "agent_paused") await db.update(agents).set({ status: "paused" }).where(eq(agents.id, f.fixture.assignedAgentId));
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
if (gate === "cleanup_pending") {
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
}
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
expect((await readHeartbeatRunsByCompanyId(companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(0);
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
if (gate === "cleanup_pending") {
expect((await readAgentWakeupRequestsById(request.requestId!))[0].status).toBe("deferred_issue_execution");
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
}
} finally { await f.service.shutdown(); }
});
it.each(["issue.comment.reopen", "boardMessageRetry"] as const)(
"resumes saved GitHub input after a successful board turn: %s", async origin => {
const f = await successfulGitHubQueuedInputFixture();
try {
const [previous] = await readHeartbeatRunsById(f.runId);
await db.update(heartbeatRuns).set({ responsibleUserId: f.input.actorId, contextSnapshot: {
...previous.contextSnapshot, ...(origin === "boardMessageRetry" ? { boardMessageRetry: true } : { source: origin }),
} }).where(eq(heartbeatRuns.id, f.runId));
const before = (await readHeartbeatRunsById(f.runId))[0];
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId: f.fixture.companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ retryOfRunId: null,
contextSnapshot: { source: "chat:github", wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readHeartbeatRunsById(f.runId))[0]).toEqual(before);
} finally { await f.service.shutdown(); }
});
it("keeps a board message queued separately when saved GitHub Retry resumes after cleanup", async () => {
const f = await successfulGitHubQueuedInputFixture();
try {
const companyId = f.fixture.companyId;
const lease = { id: randomUUID(), companyId, issueId: f.issue.id, heartbeatRunId: f.runId,
provider: "daytona", providerLeaseId: randomUUID(), leasePolicy: "ephemeral" };
await db.insert(environmentLeases).values({ ...lease, status: "pending_cleanup" });
await db.update(issues).set({ status: "in_review", executionRunId: f.runId }).where(eq(issues.id, f.issue.id));
const retry = await requestTaskRetry(f.input);
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
expect((await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeNull();
const [comment] = await db.insert(issueComments).values({ companyId, issueId: f.issue.id,
authorType: "user", authorUserId: f.input.actorId, body: "A separate board follow-up" }).returning();
const [board] = await db.insert(agentWakeupRequests).values({ companyId, agentId: f.fixture.assignedAgentId,
source: "automation", triggerDetail: "system", reason: "issue_commented", status: "deferred_issue_execution",
requestedByActorType: "user", requestedByActorId: f.input.actorId,
payload: withQueuedCommentIdsInWakePayload({ issueId: f.issue.id, commentId: comment.id,
_paperclipWakeContext: { issueId: f.issue.id, source: "issue.comment.reopen", wakeReason: "issue_commented" },
}, [comment.id]),
}).returning();
await db.update(environmentLeases).set({ status: "released", cleanupStatus: "success", releasedAt: new Date(),
metadata: { remoteExecutionTermination: remoteTerminationReceipt(lease, { providerLeaseId: lease.providerLeaseId, state: "destroyed" }) },
}).where(eq(environmentLeases.id, lease.id));
await reconcileTaskChatRetryReceipts(db, { companyId, requestId: retry.requestId! });
const [saved] = await readAgentWakeupRequestsById(f.pending!.id);
expect(saved.runId).toBeTruthy();
expect((await readHeartbeatRunsById(saved.runId!))[0]).toMatchObject({ status: "queued",
contextSnapshot: { wakeCommentIds: [f.pending!.payload.commentId] } });
expect((await readAgentWakeupRequestsById(board.id))[0]).toEqual(board);
} finally { await f.service.shutdown(); }
});
it("dispatches saved input through the task Retry HTTP endpoint and survives a repeated click", async () => {
const f = await successfulGitHubQueuedInputFixture();
const { issueRoutes } = await import("../routes/issues.js");
const app = express();
app.use(express.json());
app.use((req, _res, next) => { req.actor = boardActor(f.fixture.companyId); next(); });
app.use("/api", issueRoutes(db, createStorageService().storage, { chatRunRetries: f.service }));
app.use(errorHandler);
try {
const body = { observedRunId: f.runId, idempotencyKey: randomUUID() };
const first = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(first.status, JSON.stringify(first.body)).toBe(202);
await expect.poll(async () => (await readAgentWakeupRequestsById(f.pending!.id))[0].runId).toBeTruthy();
const second = await request(app).post(`/api/issues/${f.issue.id}/retry`).send(body);
expect(second.status).toBe(202);
expect(second.body.requestId).toBe(first.body.requestId);
expect((await readHeartbeatRunsByCompanyId(f.fixture.companyId)).filter(run => run.wakeupRequestId === f.pending!.id)).toHaveLength(1);
const detail = await request(app).get(`/api/issues/${f.issue.id}`);
expect(detail.status).toBe(200);
expect(detail.body.interruption).toMatchObject({ state: "working", canRetry: false });
expect((await db.select().from(nativeRunResults).where(eq(nativeRunResults.id, f.result.id)))[0]).toEqual(f.result);
} finally { await f.service.shutdown(); }
});
it.each(["reconnected", "newer_input", "reconnected_newer_input", "snapshot_newer_input"] as const)(
"admits current GitHub task Retry without replaying obsolete input: %s", async kind => {
const f = await currentGitHubTaskRetryFixture(kind.includes("newer_input"));
@@ -64269,7 +64493,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
status: "active", membershipRole: "operator" });
}
const queryEvents: Record<string, unknown>[] = [];
const capture = mode === "backlog_51" ? createDatabaseResponseCapture({ start: Date.now() - 1,
const capture = mode === "backlog_51" ? createFullDatabaseResponseCapture({ start: Date.now() - 1,
end: Date.now() + 59_000 }, event => queryEvents.push(event)) : undefined;
if (capture) installDatabaseResponseCapture(db, { chatActions, chatDeliveries, chatMessageLinks, issueComments,
agentWakeupRequests, chatEndpoints, chatConversations }, capture);
@@ -64289,7 +64513,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
idempotencyKey: mode === "concurrent_clicks" ? randomUUID() : input.idempotencyKey })]);
if (capture) {
capture.flush();
expect(Number(queryEvents.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.startedQueries))
expect(Number(queryEvents.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.startedQueries))
.toBeLessThan(180);
}
expect(two.requestId).toBe(one.requestId);
@@ -67489,7 +67713,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.issue.id };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -67506,7 +67730,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
@@ -77214,7 +77438,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
})));
const scope = { companyId: context.fixture.companyId, issueId: context.conversation.issueId };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions, agentWakeupRequests }, capture);
let scopes: Array<{ companyId: string; issueId: string }>;
try {
@@ -77231,7 +77455,7 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
}
expect(events.at(-1)).toMatchObject({ startedQueries: mode === "task" ? 0 : 1,
completedQueries: mode === "task" ? 0 : 1, inFlightAtEnd: 0 });
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1")
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1")
.reduce((count, event) => count + Number(event.rows), 0)).toBe(mode === "task" ? 0 : scopes.length);
} finally {
await context.service.shutdown(); await other.service.shutdown();
server/src/__tests__/company-skills-service.test.ts
+10 -10
View File
@@ -17,7 +17,7 @@ import {
projectWorkspaces,
} from "@paperclipai/db";
import { parseFrontmatterMarkdown } from "@paperclipai/shared";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
getEmbeddedPostgresTestSupport,
startEmbeddedPostgresTestDatabase,
@@ -651,17 +651,17 @@ describeEmbeddedPostgres("companySkillService.list", () => {
]);
await svc.list(companyId);
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { agents, costEvents }, capture);
let listed: Awaited<ReturnType<typeof svc.list>>;
try { listed = await svc.list(companyId); } finally { capture.flush(); }
expect(listed.find(item => item.id === first.id)?.attachedAgentCount).toBe(2);
expect(listed.find(item => item.id === second.id)?.attachedAgentCount).toBe(1);
expect(listed.find(item => item.id === ambiguous.id)?.attachedAgentCount).toBe(0);
const agentReads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "agents");
const agentReads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "agents");
expect(agentReads).toHaveLength(1);
expect(Number(agentReads[0]?.responseValueBytes)).toBeLessThan(2_000);
expect(events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
expect(events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "cost_events")).toHaveLength(0);
await db.update(agents).set({ adapterConfig: { paperclipSkillSync: { desiredSkills: [second.key] } } }).where(eq(agents.id, readerId));
const refreshed = await svc.list(companyId, { q: "First" });
expect(refreshed.find(item => item.id === first.id)?.attachedAgentCount).toBe(1);
@@ -680,12 +680,12 @@ describeEmbeddedPostgres("companySkillService.list", () => {
const before = await svc.list(companyId);
expect(before.find(item => item.id === inserted[0]!.id)?.folderPath).toBe("bundled/paperclip-core");
const events: Record<string, unknown>[] = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkills }, capture);
let after: Awaited<ReturnType<typeof svc.list>>;
try { after = await svc.list(companyId); } finally { capture.flush(); }
expect(after).toEqual(before);
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "company_skills");
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "company_skills");
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(950_000);
const stored = await svc.getById(companyId, inserted[0]!.id);
expect(stored?.markdown).toBe(savedMarkdown);
@@ -762,11 +762,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
.orderBy(companySkillVersions.id);
expect(before.length).toBeGreaterThan(0);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db).list(companyId)).toEqual(expected); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "company_skill_versions" && event.operation === "select");
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.responseValueBytes), 0)).toBeLessThan(4096);
@@ -783,11 +783,11 @@ describeEmbeddedPostgres("companySkillService.list", () => {
? { type, companyId: otherCompanyId, agentId: randomUUID() }
: { type, companyIds: [otherCompanyId] };
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { companySkillVersions }, capture);
try { expect(await companySkillService(db, { actor }).list(companyId)).toEqual([]); }
finally { capture.flush(); }
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.completedQueries).toBe(0);
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.completedQueries).toBe(0);
});
it("restores a deleted release after warm validation", async () => {
server/src/__tests__/conversation-settlement-transfer.test.ts
+3 -3
View File
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb, heartbeatRuns, issueComments, issueThreadInteractions, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { settleConversationTurnForRun } from "../services/agent-conversations.js";
@@ -52,11 +52,11 @@ const support = await getEmbeddedPostgresTestSupport();
it.each(["response", "interaction", "reset"] as const)("bounds %s settlement reads and retains all saved text and run payloads", async reply => {
const input = await seed(reply), before = await sourceDigests(input);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns, issues, issueComments }, capture);
try { expect(await settleConversationTurnForRun(db, input.companyId, input.runId)).toBe(true); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& ["heartbeat_runs", "issues", "issue_comments"].includes(String(event.table))
&& (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThanOrEqual(4);
server/src/__tests__/database-operation-attribution.test.ts
+137
View File
@@ -0,0 +1,137 @@
import express from "express";
import request from "supertest";
import { QueryPromise } from "drizzle-orm/query-promise";
import { afterEach, expect, it, vi } from "vitest";
import { withDatabaseOperation } from "@paperclipai/db";
import { installDatabaseWorkSignals } from "@paperclipai/db/work-signals";
import { databaseOperationMiddleware } from "../middleware/database-operation.js";
import { withBudgetSpan } from "../services/budget-tracing.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { DELIVERY_QUEUES, notifyDeliveryWork } from "../services/delivery-work-notifications.js";
import {
createRunPreparationTiming,
measureRunPhase,
withRunExecutionTiming,
withRunResponsePresentationTiming,
withRunWorkspaceFinalizationTiming,
} from "../services/execution-preparation-timing.js";
const { owners } = await vi.hoisted(async () => {
const { AsyncLocalStorage } = await import("node:async_hooks");
return { owners: new AsyncLocalStorage<string>() };
});
vi.mock("@paperclipai/db", async importOriginal => ({
...await importOriginal<typeof import("@paperclipai/db")>(),
withDatabaseOperation: <T>(owner: string, action: () => T): T => owners.run(owner, action),
}));
class OwnerQuery extends QueryPromise<string | undefined> {
async execute() { return owners.getStore(); }
}
afterEach(() => {
vi.unstubAllEnvs();
vi.useRealTimers();
});
it.each(["", "http://collector.test"])("attributes lazy queries to execution, preparation, and budget owners with endpoint %s", async endpoint => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
await withDatabaseOperation("http.request", async () => {
const query = new OwnerQuery();
await expect(withRunExecutionTiming("private-run", () => query)).resolves.toBe("task.execution_attempt");
await expect(withRunWorkspaceFinalizationTiming("private-run", () => query)).resolves.toBe("task.workspace_finalization");
await expect(withRunResponsePresentationTiming("private-run", () => query)).resolves.toBe("task.response_presentation");
await expect(createRunPreparationTiming("private-run")("agent_read", () => query)).resolves.toBe("heartbeat.prepare.agent_read");
await expect(createRunPreparationTiming("private-run", "native.controller")("backend_setup", () => query)).resolves.toBe("native.controller.backend_setup");
await expect(measureRunPhase("skills.inventory", () => query)).resolves.toBe("skills.inventory");
await expect(withBudgetSpan("budget.accounting.transaction", () => query)).resolves.toBe("budget.accounting.transaction");
await expect(query).resolves.toBe("http.request");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps owner attribution after the span detail limit and restores nested scopes after failure", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.test");
const failure = new Error("private accounting failure");
await withRunExecutionTiming("private-run", async () => {
for (let i = 0; i < 18; i++) {
await expect(measureRunPhase("skills.inventory", () => new OwnerQuery())).resolves.toBe("skills.inventory");
}
await expect(measureRunPhase("execution.budget_reservation", async () => {
await expect(withBudgetSpan("budget.admission.reserve", async () => {
await Promise.resolve();
expect(owners.getStore()).toBe("budget.admission.reserve");
throw failure;
})).rejects.toBe(failure);
return new OwnerQuery();
})).resolves.toBe("execution.budget_reservation");
await expect(new OwnerQuery()).resolves.toBe("task.execution_attempt");
});
expect(owners.getStore()).toBeUndefined();
});
it("keeps concurrent HTTP requests and nested work in fixed scopes without using private URLs", async () => {
vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "");
const app = express();
app.use(databaseOperationMiddleware);
app.use(express.json());
let arrived = 0;
let release!: () => void;
const both = new Promise<void>(resolve => { release = resolve; });
app.all("/:privateId", async (req, res) => {
const before = await new OwnerQuery();
const work = async () => {
if (++arrived === 2) release();
await both;
return new OwnerQuery();
};
const during = req.params.privateId === "first"
? await withBudgetSpan("budget.admission.reserve", work)
: await measureRunPhase("skills.inventory", work);
res.json({ before, during, after: await new OwnerQuery() });
});
const [first, second] = await Promise.all([
request(app).get("/first?secret=private-first").expect(200),
request(app).post("/second?secret=private-second").send({ private: "request-body" }).expect(200),
]);
expect(first.body).toEqual({ before: "http.request", during: "budget.admission.reserve", after: "http.request" });
expect(second.body).toEqual({ before: "http.request", during: "skills.inventory", after: "http.request" });
expect(owners.getStore()).toBeUndefined();
});
it("replaces producer ownership on each delivery dispatch, including retry and pending checks", async () => {
vi.useFakeTimers();
const execute = vi.fn(async () => [{ xid: "42" }]);
const database = installDatabaseWorkSignals({ execute, async transaction<T>(action: (tx: object) => Promise<T>): Promise<T> {
return action({ execute });
} });
const seen: Array<{ queue: string; phase: string; owner: string | undefined }> = [];
const coordinator = createDeliveryWorkCoordinator({ owner: database, canRun: () => true,
onError: (_error, queue) => { seen.push({ queue, phase: "failure", owner: owners.getStore() }); },
});
let fail = true;
try {
const workers = withDatabaseOperation("http.request", () => [DELIVERY_QUEUES.feedback, DELIVERY_QUEUES.email].map(queue =>
coordinator.register(queue, { retryMs: 5000,
run: async () => {
await Promise.resolve();
seen.push({ queue, phase: "run", owner: await new OwnerQuery() });
if (queue === DELIVERY_QUEUES.feedback && fail) { fail = false; throw new Error("retry"); }
},
hasPending: async () => {
seen.push({ queue, phase: "pending", owner: await new OwnerQuery() });
return false;
},
})));
await Promise.all(workers.map(worker => worker.ready));
await withDatabaseOperation("budget.accounting.transaction", () => database.transaction(tx =>
notifyDeliveryWork(tx, DELIVERY_QUEUES.feedback)));
await vi.advanceTimersByTimeAsync(1);
expect(seen.map(({ phase }) => phase).sort()).toEqual(["failure", "pending", "pending", "run", "run", "run"]);
for (const entry of seen) expect(entry.owner).toBe(`delivery.${entry.queue}`);
expect(owners.getStore()).toBeUndefined();
expect(coordinator.nextWakeAt()).toBeNull();
} finally {
await coordinator.stop();
}
});
server/src/__tests__/github-operation-credentials.test.ts
+3 -3
View File
@@ -8,7 +8,7 @@ import { createRuntimeToolsToken } from "../runtime-tools-token.js";
import { errorHandler } from "../middleware/index.js";
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import {
agents,
@@ -87,7 +87,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
});
await db.insert(toolConnections).values(irrelevant);
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { toolConnections }, capture);
try {
expect(await resolveManagedGitHubIdentitySelection(observed, input.companyId, { agentId: input.agentId, responsibleUserId: "A" }))
@@ -119,7 +119,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
}).where(eq(heartbeatRuns.id, input.runId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, input.runId));
const observed = createDb(external ?? database!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
expect(await resolveGitHubOperationCredentials(observed, input)).toMatchObject({
server/src/__tests__/heartbeat-run-event-sequencing.test.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { describe, expect, it } from "vitest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import {
agents,
companies,
@@ -63,11 +63,11 @@ describe("P6-11..13 / P6-17 canonical event allocator", () => {
protocolSchemaVersion: 1, canonicalPayload: { sourceSeq } },
});
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRunEvents }, capture);
try { expect((await append(5002)).highestContiguousSourceSeq).toBe(5000); }
finally { capture.flush(); }
const reads = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const reads = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_run_events" && (event.operation === "select" || event.operation === "execute"));
expect(reads.reduce((sum, event) => sum + Number(event.calls), 0)).toBeGreaterThan(0);
expect(reads.reduce((sum, event) => sum + Number(event.rows), 0)).toBeLessThan(16);
server/src/__tests__/heartbeat-run-log.test.ts
+18
View File
@@ -41,3 +41,21 @@ describe("compactRunLogChunk", () => {
expect(compacted).not.toContain("paperclip-flag-secret");
});
});
describe("compactRunLogChunk with long base64 commands", () => {
it("redacts a tool call that writes base64 of source text without stalling", () => {
const typescriptSource = "export const limit = 65536;\nexport function keep(value: string) {\n return value.slice(0, limit);\n}\n";
const blob = Buffer.from(typescriptSource.repeat(1_000)).toString("base64").slice(0, 48 * 1024);
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
const command = `printf '%s' '${blob}' | base64 -d > src/keep.ts && echo '{"GITHUB_TOKEN":"tool-call-secret"}'`;
const chunk = `${JSON.stringify({ type: "assistant", message: { content: [{ type: "tool_use", input: { command } }] } })}\n`;
const startedAt = performance.now();
const compacted = compactRunLogChunk(chunk);
expect(performance.now() - startedAt).toBeLessThan(1_000);
expect(compacted).toContain(blob);
expect(compacted).not.toContain("tool-call-secret");
expect(compacted).toContain("***REDACTED***");
});
});
server/src/__tests__/heartbeat-run-privacy-routes.test.ts
+2 -2
View File
@@ -1,7 +1,7 @@
import { randomUUID } from "node:crypto";
import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import express from "express";
import request from "supertest";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
@@ -268,7 +268,7 @@ describeEmbeddedPostgres("heartbeat run privacy routes", { concurrent: false },
}).where(eq(heartbeatRuns.id, f.privateRunId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.privateRunId));
const observed = createDb(externalDatabaseUrl ?? tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const app = createApp(f.companyId, f.grantedAgentId, observed);
server/src/__tests__/helpers/full-database-response-capture.test.ts
+62
View File
@@ -0,0 +1,62 @@
import { expect, it } from "vitest";
import { createFullDatabaseResponseCapture } from "./full-database-response-capture.js";
const metadata = { operation: "select", table: "heartbeat_runs", callers: [] };
it("measures every oversized string and complex value without sampling or truncation", () => {
const events: Array<Record<string, unknown>> = [];
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
const text = "private-payload🙂".repeat(30_000);
const nested = { instructions: text, additional: [{ prompt: text }] };
const rows = Object.assign([[text, nested]], { columns: [{ name: "stdout_excerpt" }, { name: "context_snapshot" }] });
const bytes = Buffer.byteLength(text) + Buffer.byteLength(JSON.stringify(nested));
try {
for (let index = 0; index < 17; index++) expect(capture.execute(() => rows, metadata)).toBe(rows);
capture.flush();
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.entry.v1",
calls: 17, rows: 17, responseValueBytes: 17 * bytes, maxResponseValueBytes: bytes,
estimatedValues: 17, serializationErrors: 0,
});
expect(events[1]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
startedQueries: 17, completedQueries: 17, inFlightAtEnd: 0, overflowCalls: 0, responseValueBytes: 17 * bytes,
});
expect(JSON.stringify(events)).not.toContain("private-payload");
expect(nested.instructions).toBe(text);
} finally { capture.flush(); }
});
it("preserves asynchronous results and original query errors", async () => {
const events: Array<Record<string, unknown>> = [];
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => 0);
const rows = Object.assign([{ value: { nested: "result" } }], { columns: [{ name: "value" }] });
const failure = new Error("private query failure");
try {
await expect(capture.execute(async () => rows, metadata)).resolves.toBe(rows);
await expect(capture.execute(async () => { throw failure; }, metadata)).rejects.toBe(failure);
expect(() => capture.execute(() => { throw failure; }, metadata)).toThrow(failure);
capture.flush();
expect(events[0]).toMatchObject({ calls: 3, rows: 1, queryErrors: 2,
responseValueBytes: Buffer.byteLength(JSON.stringify(rows[0].value)), serializationErrors: 0,
});
expect(events[1]).toMatchObject({ startedQueries: 3, completedQueries: 3, inFlightAtEnd: 0 });
expect(JSON.stringify(events)).not.toContain(failure.message);
} finally { capture.flush(); }
});
it("reports unfinished work at the deadline without cancelling it or counting a late result", async () => {
const events: Array<Record<string, unknown>> = [];
let now = 0;
const capture = createFullDatabaseResponseCapture({ start: 0, end: 1000 }, event => events.push(event), () => now);
let release!: (value: string[]) => void;
const original = new Promise<string[]>(resolve => { release = resolve; });
const pending = capture.execute(() => original, metadata);
now = 1000;
capture.flush();
release(["result"]);
await expect(pending).resolves.toEqual(["result"]);
capture.flush();
expect(events).toHaveLength(1);
expect(events[0]).toMatchObject({ event: "paperclip.test_db_response_capture.summary.v1",
startedQueries: 1, completedQueries: 0, inFlightAtEnd: 1, responseValueBytes: 0,
});
});
server/src/__tests__/helpers/full-database-response-capture.ts
+115
View File
@@ -0,0 +1,115 @@
import { randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, type ResponseCaptureWindow } from "@paperclipai/db/response-capture";
export { installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
type Capture = ReturnType<typeof createDatabaseResponseCapture>;
type Metadata = Parameters<Capture["execute"]>[1];
type Entry = Metadata & { calls: number; rows: number; bytes: number; maxBytes: number;
queryErrors: number; serializationErrors: number; estimatedValues: number };
function fullResponseValueSize(value: unknown): { bytes: number; estimatedValues: number } {
const columns = value && typeof value === "object" ? (value as { columns?: unknown }).columns : undefined;
if (!Array.isArray(value) || !Array.isArray(columns)) {
const serialized = JSON.stringify(value, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
return { bytes: serialized === undefined ? 0 : Buffer.byteLength(serialized), estimatedValues: 1 };
}
let bytes = 0, estimatedValues = 0;
for (const row of value) {
const cells = Array.isArray(row) ? row : columns.map(column => row[column.name]);
for (const cell of cells) {
if (cell === null || cell === undefined) continue;
if (typeof cell === "string") bytes += Buffer.byteLength(cell);
else if (typeof cell === "boolean") bytes += 1;
else if (typeof cell === "number" || typeof cell === "bigint") bytes += Buffer.byteLength(String(cell));
else {
estimatedValues += 1;
const serialized = JSON.stringify(cell, (_, item: unknown) => typeof item === "bigint" ? item.toString() : item);
if (serialized !== undefined) bytes += Buffer.byteLength(serialized);
}
}
}
return { bytes, estimatedValues };
}
export function createFullDatabaseResponseCapture(
window: ResponseCaptureWindow,
emit: (event: Record<string, unknown>) => void,
now: () => number = Date.now,
): Capture {
const capture = createDatabaseResponseCapture(window, () => {}, now);
const id = randomUUID(), entries = new Map<string, Entry>();
let closed = false, inFlight = 0, started = 0, completed = 0, overflowCalls = 0;
const send = (event: Record<string, unknown>) => { try { emit(event); } catch {} };
function flush() {
if (closed) return;
closed = true;
clearTimeout(timer);
capture.flush();
const ordered = [...entries.values()].sort((a, b) => b.bytes - a.bytes);
for (const [groupIndex, entry] of ordered.entries()) send({
event: "paperclip.test_db_response_capture.entry.v1", captureId: id,
groupIndex, groupPage: Math.floor(groupIndex / 128),
operation: entry.operation, table: entry.table, callers: entry.callers,
calls: entry.calls, rows: entry.rows, responseValueBytes: entry.bytes,
maxResponseValueBytes: entry.maxBytes, estimatedValues: entry.estimatedValues,
queryErrors: entry.queryErrors, serializationErrors: entry.serializationErrors,
});
send({ event: "paperclip.test_db_response_capture.summary.v1", captureId: id,
windowStart: new Date(window.start).toISOString(), windowEnd: new Date(window.end).toISOString(),
closedAt: new Date(now()).toISOString(), startedQueries: started, completedQueries: completed,
inFlightAtEnd: inFlight, groups: entries.size, groupPageSize: 128, groupPages: Math.ceil(ordered.length / 128),
overflowCalls, responseValueBytes: ordered.reduce((sum, entry) => sum + entry.bytes, 0),
rowValuesLogged: false, sqlOrParametersLogged: false, wireBytesMeasured: false,
measurementBoundary: "postgres-js results before Drizzle column decoding",
relationalQueriesMeasured: true, unsafeDriverQueriesMeasured: true,
taggedDriverQueriesMeasured: false, streamingQueriesMeasured: false,
});
entries.clear();
}
const timer = setTimeout(flush, Math.max(0, window.end - now()));
timer.unref();
function finish(metadata: Metadata, value: unknown, failed: boolean) {
if (!capture.active()) return;
inFlight -= 1;
completed += 1;
let key = JSON.stringify(metadata);
if (!entries.has(key) && entries.size >= 2047) {
overflowCalls += 1;
metadata = { operation: "other", table: "other", callers: [] };
key = "overflow";
}
let entry = entries.get(key);
if (!entry) {
entry = { ...metadata, calls: 0, rows: 0, bytes: 0, maxBytes: 0,
queryErrors: 0, serializationErrors: 0, estimatedValues: 0 };
entries.set(key, entry);
}
entry.calls += 1;
if (failed) { entry.queryErrors += 1; return; }
entry.rows += Array.isArray(value) ? value.length : 0;
try {
const size = fullResponseValueSize(value);
entry.bytes += size.bytes;
entry.maxBytes = Math.max(entry.maxBytes, size.bytes);
entry.estimatedValues += size.estimatedValues;
} catch { entry.serializationErrors += 1; }
}
function execute(action: () => unknown, metadata: Metadata) {
if (!capture.active()) return action();
inFlight += 1;
started += 1;
let result: unknown;
try { result = action(); }
catch (error) { finish(metadata, undefined, true); throw error; }
let asynchronous = false;
try { asynchronous = Boolean(result && typeof (result as { then?: unknown }).then === "function"); } catch {}
if (asynchronous) return Promise.resolve(result).then(
value => { finish(metadata, value, false); return value; },
error => { finish(metadata, undefined, true); throw error; },
);
finish(metadata, result, false);
return result;
}
return { ...capture, execute, flush };
}
server/src/__tests__/issue-creation-source-read.test.ts
+2 -2
View File
@@ -1,6 +1,6 @@
import { randomUUID } from "node:crypto";
import { activityLog, agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -55,7 +55,7 @@ describe.skipIf(!support.supported)("creation source run projections", () => {
expect(run.issueId).toBe(binding === "scoped" ? sourceId : null);
expect(run.nativeIssueId).toBeNull();
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns, issues }, capture);
try {
const result = await resolveIssueCreationSource(observed, child);
server/src/__tests__/issue-display-status.test.ts
+127
View File
@@ -0,0 +1,127 @@
import { randomUUID } from "node:crypto";
import { eq, sql } from "drizzle-orm";
import { expect, it } from "vitest";
import request from "supertest";
import { agents, heartbeatRuns, issueRecoveryActions, issueRelations, issues, statusCards } from "@paperclipai/db";
import { companySearchQuerySchema, createIssueSchema, PROVIDER_QUOTA_MONITOR_SERVICE_NAME } from "@paperclipai/shared";
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
import { issueService } from "../services/issues.js";
import { companySearchService } from "../services/company-search.js";
import { statusCardService } from "../services/status-cards.js";
import { issueRoutes } from "../routes/issues.js";
import {
describeEmbeddedPostgres, routeApp,
seedCompanyWithBoardAccess, useEmbeddedPostgres,
} from "./helpers/route-test-harness.js";
describeEmbeddedPostgres("Monitoring task presentation", () => {
const ctx = useEmbeddedPostgres("paperclip-monitoring-status-", {
resetEach: async (db) => { await db.execute(sql`SET client_min_messages = warning`); await db.execute(sql`TRUNCATE TABLE companies CASCADE`); },
});
async function fixture() {
const company = await seedCompanyWithBoardAccess(ctx.db, "Monitoring");
const agentId = randomUUID();
await ctx.db.insert(agents).values({ id: agentId, companyId: company.companyId, name: "Monitor owner", role: "engineer", status: "paused", adapterType: "process", adapterConfig: {} });
const nextCheckAt = new Date(Date.now() + 60 * 60_000);
const issueId = randomUUID();
await ctx.db.insert(issues).values({
id: issueId, companyId: company.companyId, title: "Check deployment", status: "in_progress",
assigneeAgentId: agentId, createdByUserId: company.actor.userId,
monitorNextCheckAt: nextCheckAt,
executionPolicy: { mode: "normal", stages: [], commentRequired: false, monitor: { nextCheckAt: nextCheckAt.toISOString(), notes: "Check deployment" } },
});
return { ...company, agentId, issueId };
}
async function displayStatus(id: string) {
const [row] = await ctx.db.select({ status: issues.status, displayStatus: issueDisplayStatusSql() }).from(issues).where(eq(issues.id, id));
return row;
}
it("projects a scheduled wait consistently across detail, compact rows, summaries, search and counts", async () => {
const f = await fixture();
const svc = issueService(ctx.db);
expect(await svc.getById(f.issueId)).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
expect(await svc.list(f.companyId, { displayStatus: "monitoring", listProjection: "index" })).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
expect(await svc.count(f.companyId, { displayStatus: "monitoring" })).toBe(1);
const app = routeApp(ctx.db, f.actor, issueRoutes);
const summary = await request(app).get(`/api/companies/${f.companyId}/issues/summaries?ids=${f.issueId}`).expect(200);
expect(summary.body[0]).toMatchObject({ status: "in_progress", displayStatus: "monitoring" });
const result = await companySearchService(ctx.db).search(f.companyId, companySearchQuerySchema.parse({ q: "deployment", displayStatus: "monitoring" }));
expect(result.results[0]?.issue).toMatchObject({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" });
expect(result.filterOptionCounts.displayStatus?.monitoring).toBe(1);
expect(result.filterOptionCounts.status.in_progress).toBe(1);
const [card] = await ctx.db.insert(statusCards).values({ companyId: f.companyId, interestPrompt: "Watch deployment", mentionedIssueIds: [f.issueId], refreshPolicy: { mode: "manual" } }).returning();
expect(await statusCardService(ctx.db).listMentionedIssues(card!)).toEqual([expect.objectContaining({ id: f.issueId, status: "in_progress", displayStatus: "monitoring" })]);
// Existing agent/API lifecycle filters keep their original meaning.
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(1);
expect(createIssueSchema.safeParse({ title: "Manual monitoring", status: "monitoring" }).success).toBe(false);
});
it("filters before pagination and keeps In Progress separate from Monitoring", async () => {
const f = await fixture();
await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ordinary work", status: "in_progress", assigneeAgentId: f.agentId });
const svc = issueService(ctx.db);
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 0 })).toEqual([expect.objectContaining({ id: f.issueId })]);
expect(await svc.list(f.companyId, { displayStatus: "monitoring", limit: 1, offset: 1 })).toEqual([]);
expect(await svc.count(f.companyId, { displayStatus: "in_progress" })).toBe(1);
expect(await svc.count(f.companyId, { status: "in_progress" })).toBe(2);
});
it("shows the same wait indicator in list blocker summaries and task detail relations", async () => {
const f = await fixture();
const [dependent] = await ctx.db.insert(issues).values({ companyId: f.companyId, title: "Ship after deployment", status: "blocked" }).returning();
await ctx.db.insert(issueRelations).values({ companyId: f.companyId, issueId: f.issueId, relatedIssueId: dependent!.id, type: "blocks" });
const svc = issueService(ctx.db);
const list = await svc.list(f.companyId, { includeBlockedBy: true });
expect(list.find(row => row.id === dependent!.id)?.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
const detail = await request(routeApp(ctx.db, f.actor, issueRoutes)).get(`/api/issues/${dependent!.id}`).expect(200);
expect(detail.body.blockedBy).toEqual([expect.objectContaining({ id: f.issueId, displayStatus: "monitoring" })]);
});
it("returns to In Progress during the check and Monitoring after another yield, then Done", async () => {
const f = await fixture();
const runId = randomUUID();
await ctx.db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, issueId: f.issueId, status: "queued" });
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, runId));
await ctx.db.update(issues).set({ executionRunId: runId, checkoutRunId: runId }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(heartbeatRuns).set({ status: "succeeded" }).where(eq(heartbeatRuns.id, runId));
await ctx.db.update(issues).set({ executionRunId: null, checkoutRunId: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
await ctx.db.update(issues).set({ status: "done" }).where(eq(issues.id, f.issueId));
expect(await displayStatus(f.issueId)).toEqual({ status: "done", displayStatus: "done" });
});
it.each(["blocked", "in_review", "todo", "cancelled"])("preserves %s even with a retained monitor", async (status) => {
const f = await fixture();
await ctx.db.update(issues).set({ status }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe(status);
});
it("does not label cleared, consumed, orphaned or quota-recovery schedules Monitoring", async () => {
const f = await fixture();
await ctx.db.update(issues).set({ executionState: { monitor: { status: "cleared" } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ executionState: null, monitorNextCheckAt: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ monitorNextCheckAt: new Date(), executionPolicy: { monitor: { serviceName: PROVIDER_QUOTA_MONITOR_SERVICE_NAME } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.update(issues).set({ executionPolicy: null }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
});
it("keeps recovery and review holds visible and never writes a new lifecycle state", async () => {
const f = await fixture();
await ctx.db.insert(issueRecoveryActions).values({ companyId: f.companyId, sourceIssueId: f.issueId, kind: "source", ownerType: "board", cause: "process_lost", fingerprint: randomUUID(), nextAction: "Inspect stopped run" });
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'escalated' WHERE source_issue_id = ${f.issueId}`);
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
await ctx.db.execute(sql`UPDATE issue_recovery_actions SET status = 'resolved' WHERE source_issue_id = ${f.issueId}`);
expect((await displayStatus(f.issueId)).displayStatus).toBe("monitoring");
await ctx.db.update(issues).set({ executionState: { currentParticipant: { type: "user", userId: "reviewer" } } }).where(eq(issues.id, f.issueId));
expect((await displayStatus(f.issueId)).displayStatus).toBe("in_progress");
});
});
server/src/__tests__/issue-queued-comments-routes.test.ts
+4 -4
View File
@@ -4,7 +4,7 @@ import { randomUUID } from "node:crypto";
import express from "express";
import request from "supertest";
import { eq, sql } from "drizzle-orm";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import {
agentRuntimeState,
@@ -202,7 +202,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
if (fallback) await db.update(issues).set({ executionRunId: null }).where(eq(issues.id, f.issueId));
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.runId));
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed))
@@ -238,7 +238,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
}).where(eq(heartbeatRuns.companyId, f.companyId));
const [before] = await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, f.wakeId));
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agentWakeupRequests, heartbeatRuns }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
@@ -275,7 +275,7 @@ describeEmbeddedPostgres("issue queued-comment routes", () => {
})),
]);
const observed = createDb(tempDb!.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { agentWakeupRequests }, capture);
try {
const response = await request(app(f.companyId, "queue-owner", undefined, observed)).get(path).expect(200);
server/src/__tests__/issue-summaries-routes.test.ts
+1 -1
View File
@@ -50,7 +50,7 @@ describeEmbeddedPostgres("bounded issue summaries", () => {
expect(res.headers["cache-control"]).toBe("private, no-store");
expect(res.body.map((row: { id: string }) => row.id)).toEqual([f.rows[1]!.id, f.rows[0]!.id]);
expect(Object.keys(res.body[0]).sort()).toEqual([
"companyId", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
"companyId", "displayStatus", "externalConversationState", "id", "identifier", "status", "title", "updatedAt",
]);
expect(res.body[0].externalConversationState).toBeNull();
expect(forbiddenRead).not.toHaveBeenCalled();
server/src/__tests__/issues-list-query-parsing.test.ts
+26
View File
@@ -56,4 +56,30 @@ describeEmbeddedPostgres("issue list status query parsing", () => {
it("returns every status when ?status is absent", async () => {
expect(await listStatuses("")).toEqual(["done", "in_progress", "todo"]);
});
it.each(["", "&view=compact", "&view=compact&projection=index"])("rejects unsupported creation sorting for blocked attention (%s)", async projection => {
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Blocked sorting");
await request(routeApp(ctx.db, actor, issueRoutes))
.get(`/api/companies/${companyId}/issues?attention=blocked&sortField=created&sortDir=asc${projection}`)
.expect(422);
});
it.each(["", "&view=compact", "&view=compact&projection=index"])("sorts routine tasks by creation time before pagination (%s)", async (projection) => {
const { companyId, actor } = await seedCompanyWithBoardAccess(ctx.db, "Creation sorting");
const ids = Array.from({ length: 4 }, (_, index) => `00000000-0000-4000-8000-00000000000${index}`);
await ctx.db.insert(issues).values([
...ids.map((id, index) => ({ id, companyId, title: "Routine sorting", originKind: "routine_execution", status: "todo",
createdAt: new Date(index < 2 ? "2026-01-01" : "2026-02-01"),
updatedAt: new Date(index < 2 ? "2026-03-01" : "2026-02-01"), priority: index < 2 ? "critical" : "low" })),
{ companyId, title: "Routine sorting manual", originKind: "manual", createdAt: new Date("2026-04-01") },
]);
const app = routeApp(ctx.db, actor, issueRoutes);
const path = `/api/companies/${companyId}/issues?originKind=routine_execution&includeRoutineExecutions=true&sortField=created&limit=2${projection}`;
for (const sortDir of ["asc", "desc"]) {
const first = await request(app).get(`${path}&sortDir=${sortDir}`).expect(200);
const second = await request(app).get(`${path}&sortDir=${sortDir}&offset=2`).expect(200);
expect([...first.body, ...second.body].map((issue: { id: string }) => issue.id)).toEqual(sortDir === "asc" ? ids : [...ids].reverse());
}
await request(app).get(`${path}&sortDir=invalid`).expect(400);
});
});
server/src/__tests__/legacy-workspace-restore-recovery.test.ts
+5 -5
View File
@@ -1,5 +1,5 @@
import { randomBytes, randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { and, eq, sql } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { agents, companies, createDb, environmentLeases, environments, executionWorkspaces, heartbeatRuns, issueRecoveryActions, issues, projects } from "@paperclipai/db";
@@ -54,12 +54,12 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
runnerProfileJson: { savedProfile: bulk }, stdoutExcerpt: bulk, stderrExcerpt: bulk,
}).where(eq(heartbeatRuns.id, f.runId)).returning();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
try { returned = await terminalizeLegacyExecution({ db, run: saved!, status: "failed", patch }); }
finally { capture.flush(); }
const groups = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const groups = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
const reads = groups.filter(group => group.table === "heartbeat_runs" && group.operation === "select");
expect(reads.reduce((sum, group) => sum + Number(group.calls), 0)).toBe(1);
expect(reads.reduce((sum, group) => sum + Number(group.responseValueBytes), 0)).toBeLessThan(4096);
@@ -86,7 +86,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
patch: { resultJson: { workspaceRestoreFailure: "restore_failed", futureResult: bulk } },
}))!;
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let returned: Awaited<ReturnType<typeof terminalizeLegacyExecution>>;
try { returned = await terminalizeLegacyExecution({ db, run: failed, status: "failed", recordRestoreFailureOnly,
@@ -94,7 +94,7 @@ const support = externalTestDatabaseUrl ? { supported: true } : await getEmbedde
...(recordRestoreFailureOnly ? { patch: { resultJson: failed.resultJson } } : {}),
}); }
finally { capture.flush(); }
const writes = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const writes = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "heartbeat_runs" && event.operation === "update");
expect(writes.reduce((sum, group) => sum + Number(group.rows), 0)).toBe(1);
expect(returned).toMatchObject({ id: f.runId, status: "failed", finishedAt: failed.finishedAt,
server/src/__tests__/native-environment-settlement-work.test.ts
+2 -2
View File
@@ -11,7 +11,7 @@ import { setTimeout as delay } from "node:timers/promises";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { agents, companies, completionContracts, createDb, environmentLeases, environments, heartbeatRuns, issues, nativeRunFinalizations, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js";
import { heartbeatService } from "../services/heartbeat.js";
import { environmentRuntimeService, type EnvironmentDriver } from "../services/environment-runtime.js";
@@ -180,7 +180,7 @@ describe("native environment release scheduling", () => {
}
const events: Record<string, unknown>[] = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { nativeRunFinalizations, heartbeatRuns, environmentLeases }, capture);
const settle = vi.fn(async () => {});
const work = nativeEnvironmentSettlementWork(db, settle);
server/src/__tests__/native-settled-process-release.test.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import { resolveExternalTestDatabaseUrl } from "./helpers/external-test-database.js";
import { EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db";
import { randomBytes, randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { spawn } from "node:child_process";
import os from "node:os";
import { and, eq, sql } from "drizzle-orm";
@@ -222,11 +222,11 @@ suite("settled native run process identity", () => {
const saved = await readRun(runId);
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try { expect(await releaseSettledNativeRunProcess(db, runId)).toBe("released"); }
finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.responseValueBytes).toBeLessThan(8192);
const after = await readRun(runId);
expect(after).toEqual({ ...saved, processPid: null, processGroupId: null, processStartedAt: null,
server/src/__tests__/plugin-host-services-span.test.ts
+30
View File
@@ -332,6 +332,36 @@ describe("clampProviderSpanAttributes", () => {
});
describe("nested worker timing receipts",()=>{
it("keeps lease verification beneath its worker and clamps provider state", () => {
mockRecordSpan.mockReset();
const rootParent = { traceId: "0af7651916cd43dd8448eb211c80319c", spanId: "1111111111111111", traceFlags: 1 };
const verifyParent = { ...rootParent, spanId: "2222222222222222" };
mockRecordSpan.mockReturnValueOnce(rootParent).mockReturnValueOnce(verifyParent);
const now = Date.now();
recordWorkerProviderSpan({ name: "plugin.worker", startTimeMs: now - 100, endTimeMs: now, children: [
{ name: "lease.verify", startTimeMs: now - 90, endTimeMs: now - 10, children: [
{ name: "lease.sentinel", startTimeMs: now - 80, endTimeMs: now - 20,
attributes: { "paperclip.sandbox.startup.state": "archived", token: "private-token" } },
] },
] }, { traceparent: VALID_TRACEPARENT });
expect(mockRecordSpan.mock.calls[1][0]).toMatchObject({ name: "sandbox.daytona.lease.verify", parent: rootParent });
expect(mockRecordSpan.mock.calls[2][0]).toMatchObject({ name: "sandbox.daytona.lease.sentinel", parent: verifyParent,
startTimeMs: now - 80, endTimeMs: now - 20, attributes: { "paperclip.sandbox.startup.state": "archived" } });
expect(JSON.stringify(mockRecordSpan.mock.calls)).not.toContain("private-token");
expect(clampProviderSpanAttributes({ "paperclip.sandbox.startup.state": "private-state" })).toEqual({});
});
it("admits every closed lease phase and rejects dynamic phase names", () => {
mockRecordSpan.mockReset();
const names = ["lease.wait_for_teardown", "lease.lookup", "lease.refresh", "lease.channel_cleanup",
"lease.start", "lease.recover", "lease.verify", "lease.workspace", "lease.sentinel", "lease.shell",
"lease.create", "lease.expiry", "lease.cleanup", "lease.private-resource-id"];
for (const name of names) recordWorkerProviderSpan({ name }, { traceparent: VALID_TRACEPARENT });
expect(mockRecordSpan.mock.calls.map(call => call[0].name)).toEqual([
...names.slice(0, -1).map(name => `sandbox.daytona.${name}`), "sandbox.daytona.other",
]);
});
it("uses host-generated parents, clamps content and bounds child clocks",()=>{
mockRecordSpan.mockReset();
const childParent={traceId:"0af7651916cd43dd8448eb211c80319c",spanId:"1111111111111111",traceFlags:1};
server/src/__tests__/redaction.test.ts
+70
View File
@@ -899,3 +899,73 @@ second-line\" status=401`,
});
});
});
describe("redaction of long letter/digit runs", () => {
const typescriptSource = [
"export function compactRunLogChunk(chunk: string, maxChars = 65536) {",
" const normalized = redactSensitiveText(chunk);",
" if (normalized.length <= maxChars) return normalized;",
" return normalized.slice(0, maxChars);",
"}",
"",
].join("\n");
function base64OfTypeScriptSource(length: number) {
const repeatedSource = typescriptSource.repeat(Math.ceil(length / typescriptSource.length));
return Buffer.from(repeatedSource).toString("base64").slice(0, length);
}
function redactWithinTimeBudget(input: string) {
const startedAt = performance.now();
const output = redactSensitiveText(input);
expect(performance.now() - startedAt).toBeLessThan(1_000);
return output;
}
it("redacts secret field names that carry a prefix in plain and escaped JSON", () => {
const cases = [
[`{"myApiKey": "plain-prefixed-secret"}`, `{"myApiKey": "${REDACTED_EVENT_VALUE}"}`],
[`{"x_auth_token":"snake-prefixed-secret","status":"ok"}`, `{"x_auth_token":"${REDACTED_EVENT_VALUE}","status":"ok"}`],
[`GITHUB_TOKEN: "upper-prefixed-secret"`, `GITHUB_TOKEN: "${REDACTED_EVENT_VALUE}"`],
[`{'db_password': 'single-quoted-secret'}`, `{'db_password': '${REDACTED_EVENT_VALUE}'}`],
[`text"apiKey":"glued-quote-secret"`, `text"apiKey":"${REDACTED_EVENT_VALUE}"`],
[`aGVsbG8apiKey:"glued-run-secret"`, `aGVsbG8apiKey:"${REDACTED_EVENT_VALUE}"`],
[String.raw`{\"myApiKey\":\"escaped-prefixed-secret\"}`, String.raw`{\"myApiKey\":\"${REDACTED_EVENT_VALUE}\"}`],
[
String.raw`{\"x_auth_token\":\"escaped-snake-secret\",\"status\":\"ok\"}`,
String.raw`{\"x_auth_token\":\"${REDACTED_EVENT_VALUE}\",\"status\":\"ok\"}`,
],
[String.raw`aGVsbG8\"GITHUB_TOKEN\":\"escaped-glued-secret\"`, String.raw`aGVsbG8\"GITHUB_TOKEN\":\"${REDACTED_EVENT_VALUE}\"`],
];
for (const [input, expected] of cases) {
expect(redactSensitiveText(input)).toBe(expected);
expect(redactSensitiveText(expected)).toBe(expected);
}
});
it("keeps base64 of source text intact and fast while redacting a secret glued to its end", () => {
const blob = base64OfTypeScriptSource(48 * 1024);
expect(blob).toMatch(/^[A-Za-z0-9]+$/);
const shellCommand = `printf '%s' '${blob}' | base64 -d > src/compact.ts`;
expect(redactWithinTimeBudget(shellCommand)).toBe(shellCommand);
const toolCallLine = JSON.stringify({ type: "tool_use", input: { command: shellCommand } });
expect(redactWithinTimeBudget(toolCallLine)).toBe(toolCallLine);
const serializedToolCallLine = JSON.stringify(toolCallLine);
expect(redactWithinTimeBudget(serializedToolCallLine)).toBe(serializedToolCallLine);
const gluedSecret = redactWithinTimeBudget(`${blob}apiKey:"run-glued-secret"`);
expect(gluedSecret).toBe(`${blob}apiKey:"${REDACTED_EVENT_VALUE}"`);
});
it("scans long dash-separated runs in linear time", () => {
for (const run of ["a-".repeat(60_000), "a--".repeat(40_000)]) {
expect(redactWithinTimeBudget(`${run} Authorization: Bearer dash-run-secret`))
.toBe(`${run} Authorization: ${REDACTED_EVENT_VALUE}`);
expect(redactWithinTimeBudget(`${run}Authorization: Bearer glued-run-secret`))
.toBe(`${run}Authorization: ${REDACTED_EVENT_VALUE}`);
}
});
});
server/src/__tests__/sandbox-provider-schema-reads.test.ts
+3 -3
View File
@@ -2,7 +2,7 @@ import { randomBytes, randomUUID } from "node:crypto";
import { eq, sql, type SQL } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
import { createDb, plugins, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import type { PaperclipPluginManifestV1, PluginEnvironmentDriverDeclaration } from "@paperclipai/shared";
import { resolvePluginSandboxProviderConfigSchema, resolvePluginSandboxProviderDriverByKey } from "../services/plugin-environment-driver.js";
import { resolveEnvironmentDriverConfigForRuntime, resolveSandboxProviderSecretRefPaths } from "../services/environment-config.js";
@@ -39,12 +39,12 @@ describe.skipIf(!support.supported)("sandbox provider schema reads", () => {
const lookup = (client: Db, driverKey = key) => resolvePluginSandboxProviderConfigSchema({ db: client, driverKey });
async function measure<T>(client: Db, operation: () => Promise<T>) {
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event), () => now);
installDatabaseResponseCapture(client, { plugins }, capture);
try {
const value = await operation();
capture.flush();
const entries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1");
const entries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1");
return { value, bytes: entries.reduce((sum, entry) => sum + Number(entry.responseValueBytes), 0),
calls: entries.reduce((sum, entry) => sum + Number(entry.calls), 0) };
} finally { capture.flush(); }
server/src/__tests__/tool-effective-assignments.test.ts
+6 -6
View File
@@ -5,7 +5,7 @@ import {
agents, companies, createDb, toolApplications, toolCatalogEntries,
toolConnections, toolProfileBindings, toolProfileEntries, toolProfiles, type Db,
} from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "./helpers/full-database-response-capture.js";
import { toolAccessService } from "../services/tool-access.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
@@ -149,14 +149,14 @@ describeDatabase("effective tool assignments", () => {
const before = await service.getEffectiveToolAssignmentsForAgent(f.company.id, f.agent.id);
expect(before.allowedTools).toHaveLength(102);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { toolCatalogEntries, toolProfileEntries }, capture);
let scoped: Awaited<ReturnType<typeof service.getConnectionAssignmentsForAgent>>;
try { scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]); }
finally { capture.flush(); }
expect(Number(events.at(-1)?.completedQueries)).toBeGreaterThan(0);
expect(Number(events.at(-1)?.responseValueBytes)).toBeLessThan(16 * 1024);
const catalogQueries = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1"
const catalogQueries = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1"
&& event.table === "tool_catalog_entries" && event.operation === "select");
expect(catalogQueries).toHaveLength(1);
expect(catalogQueries[0]).toMatchObject({ calls: 1, rows: 2 });
@@ -192,13 +192,13 @@ describeDatabase("effective tool assignments", () => {
const service = toolAccessService(db), before = await compare(f);
const savedRules = await db.select().from(toolProfileEntries);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { toolProfileEntries, toolProfileBindings }, capture);
try {
const scoped = await service.getConnectionAssignmentsForAgent(f.company.id, f.agent.id, [f.connection.id]);
capture.flush();
const rules = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_entries");
const bindings = events.filter(event => event.event === "paperclip.db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
const rules = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_entries");
const bindings = events.filter(event => event.event === "paperclip.test_db_response_capture.entry.v1" && event.table === "tool_profile_bindings");
expect(rules).toHaveLength(1);
expect(Number(rules[0].responseValueBytes)).toBeLessThan(4096);
expect(rules[0]).toMatchObject({ calls: 1, rows: 2 });
server/src/__tests__/voice-call-reports.test.ts
+8 -3
View File
@@ -122,16 +122,21 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
await quiet(); expect(fetchReport).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(deadline - Date.now());
await vi.waitFor(async () => {
await vi.advanceTimersByTimeAsync(1000);
// Keep time fixed during database I/O. Advancing it on every poll can
// make the first reports due again before the backlog has settled.
const next = coordinator!.nextWakeAt();
if (next !== null && next < deadline + 60_000) {
await vi.advanceTimersByTimeAsync(Math.max(0, next - Date.now()));
}
expect(idleWorkSnapshot().active).toBe(0);
expect(fetchReport).toHaveBeenCalledTimes(105);
}, { timeout: 5000 });
}, { interval: 0, timeout: 15_000 });
expect(fetchReport.mock.calls.map(([call]) => call.id).sort()).toEqual(ids.sort());
expect((await db.select().from(chatVoiceReports).where(eq(chatVoiceReports.companyId, companyId)))
.every(row => row.attempts === 1)).toBe(true);
await quiet();
} finally { reads.mockRestore(); }
});
}, 30_000);
it("leaves configured idle, live calls, old history and unidentified calls quiet", async () => {
await session({ state: "active", endedAt: null });
await session({ endedAt: new Date(Date.now() - day - 1000) });
server/src/__tests__/voice-inbound-retention.test.ts
+1
View File
@@ -145,6 +145,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu
expect(await row(call.id)).toBeUndefined(); expect(coordinator!.nextWakeAt()).toBeNull(); await quiet();
});
it("rounds PostgreSQL submillisecond deadlines up and never deletes early", async () => {
vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"], shouldAdvanceTime: false });
const call = await admission(); const dueAt = Math.trunc(Date.now()) + 60_000;
await db.execute(sql`UPDATE chat_voice_inbound_calls SET updated_at = to_timestamp(${dueAt - week}::double precision / 1000) + interval '0.0005 seconds' WHERE id = ${call.id}`);
const service = voiceInboundRetention(db), [entry] = await service.load({ kind: "admission", id: call.id });
server/src/app.ts
+2
View File
@@ -12,6 +12,7 @@ import { DELIVERY_QUEUES } from "./services/delivery-work-notifications.js";
import { createLifecycleDriver } from "./services/agent-lifecycle-driver.js";
import { startAgentLifecycle } from "./services/agent-lifecycle.js";
import { idleAdmissionMiddleware, trackIdleRequestHandlers } from "./middleware/idle-admission.js";
import { databaseOperationMiddleware } from "./middleware/database-operation.js";
import { staticUiFiles } from "./middleware/static-ui.js";
import { retainedUiAssets } from "./retained-ui-assets.js";
import { createStorageProviderFromConfig } from "./storage/provider-registry.js";
@@ -470,6 +471,7 @@ export async function createApp(
},
) {
const app = express();
app.use(databaseOperationMiddleware);
app.use(idleAdmissionMiddleware);
const staticUi = express.Router();
app.locals.paperclipDb = db;
server/src/middleware/database-operation.ts
+5
View File
@@ -0,0 +1,5 @@
import { withDatabaseOperation } from "@paperclipai/db";
import type { RequestHandler } from "express";
export const databaseOperationMiddleware: RequestHandler = (_req, _res, next) =>
withDatabaseOperation("http.request", next);
server/src/modules/wake-queue/adapters/release-run-read.test.ts
+5 -5
View File
@@ -1,6 +1,6 @@
import { randomBytes, randomUUID } from "node:crypto";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
@@ -40,7 +40,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
const scope = { companyId, runId: before.id };
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const native = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, scope));
@@ -51,7 +51,7 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
expect(native.run).not.toHaveProperty("stdoutExcerpt");
expect(native.run).not.toHaveProperty("stderrExcerpt");
} finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(1);
expect(summary.responseValueBytes).toBeLessThan(8192);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, before.id)))[0]).toEqual(before);
@@ -97,12 +97,12 @@ describe.skipIf(!support.supported)("wake release run reads", () => {
contextSnapshot: { ...context, ...launch }, resultJson: evidence }).returning();
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let selected: Awaited<ReturnType<typeof readReleaseRun>>;
try { selected = await db.transaction(tx => readReleaseRun(tx as unknown as typeof db, { companyId, runId: before.id })); }
finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(1);
expect(summary.responseValueBytes).toBeLessThan(8192);
expect(selected!.run?.contextSnapshot).toEqual(context);
server/src/redaction.ts
+4 -3
View File
@@ -2,6 +2,7 @@ import { redactCommandText } from "@paperclipai/adapter-utils";
import { isPublicExecutorToolSelector, looksLikeCredentialJwt } from "@paperclipai/adapter-utils/command-redaction";
const SECRET_FIELD_NAME_PATTERN = String.raw`[A-Za-z0-9_-]*(?:api[-_]?key|access[-_]?token|auth(?:_?token)?|token|authorization(?:[-_]?code)?|bearer|secrets?|passwd|passwords?|credentials?|jwt|private[-_]?key|cookie|connectionstring|browser[-_]?code|login[-_]?url)(?:[-_]?(?:value|header|prod(?:uction)?|dev(?:elopment)?|test|staging|primary|secondary))*`;
const SECRET_FIELD_NAME_START = String.raw`(?<![A-Za-z0-9_-])`;
const SECRET_PAYLOAD_KEY_RE = new RegExp(`^(?:${SECRET_FIELD_NAME_PATTERN})(?:_plain|_ref)?$`, "i");
// Authorization reasons are policy decision codes, not credentials. They must
@@ -369,13 +370,13 @@ const CLI_SECRET_FLAG_RE = new RegExp(
"i",
);
const JSON_SECRET_FIELD_TEXT_RE = new RegExp(
String.raw`((?:"|')?${SECRET_FIELD_NAME_PATTERN}(?:"|')?\s*:\s*(?:"|'))[^"'` +
String.raw`((?:"|')?${SECRET_FIELD_NAME_START}${SECRET_FIELD_NAME_PATTERN}(?:"|')?\s*:\s*(?:"|'))[^"'` +
"`" +
String.raw`\r\n]+((?:"|'))`,
"gi",
);
const ESCAPED_JSON_SECRET_FIELD_TEXT_RE = new RegExp(
String.raw`((?:\\")?${SECRET_FIELD_NAME_PATTERN}(?:\\")?\s*:\s*(?:\\"))[^\\\r\n]+((?:\\"))`,
String.raw`((?:\\")?${SECRET_FIELD_NAME_START}${SECRET_FIELD_NAME_PATTERN}(?:\\")?\s*:\s*(?:\\"))[^\\\r\n]+((?:\\"))`,
"gi",
);
const SECRET_TEXT_HINTS = [
@@ -666,7 +667,7 @@ function authorizationCredentialRange(
function redactAuthorizationCredentials(input: string): string {
// Include compound diagnostic labels such as proxyAuthorization while the
// value parser still requires a Basic/Bearer scheme before redacting.
const authorizationWord = /\b[A-Za-z0-9_-]*Authorization[A-Za-z0-9_-]*\b/gi;
const authorizationWord = /(?<![A-Za-z0-9_-])[A-Za-z0-9_-]*Authorization[A-Za-z0-9_-]*\b/gi;
const parts: string[] = [];
let copiedThrough = 0;
let match: RegExpExecArray | null;
server/src/routes/issues.ts
+6 -3
View File
@@ -8379,6 +8379,7 @@ export function issueRoutes(
title: row.title,
identifier: row.identifier,
status: row.status as IssueSummary["status"],
displayStatus: row.displayStatus,
updatedAt: row.updatedAt,
externalConversationState: row.externalConversationState,
}));
@@ -8548,10 +8549,10 @@ export function issueRoutes(
res.status(400).json({ error: "offset must be a non-negative integer" });
return;
}
if (sortField !== undefined && sortField !== "updated" && sortField !== "id") {
if (sortField !== undefined && sortField !== "updated" && sortField !== "created" && sortField !== "id") {
res
.status(400)
.json({ error: "sortField must be 'updated' or 'id' when provided" });
.json({ error: "sortField must be 'updated', 'created', or 'id' when provided" });
return;
}
if (sortDir !== undefined && sortDir !== "asc" && sortDir !== "desc") {
@@ -8619,6 +8620,7 @@ export function issueRoutes(
readCondition: await issueReadSqlCondition(db, req.actor),
attention: attention === "blocked" ? "blocked" : undefined,
status: req.query.status as string | string[] | undefined,
displayStatus: req.query.displayStatus as string | string[] | undefined,
assigneeAgentId,
participantAgentId: uuidFilters.participantAgentId,
assigneeUserId,
@@ -8657,7 +8659,7 @@ export function issueRoutes(
q: req.query.q as string | undefined,
limit,
offset,
sortField: sortField === "updated" || sortField === "id" ? sortField : undefined,
sortField: sortField === "updated" || sortField === "created" || sortField === "id" ? sortField : undefined,
afterId: req.query.afterId as string | undefined,
sortDir: sortDir === "asc" || sortDir === "desc" ? sortDir : undefined,
updatedSince: rawUpdatedSince,
@@ -8869,6 +8871,7 @@ export function issueRoutes(
readCondition: await issueReadSqlCondition(db, req.actor),
attention: "blocked",
status: req.query.status as string | string[] | undefined,
displayStatus: req.query.displayStatus as string | string[] | undefined,
assigneeAgentId: req.query.assigneeAgentId as string | undefined,
participantAgentId: countUuidFilters.participantAgentId,
assigneeUserId: req.query.assigneeUserId as string | undefined,
server/src/routes/user-profiles.ts
+2
View File
@@ -1,3 +1,4 @@
import { issueDisplayStatusSql } from "../services/issue-display-status.js";
import { Router } from "express";
import { and, desc, eq, gte, inArray, isNull, or, sql, type SQLWrapper } from "drizzle-orm";
import type { Db } from "@paperclipai/db";
@@ -347,6 +348,7 @@ export function userProfileRoutes(db: Db) {
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
server/src/server-info.ts
+5
View File
@@ -172,6 +172,11 @@ const GIT_INFO_CACHE_TTL_MS = 3000;
const processStartedAt = new Date().toISOString();
let gitInfoCache: { value: ServerGitInfo; expiresAt: number } | null = null;
// Lifecycle identity must not run the synchronous Git probes used by diagnostics.
export function getServerProcessStartedAt(): string {
return processStartedAt;
}
export function getServerInfoSnapshot(
opts: {
now?: number;
server/src/services/agent-directory-working-copies.ts
+17 -7
View File
@@ -2,7 +2,7 @@ import { notifyInstructionCollection } from "./instruction-collection-notificati
import fs from "node:fs/promises";
import path from "node:path";
import { isDeepStrictEqual } from "node:util";
import { and, eq, inArray, isNull, sql } from "drizzle-orm";
import { and, eq, inArray, isNull, sql, type SQL } from "drizzle-orm";
import { agents, environmentLeases, environments, agentInstructionWorkingCopies as copies, withDedicatedDbConnection, type Db } from "@paperclipai/db";
import { syncDirectoryToSsh, restoreWorkspaceFromSshExecution } from "@paperclipai/adapter-utils/ssh";
import { prepareAdapterExecutionTargetRuntime, runAdapterExecutionTargetShellCommand, type AdapterExecutionTarget, type PreparedAdapterExecutionTargetRuntime } from "@paperclipai/adapter-utils/execution-target";
@@ -16,7 +16,7 @@ import type { AuthorizationActor } from "./authorization.js";
import type { EnvironmentRuntimeService } from "./environment-runtime.js";
import type { Environment, EnvironmentLease } from "@paperclipai/shared";
import { agentDirectoryBaselineDigest, agentDirectoryProbeProgram, observeLocalAgentDirectory } from "./agent-directory-probe.js";
import { hasRemoteTerminationReceipt } from "./remote-execution-termination.js";
import { hasRemoteTerminationReceipt, resolveRemoteTerminationLease } from "./remote-execution-termination.js";
import { cachedAgentFileManifest, captureAgentFileCheckpoint, checkpointBaseline, checkpointSnapshot, type AgentFileManifest } from "./agent-file-checkpoints.js";
import { withWorkspaceLockKeepalive } from "./native-runtime/native-workspace-lock-keepalive.js";
import { logger } from "../middleware/logger.js";
@@ -46,7 +46,7 @@ function baseline(row: Copy) {
function actor(row: Copy): AuthorizationActor {
return { type: "agent", companyId: row.companyId, agentId: row.agentId, runId: row.runId, onBehalfOfUserId: row.responsibleUserId };
}
export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise<Copy | null>, patch: (row: Copy, values: Partial<typeof copies.$inferInsert>) => Promise<Copy>, environmentRuntime?: EnvironmentRuntimeService) {
export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string, runId: string) => Promise<Copy | null>, patch: (row: Copy, values: Partial<typeof copies.$inferInsert>, condition?: SQL) => Promise<Copy>, environmentRuntime?: EnvironmentRuntimeService) {
const store = agentFileStore(db);
async function insertCopy(values: typeof copies.$inferInsert) {
return db.transaction(async tx => {
@@ -503,7 +503,15 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
return row.state === "superseded" || !row.processStoppedAt ||
completed.has(row.state) && row.receipt?.cleanupPending === false;
}
async function settleCompletedCleanupRetry(row: Copy) {
if (completed.has(row.state) && row.processStoppedAt && !row.receipt?.retainedByRunId && row.receipt?.cleanupPending === false && row.nextAttemptAt) {
return patch(row, { nextAttemptAt: null }, sql`${copies.receipt} = ${JSON.stringify(row.receipt)}::jsonb
AND ${copies.processStoppedAt} IS NOT NULL`);
}
return row;
}
async function release(row: Copy, target?: AdapterExecutionTarget | null) {
row = await settleCompletedCleanupRetry(row);
if (releaseIsNoop(row) || !await owns(row)) return;
const destroyedOnly = row.receipt?.cleanupDestroyedOnly === true;
if (row.receipt?.retainedByRunId || row.state === "superseded" || !await owns(row) || !row.processStoppedAt) return;
@@ -559,7 +567,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
: eq(environmentLeases.environmentId, row.location.slice("remote:".length)),
));
if (leases.length !== 1) return false;
const lease = leases[0]!;
const lease = await resolveRemoteTerminationLease(db, leases[0]!);
if (lease.environmentId !== null && row.location !== `remote:${lease.environmentId}`) return false;
const remoteCwd = cleanup?.remoteCwd ?? lease.metadata?.remoteCwd;
return typeof remoteCwd === "string" &&
@@ -588,7 +596,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
cleanup?.leaseId ? eq(environmentLeases.id, cleanup.leaseId) : eq(environmentLeases.environmentId, row.location.slice("remote:".length)),
));
if (leases.length !== 1) return null;
const lease = leases[0]!;
const lease = await resolveRemoteTerminationLease(db, leases[0]!);
const destroyedWithoutEnvironment = lease.environmentId === null && hasRemoteTerminationReceipt(lease)
&& (lease.metadata?.remoteExecutionTermination as { state?: string } | undefined)?.state === "destroyed";
if (row.location !== `remote:${lease.environmentId}` && !destroyedWithoutEnvironment) return null;
@@ -658,10 +666,12 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string
checkpointWarm: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => current.receipt?.warm === true ? checkpoint(current, target) : Promise.resolve(current), "agent_directory_checkpoint"),
collectStopped: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => collectStopped(current, target), "agent_directory_collect"),
release: async (row: Copy) => {
const current = await get(row.companyId, row.runId);
let current = await get(row.companyId, row.runId);
if (!current) return;
current = await settleCompletedCleanupRetry(current);
// This check changes no receipt or bytes. A concurrent stop merely leaves
// cleanup to its owner or the recovery sweep; it grants no new authority.
if (!current || releaseIsNoop(current)) return;
if (releaseIsNoop(current)) return;
await serial(current, release, "agent_directory_release");
},
};
server/src/services/agent-instruction-working-copies.ts
+7 -6
View File
@@ -129,8 +129,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
};
return db.transaction(tx => write(tx as unknown as Db));
}
async function patch(row: Copy, values: Partial<typeof copies.$inferInsert>) {
const updated = await tryPatch(row, values);
async function patch(row: Copy, values: Partial<typeof copies.$inferInsert>, condition?: SQL) {
const updated = await tryPatch(row, values, false, condition);
// A late cleanup must not overwrite an explicit resolution or a newer
// baseline. Canonical content has its own independent head CAS.
return updated ?? (await get(row.companyId, row.runId))!;
@@ -499,8 +499,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
function directoryCleanupEntry(row: Copy | null): DirectoryCleanupEntry | null {
return row && ["saved", "unchanged", "resolved", "unavailable"].includes(row.state)
&& row.processStoppedAt && isAgentDirectoryCopy(row) && !row.receipt?.retainedByRunId
&& row.receipt?.cleanupPending !== false
&& (Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true)
&& (row.receipt?.cleanupPending === false ? row.nextAttemptAt !== null
: Object.hasOwn(row.receipt!, "baseline") || row.receipt?.cleanupPending === true)
? { id: row.runId, companyId: row.companyId, dueAt: row.nextAttemptAt?.getTime() ?? 0 } : null;
}
function directoryCleanupPredicate() {
@@ -508,8 +508,9 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen
AND ${copies.processStoppedAt} IS NOT NULL
AND ${copies.receipt}->>'schema' = 'paperclip.agent-files.v1'
AND coalesce(${copies.receipt}->>'retainedByRunId', '') = ''
AND ${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false'
AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')`;
AND ((${copies.receipt}->>'cleanupPending' = 'false' AND ${copies.nextAttemptAt} IS NOT NULL)
OR (${copies.receipt}->>'cleanupPending' IS DISTINCT FROM 'false'
AND (${copies.receipt} ? 'baseline' OR ${copies.receipt}->>'cleanupPending' = 'true')))`;
}
async function loadDirectoryCleanup(scope: DirectoryCleanupScope | undefined, admitted: () => boolean) {
const entries: DirectoryCleanupEntry[] = [];
server/src/services/budget-tracing.ts
+21 -12
View File
@@ -1,18 +1,27 @@
import { trace, type Span } from "@opentelemetry/api";
import { withDatabaseOperation } from "@paperclipai/db";
type BudgetOperation = "budget.accounting.lock_wait" | "budget.accounting.recover"
| "budget.accounting.settlement" | "budget.accounting.transaction"
| "budget.admission.evaluate" | "budget.admission.reserve"
| "budget.cancellation.deliver" | "budget.enforcement.deliver"
| "budget.enforcement.scope" | "budget.exposure.evaluate";
/** Operator-gated OpenTelemetry only; these spans contain no prompt, receipt,
* credential, or provider error text. Diagnostics never change accounting. */
export async function withBudgetSpan<T>(name: string, work: () => Promise<T>, attributes?: Record<string, number | boolean>) {
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work();
let operation: Promise<T> | undefined;
const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => {
try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ }
export function withBudgetSpan<T>(name: BudgetOperation, work: () => Promise<T>, attributes?: Record<string, number | boolean>) {
return withDatabaseOperation(name, async () => {
if (!process.env.OTEL_EXPORTER_OTLP_ENDPOINT?.trim()) return work();
let operation: Promise<T> | undefined;
const run = (span?: Span) => operation ??= Promise.resolve().then(work).finally(() => {
try { span?.end(); } catch { /* Diagnostics cannot change accounting. */ }
});
try {
const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span));
// Some external SDK implementations can fail before or after the callback.
// Own the accounting promise so either failure still executes work once.
void Promise.resolve(traced).catch(() => undefined);
} catch { /* Preserve the accounting operation if span creation fails. */ }
return operation ?? run();
});
try {
const traced = trace.getTracer("paperclip.budgets").startActiveSpan(name, { attributes }, span => run(span));
// Some external SDK implementations can fail before or after the callback.
// Own the accounting promise so either failure still executes work once.
void Promise.resolve(traced).catch(() => undefined);
} catch { /* Preserve the accounting operation if span creation fails. */ }
return operation ?? run();
}
server/src/services/chat-channels/authorization-run-read.test.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import { randomBytes, randomUUID } from "node:crypto";
import { agents, companies, createDb, heartbeatRuns } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../__tests__/helpers/embedded-postgres.js";
@@ -49,7 +49,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => {
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, scope.failedRunId));
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const selected = await db.transaction(tx => readChatAuthorizationRun(tx, scope, true));
@@ -58,7 +58,7 @@ describe.skipIf(!support.supported)("chat authorization run reads", () => {
expect(selected.run).not.toHaveProperty("runnerProfileJson");
expect(selected.run).not.toHaveProperty("stdoutExcerpt");
} finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBeGreaterThan(0);
expect(summary.responseValueBytes).toBeLessThan(4096);
const retry = await db.transaction(tx => readChatAuthorizationRun(tx, scope, false));
server/src/services/chat-channels/github/webhook-ingress.test.ts
+4 -4
View File
@@ -2,7 +2,7 @@ import { createHmac } from "node:crypto";
import { eq } from "drizzle-orm";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { applyPendingMigrations, chatActions, createDb } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../__tests__/helpers/full-database-response-capture.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "../../../__tests__/helpers/embedded-postgres.js";
import { resolveExternalTestDatabaseUrl } from "../../../__tests__/helpers/external-test-database.js";
import { cleanupIngressFixture, githubIngressFixture as createFixture } from "../../../__tests__/helpers/github-webhook-ingress.js";
@@ -36,7 +36,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
const f = await githubIngressFixture(db, createGitHubWebhookIngress, 768 * 1024);
const action = await f.insertAction();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions }, capture);
try { expect((await f.service.processGitHubWebhookIngress(action.id))?.status).toBe(202); }
finally { capture.flush(); }
@@ -59,7 +59,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
// A corrupt body must not let a nonclaimant overwrite a live or delayed claim.
const action = await f.insertAction({ status, result, payload: { ...f.payload, bodySha256: "0".repeat(64) } });
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 1, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { chatActions }, capture);
let response: Response | null;
try { response = await f.service.processGitHubWebhookIngress(action.id); }
@@ -67,7 +67,7 @@ describe.skipIf(!support.supported)("GitHub webhook ingress claim PostgreSQL beh
expect(response?.status ?? null).toBe(accepted ? 202 : null);
expect(await f.readAction(action.id)).toEqual(action);
expect(f.calls).toEqual([]);
expect(events.find(event => event.event === "paperclip.db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024);
expect(events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")?.responseValueBytes).toBeLessThan(1024);
});
it.each(["received", "failed", "processing"])("settles eligible malformed %s payloads without replay", async status => {
server/src/services/company-search.ts
+18 -3
View File
@@ -1,3 +1,4 @@
import { issueDisplayStatusSql } from "./issue-display-status.js";
import { externalConversationStateSql } from "./slack-conversation-state.js";
import { and, desc, eq, gte, inArray, isNotNull, isNull, notInArray, or, sql } from "drizzle-orm";
import type { SQL } from "drizzle-orm";
@@ -21,6 +22,7 @@ import {
COMPANY_ARTIFACTS_MAX_QUERY_LENGTH,
ISSUE_PRIORITIES,
ISSUE_STATUSES,
ISSUE_DISPLAY_STATUSES,
SYSTEM_ISSUE_DOCUMENT_KEYS,
type CompanyArtifact,
type CompanySearchArtifactSummary,
@@ -50,6 +52,7 @@ type IssueSearchRow = {
title: string;
description: string | null;
status: string;
displayStatus: string;
priority: string;
assigneeAgentId: string | null;
assigneeUserId: string | null;
@@ -193,6 +196,7 @@ function emptySearchCounts(): Record<CompanySearchCountType, number> {
function emptyFilterOptionCounts(): CompanySearchFilterOptionCounts {
return {
status: {},
displayStatus: {},
priority: {},
assigneeAgentId: {},
assigneeUserId: {},
@@ -224,7 +228,7 @@ function updatedWithinStart(value: string | undefined, now = new Date()): Date |
}
function issueOnlyFiltersActive(query: CompanySearchQuery) {
return query.status.length > 0
return (query.status.length > 0 || query.displayStatus.length > 0)
|| query.priority.length > 0
|| query.assigneeAgentId !== undefined
|| Boolean(query.assigneeUserId)
@@ -236,7 +240,7 @@ function issueOnlyFiltersActive(query: CompanySearchQuery) {
function activeIssueFilters(query: CompanySearchQuery): Array<{ key: CompanySearchIssueFilterKey; values: string[] }> {
const filters: Array<{ key: CompanySearchIssueFilterKey; values: string[] }> = [];
if (query.status.length > 0) filters.push({ key: "status", values: query.status });
if (query.status.length > 0 || query.displayStatus.length > 0) filters.push({ key: "status", values: [...query.status, ...query.displayStatus] });
if (query.assigneeAgentId !== undefined) filters.push({ key: "assigneeAgentId", values: [query.assigneeAgentId ?? "null"] });
if (query.assigneeUserId) filters.push({ key: "assigneeUserId", values: [query.assigneeUserId] });
if (query.projectId) filters.push({ key: "projectId", values: [query.projectId] });
@@ -251,6 +255,7 @@ function queryWithoutFilter(query: CompanySearchQuery, key: CompanySearchIssueFi
return {
...query,
status: key === "status" ? [] : query.status,
displayStatus: key === "status" ? [] : query.displayStatus,
priority: key === "priority" ? [] : query.priority,
assigneeAgentId: key === "assigneeAgentId" ? undefined : query.assigneeAgentId,
assigneeUserId: key === "assigneeUserId" ? undefined : query.assigneeUserId,
@@ -265,6 +270,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery
return {
...query,
status: [],
displayStatus: [],
priority: [],
assigneeAgentId: undefined,
assigneeUserId: undefined,
@@ -277,6 +283,7 @@ function queryWithoutIssueFilters(query: CompanySearchQuery): CompanySearchQuery
function issueFilterConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] {
const conditions: SQL[] = [];
if (omit !== "status" && query.displayStatus.length) conditions.push(inArray(issueDisplayStatusSql(), query.displayStatus));
if (omit !== "status" && query.status.length > 0) {
conditions.push(query.status.length === 1 ? eq(issues.status, query.status[0]!) : inArray(issues.status, query.status));
}
@@ -315,6 +322,7 @@ function issueFilterConditions(companyId: string, query: CompanySearchQuery, omi
// matched-issues CTE (plain columns, no drizzle table references).
function matchedFacetConditions(companyId: string, query: CompanySearchQuery, omit?: CompanySearchIssueFilterKey): SQL[] {
const conditions: SQL[] = [];
if (omit !== "status" && query.displayStatus.length) conditions.push(sql`m.display_status = ANY(${sqlTextArray(query.displayStatus)})`);
if (omit !== "status" && query.status.length > 0) {
conditions.push(sql`m.status = ANY(${sqlTextArray(query.status)})`);
}
@@ -447,6 +455,7 @@ function issueResult(row: IssueSearchRow, prefix: string, normalizedQuery: strin
identifier: row.identifier,
title: row.title,
status: row.status as CompanySearchIssueSummary["status"],
displayStatus: row.displayStatus as CompanySearchIssueSummary["displayStatus"],
priority: row.priority as CompanySearchIssueSummary["priority"],
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
@@ -626,7 +635,7 @@ export function companySearchService(db: Db) {
conditions.length > 0 ? sql`WHERE ${sql.join(conditions, sql` AND `)}` : sql``;
// Count branches must match the result branch's column list; the
// trailing NULLs pad the issue data columns.
const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`;
const countTail = sql`, NULL::uuid, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::text, NULL::uuid, NULL::text, NULL::uuid, NULL::timestamptz, NULL::timestamptz, NULL::double precision, NULL::text[]`;
const branches: SQL[] = [];
const wantResultRows = scopeIncludesIssues(scope)
@@ -652,6 +661,7 @@ export function companySearchService(db: Db) {
(SELECT issue_text.description FROM issues issue_text
WHERE issue_text.id = m.id AND issue_text.company_id = ${companyId}) AS description,
m.status,
m.display_status AS "displayStatus",
m.priority,
m.assignee_agent_id AS "assigneeAgentId",
m.assignee_user_id AS "assigneeUserId",
@@ -690,6 +700,7 @@ export function companySearchService(db: Db) {
GROUP BY 2
`;
branches.push(facetBranch("facet:status", sql`m.status`, "status"));
branches.push(facetBranch("facet:displayStatus", sql`m.display_status`, "status"));
branches.push(facetBranch("facet:priority", sql`m.priority`, "priority"));
branches.push(facetBranch("facet:assigneeAgentId", sql`m.assignee_agent_id`, "assigneeAgentId", [sql`m.assignee_agent_id IS NOT NULL`]));
branches.push(facetBranch("facet:assigneeUserId", sql`m.assignee_user_id`, "assigneeUserId", [sql`m.assignee_user_id IS NOT NULL`]));
@@ -757,6 +768,7 @@ export function companySearchService(db: Db) {
title: row.title,
description: row.description,
status: row.status,
displayStatus: row.displayStatus,
priority: row.priority,
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
@@ -780,6 +792,9 @@ export function companySearchService(db: Db) {
if (row.kind === "type:issue") aggregates.typeCounts.issue = count;
else if (row.kind === "type:comment") aggregates.typeCounts.comment = count;
else if (row.kind === "type:document") aggregates.typeCounts.document = count;
else if (row.kind === "facet:displayStatus" && row.value && (ISSUE_DISPLAY_STATUSES as readonly string[]).includes(row.value)) {
aggregates.filterOptionCounts.displayStatus![row.value as keyof NonNullable<CompanySearchFilterOptionCounts["displayStatus"]>] = count;
}
else if (row.kind === "facet:status" && row.value && (ISSUE_STATUSES as readonly string[]).includes(row.value)) {
aggregates.filterOptionCounts.status[row.value as keyof CompanySearchFilterOptionCounts["status"]] = count;
} else if (row.kind === "facet:priority" && row.value && (ISSUE_PRIORITIES as readonly string[]).includes(row.value)) {
server/src/services/delivery-work-coordinator.ts
+3 -2
View File
@@ -1,4 +1,5 @@
import { subscribeDatabaseWork, databaseWorkPending, reconcileDatabaseWork } from "@paperclipai/db/work-signals";
import { withDatabaseOperation } from "@paperclipai/db";
import { createWorkScheduler } from "./work-scheduler.js";
import { type DeliveryQueue } from "./delivery-work-notifications.js";
import { beginIdleTrackedWork } from "./task-admission.js";
@@ -70,7 +71,7 @@ export function createDeliveryWorkCoordinator(input: {
const attempt = new AbortController();
let runFailed = false;
controller = attempt;
running = Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue))
running = withDatabaseOperation(`delivery.${queue}`, () => Promise.resolve().then(() => reconcileDatabaseWork(input.owner, queue))
.then(() => workerStopped || !(input.canRun() || (task.runDuringDrain && input.canReconcile?.())) ? undefined : task.run(attempt.signal).catch(error => { runFailed = true; throw error; })).then(() => workerStopped ? false : task.hasPending()).then(pending => {
pendingWork = pending || databaseWorkPending(input.owner, queue);
if (pendingWork) schedule(task.retryMs);
@@ -98,7 +99,7 @@ export function createDeliveryWorkCoordinator(input: {
controller = null;
finish();
if (dirty) schedule(0);
});
}));
}
function wake() {
if (workerStopped) return;
server/src/services/durable-chat-wakeup.ts
+18 -2
View File
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { agentWakeupRequests, chatActions, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db";
import { and, eq, inArray, or, sql, type SQLWrapper } from "drizzle-orm";
import { agentWakeupRequests, chatActions, chatConversations, chatDeliveries, chatEndpoints, type Db } from "@paperclipai/db";
import { and, eq, exists, inArray, or, sql, type SQLWrapper } from "drizzle-orm";
import { HttpError } from "../errors.js";
export interface CommittedChatResponseAuthorizationInput {
@@ -233,6 +233,22 @@ export interface DeferredChatWakeScope {
issueId: string;
requestId: string;
}
/** Identify saved input by its canonical receipt, never by a caller's source
* label. This is selection only; the inbound owner rechecks live authority. */
export function deferredChatInputPredicate(db: Db) {
return exists(db.select({ id: chatActions.id }).from(chatActions)
.innerJoin(chatConversations, and(eq(chatConversations.companyId, chatActions.companyId),
eq(chatConversations.id, chatActions.conversationId)))
.innerJoin(chatEndpoints, and(eq(chatEndpoints.companyId, chatActions.companyId),
eq(chatEndpoints.id, chatActions.endpointId), eq(chatEndpoints.id, chatConversations.endpointId)))
.where(and(eq(chatActions.companyId, agentWakeupRequests.companyId), eq(chatActions.id, agentWakeupRequests.id),
eq(chatActions.kind, "inbound_wakeup"), eq(chatActions.status, "processed"),
eq(chatEndpoints.assignedAgentId, agentWakeupRequests.agentId), eq(chatEndpoints.externalExecutionPolicy, "restricted"),
sql`${chatConversations.issueId}::text = ${agentWakeupRequests.payload}->>'issueId'`,
sql`${chatActions.payload}->>'issueId' = ${agentWakeupRequests.payload}->>'issueId'`,
sql`${chatActions.payload}->>'agentId' = ${agentWakeupRequests.agentId}::text`)));
}
const deferredChatWakeOwners = new WeakMap<Db, (input: DeferredChatWakeScope) => Promise<void>>();
/** The live inbound owner reauthorizes the original source. Cleanup owns no chat authority. */
server/src/services/execution-preparation-timing.ts
+16 -13
View File
@@ -1,4 +1,5 @@
import { AsyncLocalStorage } from "node:async_hooks";
import { withDatabaseOperation } from "@paperclipai/db";
import { EXECUTION_PREPARATION_SPAN_ATTRS, getActiveStepContext, runWithRuntimeParent } from "@paperclipai/adapter-utils/acpx-engine/startup-timing";
import { createHash } from "node:crypto";
import { isSpanContextValid, ROOT_CONTEXT, SpanStatusCode, trace, type Context } from "@opentelemetry/api";
@@ -156,7 +157,7 @@ function measure<T>(runId: string, name: string, work: () => T | Promise<T>, ind
ended: false,
details: !independent && previous?.runId === runId ? previous.details : new Map<string, number>(),
};
operation = Promise.resolve().then(() =>
operation = withDatabaseOperation(name, () => Promise.resolve().then(() =>
ownership.run(owned, () => runWithRuntimeParent(owned.parent, work))).then(value => {
try { span?.end(); } catch {}
owned.ended = true;
@@ -166,10 +167,10 @@ function measure<T>(runId: string, name: string, work: () => T | Promise<T>, ind
try { span?.end(); } catch {}
owned.ended = true;
throw error;
});
}));
return operation;
};
if (!endpointEnabled()) return (async () => work())();
if (!endpointEnabled()) return withDatabaseOperation(name, async () => work());
try {
const parent = independent ? undefined : activeParent ?? getRunExecutionTraceContext(runId);
const trigger = parent ? undefined : trace.getActiveSpan()?.spanContext();
@@ -202,20 +203,22 @@ export function withRunWorkspaceFinalizationTiming<T>(runId: string, work: () =>
export function withRunResponsePresentationTiming<T>(runId: string, work: () => T | Promise<T>): Promise<T> {
const owned = ownership.getStore();
if (owned?.runId === runId && owned.phase === "task.response_presentation" && !owned.ended) {
return (async () => work())();
return withDatabaseOperation("task.response_presentation", async () => work());
}
return measure(runId, "task.response_presentation", work, true);
}
export async function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
const owned = ownership.getStore();
if (!endpointEnabled() || !owned || owned.ended) return work();
const count = owned.details.get(phase) ?? 0;
if (count >= 16) return work();
owned.details.set(phase, count + 1);
// A startup step can be more specific than the owning run phase. Keep that
// causal parent while the child publishes its own context to plugin workers.
return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined);
export function measureRunPhase<T>(phase: ExecutionPhase, work: () => T | Promise<T>, counts?: { fileCount?: number; bytes?: number }): Promise<T> {
return withDatabaseOperation(phase, async () => {
const owned = ownership.getStore();
if (!endpointEnabled() || !owned || owned.ended) return work();
const count = owned.details.get(phase) ?? 0;
if (count >= 16) return work();
owned.details.set(phase, count + 1);
// A startup step can be more specific than the owning run phase. Keep that
// causal parent while the child publishes its own context to plugin workers.
return measure(owned.runId, phase, work, false, counts, getActiveStepContext()?.parentContext as Context | undefined);
});
}
export function beginRunPhase(phase: ExecutionPhase): (failed?: boolean) => void {
server/src/services/heartbeat/native-stop-evidence-read.test.ts
+3 -3
View File
@@ -1,6 +1,6 @@
import { randomBytes, randomUUID } from "node:crypto";
import { heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
import { eq, sql } from "drizzle-orm";
import { describe, expect, it, vi } from "vitest";
import { isAcknowledgedNativeStop } from "../acknowledged-native-stop.js";
@@ -31,7 +31,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read
contextSnapshot: { savedInput: bulk }, runnerProfileJson: { savedInput: bulk }, stdoutExcerpt: bulk,
stderrExcerpt: bulk, resultJson: result }).returning();
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
let evidence: Awaited<ReturnType<typeof readNativeStopExecutionEvidence>>;
try { evidence = await readNativeStopExecutionEvidence(db, company.id, id); }
@@ -68,7 +68,7 @@ describe.skipIf(!runControlPostgresSupport.supported)("native Stop evidence read
runtimeMode: "native", status: "cancelled" }).returning();
await db.execute(sql`update ${heartbeatRuns} set result_json = ${JSON.stringify(saved)}::jsonb where id = ${run.id}`);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try {
const evidence = await readNativeStopExecutionEvidence(db, company.id, run.id);
server/src/services/heartbeat/queue.ts
+4 -2
View File
@@ -2299,7 +2299,7 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
sql`${agentWakeupRequests.payload}->>'issueId' = ${issueId ?? ""}`,
));
const retry = readTaskRetryIntent(receipt?.payload);
if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId ||
if (!receipt || retry?.generationId !== receipt.id || retry.canonicalIntentId || retry.queuedChatRequestId ||
retry.actorId !== receipt.requestedByActorId) return null;
const chatRetry = retry.chatActionId && durableRequest?.id === retry.chatActionId &&
durableRequest.failedRunRetry?.failedRunId === retry.sourceRunId;
@@ -4369,7 +4369,9 @@ export function createHeartbeatQueue(db: Db, dependencies: HeartbeatQueueDepende
or(isNull(heartbeatRuns.nativeIssueId), eq(heartbeatRuns.nativeIssueId, issue.id)),
)).then(rows => rows[0] ?? null)
: null;
const canCoalesceComments = !isConversation(issue) && opts.allowRunCoalescing !== false;
// External input owns an exact authorized source batch. Board input
// must retain its separate receipt, including during normal resume.
const canCoalesceComments = !durableRequest && !isConversation(issue) && opts.allowRunCoalescing !== false;
// A resumed receipt already passed admission under this issue lock.
// Consume it with its successor even when chat keeps other messages
// in separate turns, or finalization will deliver it a second time.
server/src/services/heartbeat/run-control.ts
+5
View File
@@ -763,6 +763,10 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
eq(agentWakeupRequests.requestedByActorType, "user"),
));
const intent = readTaskRetryIntent(wake?.payload);
if (intent?.queuedChatRequestId) {
await reconcileTaskChatRetryReceipts(db, { companyId, requestId });
return;
}
if (!wake || !intent || intent.chatActionId || intent.canonicalIntentId || intent.generationId !== wake.id ||
intent.actorId !== wake.requestedByActorId) return;
const issueId = readNonEmptyString(wake.payload?.issueId);
@@ -788,6 +792,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont
scope.requestId ? eq(agentWakeupRequests.id, scope.requestId) : undefined,
eq(agentWakeupRequests.reason, "task_retry"), eq(agentWakeupRequests.status, "deferred_issue_execution"),
sql`${agentWakeupRequests.payload}->'taskRetry'->>'chatActionId' is null`,
sql`${agentWakeupRequests.payload}->'taskRetry'->>'queuedChatRequestId' is null`,
notInArray(issues.status, ["done", "cancelled"])))
.orderBy(asc(agentWakeupRequests.updatedAt)).limit(50);
const chatEnabled = !retries.some(retry => retry.conversationAgentId)
server/src/services/heartbeat/run-state.test.ts
+9 -9
View File
@@ -1,5 +1,5 @@
import { randomBytes, randomUUID } from "node:crypto";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../__tests__/helpers/full-database-response-capture.js";
import type { Db } from "@paperclipai/db";
import {
agents,
@@ -186,13 +186,13 @@ describePostgres("heartbeat run state database wiring", () => {
stdoutExcerpt: payload, stderrExcerpt: payload,
}).returning();
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const state = extracted.createHeartbeatRunState(observed);
const read = await state.getRunExecutionWait(companyId, run.id);
capture.flush();
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(2);
expect(summary.responseValueBytes).toBeLessThan(4096);
expect(read).toMatchObject({ id: run.id, companyId, agentId: agent.id, status: "cancelled", contextSnapshot: context,
@@ -228,7 +228,7 @@ describePostgres("heartbeat run state database wiring", () => {
resultJson: { stdout: payload }, stdoutExcerpt: payload, stderrExcerpt: payload,
}).returning();
const observed = createDb(database.connectionString), events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const state = extracted.createHeartbeatRunState(observed);
@@ -236,7 +236,7 @@ describePostgres("heartbeat run state database wiring", () => {
id: run.id, companyId, scopeKind: "issue", issueId: issue.id, executionWorkspaceId: workspaceId,
});
capture.flush();
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(1);
expect(summary.responseValueBytes).toBeLessThan(1024);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
@@ -260,7 +260,7 @@ describePostgres("heartbeat run state database wiring", () => {
}).returning();
const observed = createDb(database.connectionString);
const events: Array<Record<string, unknown>> = [], now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(observed, { heartbeatRuns }, capture);
try {
const state = extracted.createHeartbeatRunState(observed);
@@ -270,7 +270,7 @@ describePostgres("heartbeat run state database wiring", () => {
runtimeModeResolvedAt: null, startedAt: run.startedAt, createdAt: run.createdAt,
contextSnapshot: { issueId: issue.id }, runnerProfileJson: { adapterDispatch: { adapterType: "paperclip_runner" } } });
capture.flush();
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(2);
expect(summary.responseValueBytes).toBeLessThan(2048);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
@@ -321,7 +321,7 @@ describePostgres("heartbeat run state database wiring", () => {
await state.getRunWorkspaceSettlement(run.id);
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
expect(await state.getRunCleanupState(run.id)).toEqual({
id: run.id, companyId, agentId: agent.id, runtimeMode: run.runtimeMode,
@@ -334,7 +334,7 @@ describePostgres("heartbeat run state database wiring", () => {
contextSnapshot: { timerClaimWasFirstHeartbeat: true },
});
capture.flush();
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBe(2);
expect(summary.responseValueBytes).toBeLessThan(2000);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toEqual(run);
server/src/services/issue-display-status.ts
+37
View File
@@ -0,0 +1,37 @@
import { sql, type SQL } from "drizzle-orm";
import { issues } from "@paperclipai/db";
import { PROVIDER_QUOTA_MONITOR_SERVICE_NAME, type IssueDisplayStatus } from "@paperclipai/shared";
/** Read-only projection, shared by task reads, search, filters and counts.
* The persisted deadline is the scheduler's source of truth. Locks and recovery
* take precedence; a retained monitor alone must not disguise active work.
*/
export function issueDisplayStatusSql(): SQL<IssueDisplayStatus> {
// Drizzle removes column qualifiers in a single-table SELECT. Keep these
// outer references explicitly qualified so correlated reads cannot bind to
// the run/recovery table's own id or company_id instead.
const issueId = sql`"issues"."id"`;
const companyId = sql`"issues"."company_id"`;
return sql<IssueDisplayStatus>`CASE WHEN
${issues.status} = 'in_progress'
AND ${issues.assigneeAgentId} IS NOT NULL AND ${issues.assigneeUserId} IS NULL
AND ${issues.monitorNextCheckAt} IS NOT NULL
AND ${issues.executionRunId} IS NULL AND ${issues.checkoutRunId} IS NULL
AND ${issues.executionState}->>'currentParticipant' IS NULL
AND COALESCE(${issues.executionState}#>>'{monitor,status}', 'scheduled') = 'scheduled'
AND ${issues.executionPolicy}->'monitor' IS NOT NULL
AND COALESCE(${issues.executionPolicy}#>>'{monitor,serviceName}', '') <> ${PROVIDER_QUOTA_MONITOR_SERVICE_NAME}
AND NOT EXISTS (
SELECT 1 FROM heartbeat_runs display_run
WHERE display_run.company_id = ${companyId}
AND display_run.issue_id = ${issueId}
AND display_run.status IN ('queued', 'running')
)
AND NOT EXISTS (
SELECT 1 FROM issue_recovery_actions display_recovery
WHERE display_recovery.company_id = ${companyId}
AND display_recovery.source_issue_id = ${issueId}
AND display_recovery.status IN ('active', 'escalated')
)
THEN 'monitoring' ELSE ${issues.status} END`;
}
server/src/services/issue-summaries.ts
+2
View File
@@ -1,3 +1,4 @@
import { issueDisplayStatusSql } from "./issue-display-status.js";
import { and, eq, inArray, isNull, type SQL } from "drizzle-orm";
import { issues, type Db } from "@paperclipai/db";
import { ISSUE_SUMMARIES_MAX_IDS } from "@paperclipai/shared";
@@ -18,6 +19,7 @@ export async function listIssueSummaries(
title: issues.title,
identifier: issues.identifier,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
updatedAt: issues.updatedAt,
externalConversationState: externalConversationStateSql(),
projectId: issues.projectId,
server/src/services/issues.ts
+31 -4
View File
@@ -1,3 +1,5 @@
import type { IssueDisplayStatus } from "@paperclipai/shared";
import { issueDisplayStatusSql } from "./issue-display-status.js";
import { notifyStrandedAgentStatus, notifyActiveRunSilence } from "./live-run-recovery-notifications.js";
import { finishedIdleWake } from "./idle-work-proof.js";
import { notifyDeferredExecutionEligibility } from "./deferred-execution-notifications.js";
@@ -1900,6 +1902,7 @@ export function parseStatusFilter(
export interface IssueFilters {
attention?: "blocked";
status?: string | readonly string[];
displayStatus?: string | readonly string[];
/**
* Filter by assignee agent ID.
* - `string` (UUID): match issues assigned to that agent.
@@ -1944,14 +1947,14 @@ export interface IssueFilters {
q?: string;
limit?: number;
offset?: number;
sortField?: "updated" | "id";
sortField?: "updated" | "created" | "id";
afterId?: string;
sortDir?: "asc" | "desc";
/** ISO 8601 timestamp — only return issues with updatedAt strictly after this value. */
updatedSince?: string;
}
type IssueRow = typeof issues.$inferSelect & { externalConversationState?: "active" | "waiting" | null };
type IssueRow = typeof issues.$inferSelect & { displayStatus?: IssueDisplayStatus; externalConversationState?: "active" | "waiting" | null };
type IssueLabelRow = typeof labels.$inferSelect;
type IssuePlanDecompositionRow = typeof issuePlanDecompositions.$inferSelect;
type IssueActiveRunRow = {
@@ -3218,6 +3221,11 @@ function issueListOrderBy(
},
) {
if (sortField === "id") return [sortDir === "desc" ? desc(issues.id) : asc(issues.id)];
if (sortField === "created") {
return sortDir === "asc"
? [asc(issues.createdAt), asc(issues.id)]
: [desc(issues.createdAt), desc(issues.id)];
}
const canonicalLastActivityAt = issueCanonicalLastActivityAtExpr(companyId);
if (sortField === "updated") {
const activityOrder =
@@ -3603,6 +3611,7 @@ function appendBlockerAttentionEdges(
}
type IssueRelationSummaryRow = {
displayStatus?: IssueDisplayStatus;
relatedId: string;
identifier: string | null;
title: string;
@@ -3620,6 +3629,7 @@ function summarizeIssueRelationRow(
identifier: row.identifier,
title: row.title,
status: row.status as IssueRelationIssueSummary["status"],
displayStatus: row.displayStatus,
priority: row.priority as IssueRelationIssueSummary["priority"],
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
@@ -3735,6 +3745,7 @@ async function terminalExplicitBlockersByRoot(
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
@@ -5039,6 +5050,7 @@ const issueListSelect = {
END
`,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
statusVersion: issues.statusVersion,
lastStatusDecisionId: issues.lastStatusDecisionId,
workMode: issues.workMode,
@@ -5279,6 +5291,7 @@ async function blockedByMapForIssues(
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
@@ -5301,6 +5314,7 @@ async function blockedByMapForIssues(
identifier: row.identifier,
title: row.title,
status: row.status as IssueRelationIssueSummary["status"],
displayStatus: row.displayStatus,
priority: row.priority as IssueRelationIssueSummary["priority"],
assigneeAgentId: row.assigneeAgentId,
assigneeUserId: row.assigneeUserId,
@@ -6420,6 +6434,8 @@ async function blockedInboxIssueConditions(
filters?.lowTrustBoundary,
);
if (lowTrustCondition) conditions.push(lowTrustCondition);
const displayStatuses = parseStatusFilter(filters?.displayStatus);
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
const statuses = parseStatusFilter(filters?.status);
if (statuses.length > 0) {
conditions.push(
@@ -6872,7 +6888,7 @@ export function issueService(db: Db) {
async function getIssueByUuid(id: string) {
const row = await retryIdempotentDatabaseOperation(() => db
.select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() })
.select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() })
.from(issues)
.where(eq(issues.id, id))
.then((rows) => rows[0] ?? null));
@@ -6883,7 +6899,7 @@ export function issueService(db: Db) {
async function getIssueByIdentifier(identifier: string) {
const row = await retryIdempotentDatabaseOperation(() => db
.select({ ...getTableColumns(issues), externalConversationState: externalConversationStateSql() })
.select({ ...getTableColumns(issues), displayStatus: issueDisplayStatusSql(), externalConversationState: externalConversationStateSql() })
.from(issues)
.where(eq(issues.identifier, identifier.toUpperCase()))
.then((rows) => rows[0] ?? null));
@@ -7482,6 +7498,7 @@ export function issueService(db: Db) {
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
@@ -7502,6 +7519,7 @@ export function issueService(db: Db) {
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
@@ -8193,6 +8211,9 @@ export function issueService(db: Db) {
if (filters?.sortField === "id" && filters.attention) {
throw unprocessable("ID ordering is not supported for blocked attention lists");
}
if (filters?.sortField === "created" && filters.attention) {
throw unprocessable("Creation-time ordering is not supported for blocked attention lists");
}
if (filters?.afterId !== undefined && (
!isUuidLike(filters.afterId) || filters.sortField !== "id" ||
filters.sortDir !== "asc" || (filters.offset ?? 0) !== 0
@@ -8281,6 +8302,8 @@ export function issueService(db: Db) {
filters?.lowTrustBoundary,
);
if (lowTrustCondition) conditions.push(lowTrustCondition);
const displayStatuses = parseStatusFilter(filters?.displayStatus);
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
const statuses = parseStatusFilter(filters?.status);
if (statuses.length === 1) {
conditions.push(eq(issues.status, statuses[0]));
@@ -8574,6 +8597,8 @@ export function issueService(db: Db) {
conditions.push(nonIdleSlackIssueCondition());
}
}
const displayStatuses = parseStatusFilter(filters?.displayStatus);
if (displayStatuses.length) conditions.push(inArray(issueDisplayStatusSql(), displayStatuses));
const statuses = parseStatusFilter(filters?.status);
if (statuses.length === 1)
conditions.push(eq(issues.status, statuses[0]!));
@@ -8895,6 +8920,7 @@ export function issueService(db: Db) {
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
@@ -9522,6 +9548,7 @@ export function issueService(db: Db) {
identifier: issues.identifier,
title: issues.title,
status: issues.status,
displayStatus: issueDisplayStatusSql(),
priority: issues.priority,
assigneeAgentId: issues.assigneeAgentId,
assigneeUserId: issues.assigneeUserId,
server/src/services/native-runtime/__tests__/helpers/committed-replay.ts
+3 -3
View File
@@ -1,5 +1,5 @@
import { heartbeatRuns, type Db } from "@paperclipai/db";
import { createDatabaseResponseCapture, installDatabaseResponseCapture } from "@paperclipai/db/response-capture";
import { createFullDatabaseResponseCapture, installDatabaseResponseCapture } from "../../../../__tests__/helpers/full-database-response-capture.js";
import { eq } from "drizzle-orm";
import { expect } from "vitest";
@@ -21,11 +21,11 @@ export async function verifyCommittedReplayPayloadReads(
const [before] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
const events: Array<Record<string, unknown>> = [];
const now = Date.now();
const capture = createDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
const capture = createFullDatabaseResponseCapture({ start: now - 100, end: now + 59_000 }, event => events.push(event));
installDatabaseResponseCapture(db, { heartbeatRuns }, capture);
try { await replay(); }
finally { capture.flush(); }
const summary = events.find(event => event.event === "paperclip.db_response_capture.summary.v1")!;
const summary = events.find(event => event.event === "paperclip.test_db_response_capture.summary.v1")!;
expect(summary.completedQueries).toBeGreaterThan(0);
expect(summary.responseValueBytes).toBeLessThan(100_000);
const [after] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, runId));
server/src/services/native-runtime/assigned-mcp-tools.test.ts
+44
View File
@@ -19,6 +19,50 @@ describe("assigned MCP runner tools", () => {
const searchName = "paperclip_search_assigned_tools";
const callName = "paperclip_call_assigned_tool";
it.each(["", " \t\n"])("discovers and calls assigned tools through MCP with a blank schema reference %j", async schemaTool => {
const github = descriptor("github_bot:read_pull_request");
const f = fixture([github, descriptor("calendar.search")]);
const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" });
const bridge = await startOpenCodeMcpBridge({ tools: assigned.definitions(), handler: call => assigned.execute(call) });
let id = 0;
const call = async (name: string, args: Record<string, unknown>) => {
const response = await fetch(bridge.url, {
method: "POST", headers: { Authorization: `Bearer ${bridge.secret}`, "Content-Type": "application/json" },
body: JSON.stringify({ jsonrpc: "2.0", id: ++id, method: "tools/call", params: { name, arguments: args } }),
});
const result = (await response.json()).result;
expect(result.isError).not.toBe(true);
return JSON.parse(result.content[0].text);
};
try {
const page = await call(searchName, { query: "github", schemaTool });
expect(page.tools).toHaveLength(1);
expect(page.tools[0]).toMatchObject({ description: expect.stringContaining(github.name), inputSchema: github.parametersSchema });
expect(await call(searchName, { query: "github", schemaTool, schemaOffset: 0 })).toEqual(page);
expect((await call(searchName, { query: "", schemaTool })).tools).toHaveLength(2);
await call(callName, { name: page.tools[0].name, arguments: { query: "fixture" } });
expect(f.executeTool).toHaveBeenCalledExactlyOnceWith(expect.objectContaining({
tool: github.name, parameters: { query: "fixture" }, gatewayPublicId: f.gatewayPublicId, sessionToken: f.bearerToken,
}));
f.listToolsForNamedGateway.mockResolvedValue([]);
expect(await call(searchName, { query: "github", schemaTool })).toEqual({ tools: [], nextOffset: null });
} finally {
await bridge.close();
}
});
it("explains how to recover from an unknown schema reference or an offset without a reference", async () => {
const f = fixture([descriptor("github_bot:read_pull_request")]);
const assigned = await createAssignedMcpTools({ ...f, delivery: "on_demand" });
await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool: "unknown" } }))
.rejects.toThrow("Omit schemaTool to search assigned tools");
for (const schemaTool of [undefined, "", " \t\n"]) {
await expect(assigned.execute({ tool: searchName, arguments: { query: "github", schemaTool, schemaOffset: 1 } }))
.rejects.toThrow("A positive schemaOffset requires an exact inputSchemaRef in schemaTool");
}
expect(f.executeTool).not.toHaveBeenCalled();
});
it.each([undefined, "on_demand"] as const)("advertises usable page limits through MCP discovery (%s)", async delivery => {
const f = fixture(Array.from({ length: 25 }, (_, i) => descriptor(`operation_${i}`)));
const assigned = await createAssignedMcpTools({ ...f, delivery });
server/src/services/native-runtime/assigned-mcp-tools.ts
+9 -6
View File
@@ -21,8 +21,8 @@ const ON_DEMAND_TOOLS: ToolDefinition[] = [
query: { type: "string", maxLength: 200 },
offset: { type: "integer", minimum: 0 },
limit: { type: "integer", minimum: 1, maximum: 20, default: 5, description: "Maximum results per page (1 to 20; default 5)." },
schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result." },
schemaOffset: { type: "integer", minimum: 0 },
schemaTool: { type: "string", description: "Exact inputSchemaRef from a search result. Omit or leave blank for a normal tool search." },
schemaOffset: { type: "integer", minimum: 0, description: "Schema chunk offset. A positive offset requires an exact inputSchemaRef in schemaTool." },
},
required: ["query"],
},
@@ -116,14 +116,17 @@ export async function createAssignedMcpTools(input: {
const offset = args.offset ?? 0;
const limit = args.limit ?? 5;
const schemaOffset = args.schemaOffset ?? 0;
const hasSchemaTool = typeof args.schemaTool === "string" && args.schemaTool.trim().length > 0;
if (typeof args.query !== "string" || args.query.length > 200 ||
typeof offset !== "number" || !Number.isSafeInteger(offset) || offset < 0 ||
typeof limit !== "number" || !Number.isSafeInteger(limit) || limit < 1 || limit > 20 ||
(args.schemaTool !== undefined && typeof args.schemaTool !== "string") ||
typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0 ||
(args.schemaOffset !== undefined && args.schemaTool === undefined)) {
typeof schemaOffset !== "number" || !Number.isSafeInteger(schemaOffset) || schemaOffset < 0) {
throw new Error("assigned_mcp_tool_invalid_arguments");
}
if (schemaOffset > 0 && !hasSchemaTool) {
throw new Error("assigned_mcp_tool_invalid_arguments: A positive schemaOffset requires an exact inputSchemaRef in schemaTool. Omit schemaTool and schemaOffset to search assigned tools.");
}
// Intersect fresh grants with this session's pinned catalog. Revocations
// take effect immediately; newly assigned tools require a new session.
const current = new Map(deduplicateToolDescriptors(await input.gateway.listToolsForNamedGateway({
@@ -135,9 +138,9 @@ export async function createAssignedMcpTools(input: {
return fresh && permits(tool) && permits(tool, currentWorkMode) &&
permits(fresh) && permits(fresh, currentWorkMode);
});
if (typeof args.schemaTool === "string") {
if (hasSchemaTool) {
const selected = authorized.find(([name]) => name === args.schemaTool);
if (!selected) throw new Error("assigned_mcp_tool_unknown");
if (!selected) throw new Error("assigned_mcp_tool_unknown: schemaTool must match an authorized inputSchemaRef. Omit schemaTool to search assigned tools, then use an exact inputSchemaRef from the results.");
const schema = JSON.stringify(selected[1].parametersSchema);
const end = schemaOffset + schemaChunkCharacters;
return {
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff. Show more